From aac32a2b1a3a6d198d9e2296f73cbe3264b9cc82 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 18 Jul 2026 11:54:12 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: promotion/ronpo-qwen3-8b-fair-ronpo-full-expect-s42 Source: Original Platform --- .gitattributes | 36 + README.md | 12 + all_results.json | 9 + chat_template.jinja | 89 + config.json | 71 + config.yaml | 58 + experiment/evaluator_lock.json | 85 + experiment/selection_lock.json | 99 + experiment/sweep_grid.json | 46 + experiment/training_status.json | 19 + generation_config.json | 12 + model.safetensors | 3 + tokenizer.json | 3 + tokenizer_config.json | 30 + train_results.json | 9 + trainer_state.json | 4903 +++++++++++++++++++++++++++++++ training_args.bin | 3 + training_status.json | 19 + 18 files changed, 5506 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 config.yaml create mode 100644 experiment/evaluator_lock.json create mode 100644 experiment/selection_lock.json create mode 100644 experiment/sweep_grid.json create mode 100644 experiment/training_status.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin create mode 100644 training_status.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..a2ef8a4 --- /dev/null +++ b/README.md @@ -0,0 +1,12 @@ +--- +base_model: Qwen/Qwen3-8B +library_name: transformers +tags: +- preference-optimization +- ronpo +- qwen3 +--- + +# Qwen3-8B fair-demo checkpoint: ronpo_full_expect + +Validation-selected candidate: `ronpo_full_a`. Exact evaluator, selection, sweep, and training metadata are stored under `experiment/`. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..cfa9578 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.2860548271752086, + "total_flos": 0.0, + "train_loss": 0.430257171690464, + "train_runtime": 7318.8237, + "train_samples": 50340, + "train_samples_per_second": 1.968, + "train_steps_per_second": 0.123 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..2ceef7c --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +model_name_or_path: /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218 +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/ronpo_full_expect: 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: ronpo +eta: 0.0075 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.25 +ronpo_tau: 0.1 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.075 +preference_sft_weight: 0.0075 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 1.0e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.15 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 42 +gradient_accumulation_steps: 16 +gradient_checkpointing: true +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 1 +per_device_eval_batch_size: 1 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/fair_demo_20260715/sweep/candidates/ronpo_full_a +run_name: qwen3-8b-fair-demo-ronpo_full_a diff --git a/experiment/evaluator_lock.json b/experiment/evaluator_lock.json new file mode 100644 index 0000000..b406c9a --- /dev/null +++ b/experiment/evaluator_lock.json @@ -0,0 +1,85 @@ +{ + "status": "LOCKED_BEFORE_ANY_NEW_METHOD_RANKING", + "locked_at": "2026-07-15T20:20:29+09:00", + "objective_signals": [ + "skywork", + "armo_safety", + "length_conciseness" + ], + "objective_semantics": [ + "helpfulness", + "safety", + "conciseness" + ], + "primary": { + "name": "open_weight_panel_mean_prompt_worst_vs_base", + "definition": "For each prompt and objective, average win=1/tie=0.5/loss=0 over two A/B positions and two judges; take the minimum objective, then mean over prompts.", + "judges": [ + { + "model": "openai/gpt-oss-120b", + "revision": "b5c939de8f754692c1647ca79fbf85e8c1e70f8a" + }, + { + "model": "Qwen/Qwen3-32B", + "revision": "9216db5781bf21249d130ec9da846c4624c16137" + } + ], + "decode": { + "temperature": 0.0, + "top_p": 1.0, + "max_new_tokens": 512, + "seed": 42 + }, + "position_swap": true, + "validation_selection_rule": "Highest eligible mean prompt-level worst panel score within each method; an exact numeric tie is broken by candidate_id lexical order." + }, + "secondary": { + "normalization": "For each locked signal, divide each evaluation prompt's candidate-minus-base raw delta by the population SD of the diagnostic base and matched-control scores; take the prompt-level minimum across objectives, then the mean. No per-prompt min-max.", + "diagnostic_control_scale": { + "skywork": 22.807301785782975, + "armo_safety": 0.3098083353203591, + "length_conciseness": 1.5891969646897472 + }, + "reward_signal_provenance": { + "skywork": { + "model": "Skywork/Skywork-Reward-V2-Llama-3.1-8B", + "revision": "cba2f842f3f1af2f1b2f0d35e794d789976390c5", + "semantics": "helpfulness" + }, + "armo_safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "revision": "eb2676d20da2f2d41082289d23c59b9f7427f955", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "length_conciseness": { + "deterministic": "-log1p(response_word_count)", + "tokenization": "Python str.split whitespace words" + } + }, + "report_raw_paired_deltas": true + }, + "bootstrap": { + "paired_prompt_resamples": 2000, + "seed": 42, + "interval": "percentile_95" + }, + "power": { + "target_absolute_effect": 0.05, + "paired_sd": 0.13831629563357775, + "required_prompts_80pct_power": 61, + "planned_fresh_test_prompts": 1024, + "alpha_two_sided": 0.05, + "power": 0.8 + }, + "fresh_test_source": { + "dataset": "HuggingFaceH4/ultrachat_200k", + "revision": "8049631c405ae6576f93f445c6b8166f76f5505a", + "split": "test_sft" + }, + "diagnostic_summary_sha256": "e365ed814926ac5c17777e0fc65d0299a45b81bd390542889bbd55b5453d4ffe", + "judge_diagnostic_lock_sha256": "fce2d97806053cabdf3d358806f4baa80df7f89d71d8c1c298883327d4e02635", + "prereg_sha256": "71cbd1ac17c3e807f2ece1cb0a82374bbe74edb654c2845b97c3222ed680f0d0", + "method_ranking_computed": false, + "spent_sealed_split_touched": false +} diff --git a/experiment/selection_lock.json b/experiment/selection_lock.json new file mode 100644 index 0000000..bae8b9c --- /dev/null +++ b/experiment/selection_lock.json @@ -0,0 +1,99 @@ +{ + "status": "VALIDATION_SELECTION_LOCKED_BEFORE_FRESH_TEST", + "locked_at": "2026-07-15T20:37:11+09:00", + "selection_metric": "open_weight_panel_mean_prompt_worst_vs_base", + "tie_break": "candidate_id lexical order", + "selected_by_method": { + "dpo": { + "candidate_id": "dpo_b", + "validation_primary": 0.40625, + "validation_primary_ci95": [ + 0.3837890625, + 0.427734375 + ], + "eligible_candidates": [ + "dpo_a", + "dpo_b" + ] + }, + "ht_mnpo_helpfulness": { + "candidate_id": "ht_help_b", + "validation_primary": 0.4072265625, + "validation_primary_ci95": [ + 0.3828125, + 0.4296875 + ], + "eligible_candidates": [ + "ht_help_b" + ] + }, + "ht_mnpo_safety": { + "candidate_id": "ht_safety_a", + "validation_primary": 0.4150390625, + "validation_primary_ci95": [ + 0.3925537109375, + 0.4384765625 + ], + "eligible_candidates": [ + "ht_safety_a", + "ht_safety_b" + ] + }, + "inpo_avg": { + "candidate_id": "inpo_b", + "validation_primary": 0.40625, + "validation_primary_ci95": [ + 0.3818359375, + 0.427734375 + ], + "eligible_candidates": [ + "inpo_b" + ] + }, + "ronpo_full_expect": { + "candidate_id": "ronpo_full_a", + "validation_primary": 0.4033203125, + "validation_primary_ci95": [ + 0.37890625, + 0.42580566406249987 + ], + "eligible_candidates": [ + "ronpo_full_a", + "ronpo_full_b" + ] + }, + "ronpo_k_only": { + "candidate_id": "ronpo_top_a", + "validation_primary": 0.412109375, + "validation_primary_ci95": [ + 0.3876953125, + 0.43557128906249987 + ], + "eligible_candidates": [ + "ronpo_top_a" + ] + }, + "sppo_avg": { + "candidate_id": "sppo_b", + "validation_primary": 0.4111328125, + "validation_primary_ci95": [ + 0.3866943359375, + 0.4345703125 + ], + "eligible_candidates": [ + "sppo_b" + ] + } + }, + "selected_ronpo_overall": "ronpo_top_a", + "failed_methods": [ + "ht_mnpo_conciseness", + "ipo", + "simpo" + ], + "panel_summary_sha256": "8850f23e16cadf56a99e3d562d224ce5337ab36fa588d8a843382853e6ad17bd", + "evaluator_lock_sha256": "ab7b12c84932ed57acfec7edbcbc1ff1df5e9acd203869b1458343adf6e5b2c8", + "grid_sha256": "abfd982ef10c3bc71364752241150286ebf06be5e97f087596adf80e55d459e0", + "fresh_test_opened": false, + "spent_sealed_split_touched": false +} diff --git a/experiment/sweep_grid.json b/experiment/sweep_grid.json new file mode 100644 index 0000000..1ee759f --- /dev/null +++ b/experiment/sweep_grid.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "status": "frozen_before_training_and_validation_ranking", + "frozen_at_kst": "2026-07-15T09:01:45+09:00", + "seed": 42, + "common": { + "optimizer_steps": 900, + "effective_batch_size": 16, + "per_device_train_batch_size": 1, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "bf16": true, + "attn_implementation": "sdpa", + "cudnn_sdpa": false, + "lr_scheduler_type": "cosine", + "max_length": 2048, + "max_prompt_length": 1800, + "save_total_limit": 1, + "wandb_entity": "promotion-kim", + "wandb_project": "mnpo", + "wandb_group": "qwen3-8b-fair-demo-20260715" + }, + "budget_rule": "Exactly two configs per reported method; no extension after validation rankings are visible.", + "candidates": [ + {"id":"ronpo_full_a","method":"ronpo_full_expect","dataset":"ronpo_full_expect_kall","learning_rate":1e-7,"warmup_ratio":0.15,"ronpo_alpha":0.25,"ronpo_tau":0.10,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075,"expected_support_k":0}, + {"id":"ronpo_full_b","method":"ronpo_full_expect","dataset":"ronpo_full_expect_k6","learning_rate":2.5e-7,"warmup_ratio":0.20,"ronpo_alpha":0.35,"ronpo_tau":0.05,"reference_anchor_weight":0.10,"preference_sft_weight":0.01,"expected_support_k":6}, + {"id":"ronpo_top_a","method":"ronpo_k_only","dataset":"ronpo_k_only_k1","learning_rate":5e-8,"warmup_ratio":0.20,"ronpo_alpha":0.15,"ronpo_tau":0.10,"reference_anchor_weight":0.10,"preference_sft_weight":0.01,"expected_support_k":1}, + {"id":"ronpo_top_b","method":"ronpo_k_only","dataset":"ronpo_k_only_k2","learning_rate":1e-7,"warmup_ratio":0.15,"ronpo_alpha":0.25,"ronpo_tau":0.10,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075,"expected_support_k":2}, + {"id":"dpo_a","method":"dpo","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.10,"dpo_beta":0.05,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"dpo_b","method":"dpo","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.15,"dpo_beta":0.10,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ipo_a","method":"ipo","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.15,"dpo_beta":0.20,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ipo_b","method":"ipo","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.20,"dpo_beta":0.50,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075}, + {"id":"simpo_a","method":"simpo","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.10,"simpo_beta":1.0,"simpo_gamma":0.3,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"simpo_b","method":"simpo","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.15,"simpo_beta":2.0,"simpo_gamma":0.6,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"sppo_a","method":"sppo_avg","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.15,"eta":0.005,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"sppo_b","method":"sppo_avg","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.20,"eta":0.010,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075}, + {"id":"inpo_a","method":"inpo_avg","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.15,"eta":0.050,"ratio":0.3333,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"inpo_b","method":"inpo_avg","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.20,"eta":0.100,"ratio":0.3333,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075}, + {"id":"ht_help_a","method":"ht_mnpo_helpfulness","dataset":"ht_mnpo_helpfulness","learning_rate":2.5e-7,"warmup_ratio":0.10,"eta":0.005,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"ht_help_b","method":"ht_mnpo_helpfulness","dataset":"ht_mnpo_helpfulness","learning_rate":5e-7,"warmup_ratio":0.15,"eta":0.010,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ht_safety_a","method":"ht_mnpo_safety","dataset":"ht_mnpo_safety","learning_rate":2.5e-7,"warmup_ratio":0.10,"eta":0.005,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"ht_safety_b","method":"ht_mnpo_safety","dataset":"ht_mnpo_safety","learning_rate":5e-7,"warmup_ratio":0.15,"eta":0.010,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ht_concise_a","method":"ht_mnpo_conciseness","dataset":"ht_mnpo_conciseness","learning_rate":2.5e-7,"warmup_ratio":0.10,"eta":0.005,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"ht_concise_b","method":"ht_mnpo_conciseness","dataset":"ht_mnpo_conciseness","learning_rate":5e-7,"warmup_ratio":0.15,"eta":0.010,"reference_anchor_weight":0.05,"preference_sft_weight":0.005} + ] +} diff --git a/experiment/training_status.json b/experiment/training_status.json new file mode 100644 index 0000000..244a146 --- /dev/null +++ b/experiment/training_status.json @@ -0,0 +1,19 @@ +{ + "status": "completed", + "candidate_id": "ronpo_full_a", + "method": "ronpo_full_expect", + "seed": 42, + "gpu": 1, + "config": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/fair_demo_20260715/sweep/candidates/ronpo_full_a/config.yaml", + "dataset": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/ronpo_full_expect", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "234ea82cbcb3", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/234ea82cbcb3", + "launcher": "supplemental_one_candidate", + "started_at": "2026-07-15T18:02:38+09:00", + "returncode": 0, + "model_complete": true, + "finite_results": true, + "completed_at": "2026-07-15T20:05:18+09:00" +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..e78bb6b --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd843af4d04d547751b4f721eb1dbea379b68fe042361ad8df4db46fcf8c5e2a +size 16381517208 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..e2cb12d --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": true, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..cfa9578 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.2860548271752086, + "total_flos": 0.0, + "train_loss": 0.430257171690464, + "train_runtime": 7318.8237, + "train_samples": 50340, + "train_samples_per_second": 1.968, + "train_steps_per_second": 0.123 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..4761b5d --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,4903 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.2860548271752086, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "drift/chosen_abs": 0.2889712452888489, + "drift/rejected_abs": 0.17483893036842346, + "epoch": 0.0015891934843067143, + "grad_norm": 0.8203125, + "learning_rate": 2.962962962962963e-09, + "logits/chosen": -2.2284557819366455, + "logits/rejected": -2.154376983642578, + "logps/chosen": -0.2503115236759186, + "logps/rejected": -0.2550564706325531, + "loss": 0.26546722650527954, + "loss/core": 0.23584802448749542, + "loss/preference_sft": 0.2586488723754883, + "loss/reference_anchor": 0.3690585196018219, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.8897125720977783, + "rewards/margins": 1.1418126821517944, + "rewards/rejected": 1.7478997707366943, + "ronpo/logit": 0.11418129503726959, + "ronpo/residual": 0.11854199320077896, + "ronpo/residual_abs": 0.20326364040374756, + "ronpo/target": -0.004360715392976999, + "ronpo/target_abs": 0.03454750031232834, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 5 + }, + { + "drift/chosen_abs": 0.25061291456222534, + "drift/rejected_abs": 0.1725693792104721, + "epoch": 0.0031783869686134287, + "grad_norm": 5.34375, + "learning_rate": 6.666666666666666e-09, + "logits/chosen": -1.9815419912338257, + "logits/rejected": -1.9222148656845093, + "logps/chosen": -0.2649174630641937, + "logps/rejected": -0.27258291840553284, + "loss": 0.16829581558704376, + "loss/core": 0.14190886914730072, + "loss/preference_sft": 0.2717735469341278, + "loss/reference_anchor": 0.3246484398841858, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.5059151649475098, + "rewards/margins": 0.7813706398010254, + "rewards/rejected": 1.7245442867279053, + "ronpo/logit": 0.07813705503940582, + "ronpo/residual": 0.06967471539974213, + "ronpo/residual_abs": 0.15364500880241394, + "ronpo/target": 0.008462338708341122, + "ronpo/target_abs": 0.041159503161907196, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 10 + }, + { + "drift/chosen_abs": 0.14762550592422485, + "drift/rejected_abs": 0.08012887835502625, + "epoch": 0.004767580452920143, + "grad_norm": 7.15625, + "learning_rate": 1.0370370370370369e-08, + "logits/chosen": -2.2923481464385986, + "logits/rejected": -2.2468485832214355, + "logps/chosen": -0.2759386897087097, + "logps/rejected": -0.2823220193386078, + "loss": 0.04410820081830025, + "loss/core": 0.03747672960162163, + "loss/preference_sft": 0.27749139070510864, + "loss/reference_anchor": 0.06067045405507088, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.4754669666290283, + "rewards/margins": 0.675328254699707, + "rewards/rejected": 0.8001387715339661, + "ronpo/logit": 0.06753282248973846, + "ronpo/residual": 0.08304200321435928, + "ronpo/residual_abs": 0.10613694041967392, + "ronpo/target": -0.015509175136685371, + "ronpo/target_abs": 0.038143090903759, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 15 + }, + { + "drift/chosen_abs": 0.14132478833198547, + "drift/rejected_abs": 0.06987211853265762, + "epoch": 0.006356773937226857, + "grad_norm": 1.15625, + "learning_rate": 1.4074074074074075e-08, + "logits/chosen": -2.2929229736328125, + "logits/rejected": -2.218808650970459, + "logps/chosen": -0.27541348338127136, + "logps/rejected": -0.27921879291534424, + "loss": 0.02784859575331211, + "loss/core": 0.02186846174299717, + "loss/preference_sft": 0.27790480852127075, + "loss/reference_anchor": 0.05194469541311264, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 1.4130252599716187, + "rewards/margins": 0.7176119089126587, + "rewards/rejected": 0.6954135894775391, + "ronpo/logit": 0.07176118344068527, + "ronpo/residual": 0.07096967101097107, + "ronpo/residual_abs": 0.08470381796360016, + "ronpo/target": 0.0007915133610367775, + "ronpo/target_abs": 0.03379639983177185, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 20 + }, + { + "drift/chosen_abs": 0.23398974537849426, + "drift/rejected_abs": 0.07662718743085861, + "epoch": 0.007945967421533572, + "grad_norm": 1.1171875, + "learning_rate": 1.7777777777777777e-08, + "logits/chosen": -2.3544507026672363, + "logits/rejected": -2.392819881439209, + "logps/chosen": -0.24027124047279358, + "logps/rejected": -0.24880602955818176, + "loss": 0.3059253394603729, + "loss/core": 0.2891264259815216, + "loss/preference_sft": 0.2509022355079651, + "loss/reference_anchor": 0.19889584183692932, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.3385119438171387, + "rewards/margins": 1.572943925857544, + "rewards/rejected": 0.765568196773529, + "ronpo/logit": 0.15729442238807678, + "ronpo/residual": 0.15887948870658875, + "ronpo/residual_abs": 0.17786964774131775, + "ronpo/target": -0.0015850653871893883, + "ronpo/target_abs": 0.03792804479598999, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 25 + }, + { + "drift/chosen_abs": 0.20628702640533447, + "drift/rejected_abs": 0.08774978667497635, + "epoch": 0.009535160905840286, + "grad_norm": 316.0, + "learning_rate": 2.148148148148148e-08, + "logits/chosen": -2.2282731533050537, + "logits/rejected": -2.1223556995391846, + "logps/chosen": -0.2855835556983948, + "logps/rejected": -0.2826509475708008, + "loss": 0.18014441430568695, + "loss/core": 0.16145095229148865, + "loss/preference_sft": 0.2912216782569885, + "loss/reference_anchor": 0.2201242446899414, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.0624454021453857, + "rewards/margins": 1.1857421398162842, + "rewards/rejected": 0.876703143119812, + "ronpo/logit": 0.11857421696186066, + "ronpo/residual": 0.11068874597549438, + "ronpo/residual_abs": 0.14063629508018494, + "ronpo/target": 0.007885478436946869, + "ronpo/target_abs": 0.0422462597489357, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 30 + }, + { + "drift/chosen_abs": 0.20925363898277283, + "drift/rejected_abs": 0.1305985450744629, + "epoch": 0.011124354390147001, + "grad_norm": 3.375, + "learning_rate": 2.5185185185185182e-08, + "logits/chosen": -2.122819185256958, + "logits/rejected": -2.058803081512451, + "logps/chosen": -0.27063214778900146, + "logps/rejected": -0.28141161799430847, + "loss": 0.0532744899392128, + "loss/core": 0.03867238387465477, + "loss/preference_sft": 0.2803190052509308, + "loss/reference_anchor": 0.16666279733181, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.0918078422546387, + "rewards/margins": 0.7871139049530029, + "rewards/rejected": 1.3046939373016357, + "ronpo/logit": 0.07871139049530029, + "ronpo/residual": 0.07295581698417664, + "ronpo/residual_abs": 0.09958575665950775, + "ronpo/target": 0.005755572579801083, + "ronpo/target_abs": 0.036900341510772705, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 35 + }, + { + "drift/chosen_abs": 0.13457752764225006, + "drift/rejected_abs": 0.10919027030467987, + "epoch": 0.012713547874453715, + "grad_norm": 442.0, + "learning_rate": 2.8888888888888884e-08, + "logits/chosen": -2.0784153938293457, + "logits/rejected": -2.0388412475585938, + "logps/chosen": -0.31832799315452576, + "logps/rejected": -0.32437992095947266, + "loss": 0.23646362125873566, + "loss/core": 0.22246842086315155, + "loss/preference_sft": 0.32219210267066956, + "loss/reference_anchor": 0.15438354015350342, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.3454421758651733, + "rewards/margins": 0.2544054090976715, + "rewards/rejected": 1.0910369157791138, + "ronpo/logit": 0.025440547615289688, + "ronpo/residual": 0.021605418995022774, + "ronpo/residual_abs": 0.1361103653907776, + "ronpo/target": 0.003835138399153948, + "ronpo/target_abs": 0.034678228199481964, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 40 + }, + { + "drift/chosen_abs": 0.2000904381275177, + "drift/rejected_abs": 0.10357439517974854, + "epoch": 0.014302741358760428, + "grad_norm": 3.953125, + "learning_rate": 3.2592592592592596e-08, + "logits/chosen": -2.1482901573181152, + "logits/rejected": -2.0773396492004395, + "logps/chosen": -0.30437272787094116, + "logps/rejected": -0.29646673798561096, + "loss": 0.08828647434711456, + "loss/core": 0.07516898214817047, + "loss/preference_sft": 0.3076452314853668, + "loss/reference_anchor": 0.14413544535636902, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.0003647804260254, + "rewards/margins": 0.966138482093811, + "rewards/rejected": 1.0342261791229248, + "ronpo/logit": 0.09661386907100677, + "ronpo/residual": 0.0960509404540062, + "ronpo/residual_abs": 0.12047521770000458, + "ronpo/target": 0.0005629155784845352, + "ronpo/target_abs": 0.0397389717400074, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 45 + }, + { + "drift/chosen_abs": 0.209742471575737, + "drift/rejected_abs": 0.17415867745876312, + "epoch": 0.015891934843067144, + "grad_norm": 11.875, + "learning_rate": 3.62962962962963e-08, + "logits/chosen": -2.2329165935516357, + "logits/rejected": -2.2811379432678223, + "logps/chosen": -0.38607966899871826, + "logps/rejected": -0.37516289949417114, + "loss": 0.505488932132721, + "loss/core": 0.46689605712890625, + "loss/preference_sft": 0.3818490207195282, + "loss/reference_anchor": 0.4763876497745514, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.0969882011413574, + "rewards/margins": 1.4074676036834717, + "rewards/rejected": 0.6895202994346619, + "ronpo/logit": 0.14074677228927612, + "ronpo/residual": 0.13970384001731873, + "ronpo/residual_abs": 0.2018175572156906, + "ronpo/target": 0.0010429281974211335, + "ronpo/target_abs": 0.039138179272413254, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 50 + }, + { + "drift/chosen_abs": 0.2418314665555954, + "drift/rejected_abs": 0.11427447944879532, + "epoch": 0.01748112832737386, + "grad_norm": 1.2421875, + "learning_rate": 4e-08, + "logits/chosen": -2.1331610679626465, + "logits/rejected": -2.10676646232605, + "logps/chosen": -0.3028566837310791, + "logps/rejected": -0.29301467537879944, + "loss": 0.2809545695781708, + "loss/core": 0.26085159182548523, + "loss/preference_sft": 0.291143000125885, + "loss/reference_anchor": 0.2389260083436966, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.417487621307373, + "rewards/margins": 1.2787439823150635, + "rewards/rejected": 1.1387437582015991, + "ronpo/logit": 0.12787440419197083, + "ronpo/residual": 0.12766289710998535, + "ronpo/residual_abs": 0.15509015321731567, + "ronpo/target": 0.00021149823442101479, + "ronpo/target_abs": 0.0375971794128418, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 55 + }, + { + "drift/chosen_abs": 0.28653964400291443, + "drift/rejected_abs": 0.07680980116128922, + "epoch": 0.01907032181168057, + "grad_norm": 7.15625, + "learning_rate": 4.37037037037037e-08, + "logits/chosen": -2.0188426971435547, + "logits/rejected": -2.038294553756714, + "logps/chosen": -0.2578909993171692, + "logps/rejected": -0.26004260778427124, + "loss": 0.7025126814842224, + "loss/core": 0.6709312200546265, + "loss/preference_sft": 0.25844430923461914, + "loss/reference_anchor": 0.39524129033088684, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 2.8651163578033447, + "rewards/margins": 2.112001419067383, + "rewards/rejected": 0.7531147003173828, + "ronpo/logit": 0.21120016276836395, + "ronpo/residual": 0.2195129692554474, + "ronpo/residual_abs": 0.2346762865781784, + "ronpo/target": -0.00831279531121254, + "ronpo/target_abs": 0.03656155616044998, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 60 + }, + { + "drift/chosen_abs": 0.23444890975952148, + "drift/rejected_abs": 0.08467339724302292, + "epoch": 0.020659515295987287, + "grad_norm": 163.0, + "learning_rate": 4.7407407407407405e-08, + "logits/chosen": -2.3911728858947754, + "logits/rejected": -2.461418628692627, + "logps/chosen": -0.2507760226726532, + "logps/rejected": -0.24406008422374725, + "loss": 0.24763154983520508, + "loss/core": 0.23287668824195862, + "loss/preference_sft": 0.25262296199798584, + "loss/reference_anchor": 0.17146925628185272, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.344456672668457, + "rewards/margins": 1.498246431350708, + "rewards/rejected": 0.8462101221084595, + "ronpo/logit": 0.14982464909553528, + "ronpo/residual": 0.13953104615211487, + "ronpo/residual_abs": 0.16128280758857727, + "ronpo/target": 0.01029361505061388, + "ronpo/target_abs": 0.03700987249612808, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 65 + }, + { + "drift/chosen_abs": 0.21904170513153076, + "drift/rejected_abs": 0.11182647943496704, + "epoch": 0.022248708780294002, + "grad_norm": 3.40625, + "learning_rate": 5.111111111111111e-08, + "logits/chosen": -2.1311838626861572, + "logits/rejected": -2.13192081451416, + "logps/chosen": -0.26317623257637024, + "logps/rejected": -0.26813703775405884, + "loss": 0.07477524131536484, + "loss/core": 0.0641421526670456, + "loss/preference_sft": 0.2720448076725006, + "loss/reference_anchor": 0.11457005888223648, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.1900362968444824, + "rewards/margins": 1.0720874071121216, + "rewards/rejected": 1.1179488897323608, + "ronpo/logit": 0.1072087436914444, + "ronpo/residual": 0.1019325703382492, + "ronpo/residual_abs": 0.12027925252914429, + "ronpo/target": 0.005276172887533903, + "ronpo/target_abs": 0.03518466651439667, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 70 + }, + { + "drift/chosen_abs": 0.2813442051410675, + "drift/rejected_abs": 0.14759781956672668, + "epoch": 0.023837902264600714, + "grad_norm": 6.03125, + "learning_rate": 5.481481481481482e-08, + "logits/chosen": -2.1067919731140137, + "logits/rejected": -2.0403101444244385, + "logps/chosen": -0.27957314252853394, + "logps/rejected": -0.2719704210758209, + "loss": 0.6640143394470215, + "loss/core": 0.6304754614830017, + "loss/preference_sft": 0.27398350834846497, + "loss/reference_anchor": 0.41978755593299866, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.81268310546875, + "rewards/margins": 1.3621594905853271, + "rewards/rejected": 1.4505234956741333, + "ronpo/logit": 0.1362159550189972, + "ronpo/residual": 0.13798193633556366, + "ronpo/residual_abs": 0.26703405380249023, + "ronpo/target": -0.001765978755429387, + "ronpo/target_abs": 0.04140448570251465, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 75 + }, + { + "drift/chosen_abs": 0.32669883966445923, + "drift/rejected_abs": 0.1386711448431015, + "epoch": 0.02542709574890743, + "grad_norm": 40.75, + "learning_rate": 5.851851851851851e-08, + "logits/chosen": -2.052741765975952, + "logits/rejected": -2.021430730819702, + "logps/chosen": -0.27059227228164673, + "logps/rejected": -0.2733466327190399, + "loss": 0.387042760848999, + "loss/core": 0.3550611138343811, + "loss/preference_sft": 0.26973801851272583, + "loss/reference_anchor": 0.39944788813591003, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.2669882774353027, + "rewards/margins": 1.8841441869735718, + "rewards/rejected": 1.3828439712524414, + "ronpo/logit": 0.18841443955898285, + "ronpo/residual": 0.1888277232646942, + "ronpo/residual_abs": 0.21059706807136536, + "ronpo/target": -0.00041329022496938705, + "ronpo/target_abs": 0.04022369906306267, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 80 + }, + { + "drift/chosen_abs": 0.2280385047197342, + "drift/rejected_abs": 0.07840272039175034, + "epoch": 0.027016289233214145, + "grad_norm": 1012.0, + "learning_rate": 6.222222222222221e-08, + "logits/chosen": -2.366596221923828, + "logits/rejected": -2.309652805328369, + "logps/chosen": -0.2941826283931732, + "logps/rejected": -0.266676664352417, + "loss": 0.5611018538475037, + "loss/core": 0.5350865721702576, + "loss/preference_sft": 0.2718455195426941, + "loss/reference_anchor": 0.3196846544742584, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.2803852558135986, + "rewards/margins": 1.5015474557876587, + "rewards/rejected": 0.7788375020027161, + "ronpo/logit": 0.1501547396183014, + "ronpo/residual": 0.1435343623161316, + "ronpo/residual_abs": 0.17277644574642181, + "ronpo/target": 0.006620384752750397, + "ronpo/target_abs": 0.039146699011325836, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 85 + }, + { + "drift/chosen_abs": 0.18973015248775482, + "drift/rejected_abs": 0.16024281084537506, + "epoch": 0.028605482717520857, + "grad_norm": 1.21875, + "learning_rate": 6.592592592592592e-08, + "logits/chosen": -2.26116943359375, + "logits/rejected": -2.2707791328430176, + "logps/chosen": -0.28311672806739807, + "logps/rejected": -0.27832671999931335, + "loss": 0.27082979679107666, + "loss/core": 0.25301414728164673, + "loss/preference_sft": 0.28250330686569214, + "loss/reference_anchor": 0.20929114520549774, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8972270488739014, + "rewards/margins": 0.29858314990997314, + "rewards/rejected": 1.5986440181732178, + "ronpo/logit": 0.029858315363526344, + "ronpo/residual": 0.026583939790725708, + "ronpo/residual_abs": 0.1573331356048584, + "ronpo/target": 0.003274365561082959, + "ronpo/target_abs": 0.041410621255636215, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 90 + }, + { + "drift/chosen_abs": 0.22587072849273682, + "drift/rejected_abs": 0.1010076031088829, + "epoch": 0.030194676201827572, + "grad_norm": 3.453125, + "learning_rate": 6.962962962962963e-08, + "logits/chosen": -2.3380606174468994, + "logits/rejected": -2.2585723400115967, + "logps/chosen": -0.27271103858947754, + "logps/rejected": -0.2697039246559143, + "loss": 0.37328386306762695, + "loss/core": 0.3553915023803711, + "loss/preference_sft": 0.27376872301101685, + "loss/reference_anchor": 0.21118827164173126, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.258707046508789, + "rewards/margins": 1.2546396255493164, + "rewards/rejected": 1.0040676593780518, + "ronpo/logit": 0.12546396255493164, + "ronpo/residual": 0.13282093405723572, + "ronpo/residual_abs": 0.19285807013511658, + "ronpo/target": -0.0073569887317717075, + "ronpo/target_abs": 0.043073222041130066, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 95 + }, + { + "drift/chosen_abs": 0.2537795305252075, + "drift/rejected_abs": 0.10038461536169052, + "epoch": 0.03178386968613429, + "grad_norm": 9.9375, + "learning_rate": 7.333333333333333e-08, + "logits/chosen": -1.9689257144927979, + "logits/rejected": -2.040370464324951, + "logps/chosen": -0.29754379391670227, + "logps/rejected": -0.288893461227417, + "loss": 0.2434326410293579, + "loss/core": 0.22620365023612976, + "loss/preference_sft": 0.28982117772102356, + "loss/reference_anchor": 0.2007375955581665, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.535590887069702, + "rewards/margins": 1.5332473516464233, + "rewards/rejected": 1.0023434162139893, + "ronpo/logit": 0.15332476794719696, + "ronpo/residual": 0.1544944941997528, + "ronpo/residual_abs": 0.17103642225265503, + "ronpo/target": -0.0011697321897372603, + "ronpo/target_abs": 0.03966054320335388, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 100 + }, + { + "drift/chosen_abs": 0.3896755576133728, + "drift/rejected_abs": 0.12742675840854645, + "epoch": 0.033373063170441, + "grad_norm": 94.0, + "learning_rate": 7.703703703703704e-08, + "logits/chosen": -1.9289357662200928, + "logits/rejected": -1.9362056255340576, + "logps/chosen": -0.2808288335800171, + "logps/rejected": -0.2832654118537903, + "loss": 0.7134211659431458, + "loss/core": 0.6654593348503113, + "loss/preference_sft": 0.2797420024871826, + "loss/reference_anchor": 0.6115163564682007, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.8960719108581543, + "rewards/margins": 2.6388587951660156, + "rewards/rejected": 1.2572133541107178, + "ronpo/logit": 0.26388588547706604, + "ronpo/residual": 0.2673845887184143, + "ronpo/residual_abs": 0.28307345509529114, + "ronpo/target": -0.003498706966638565, + "ronpo/target_abs": 0.041351594030857086, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 105 + }, + { + "drift/chosen_abs": 0.24370336532592773, + "drift/rejected_abs": 0.12277977168560028, + "epoch": 0.03496225665474772, + "grad_norm": 8.5, + "learning_rate": 8.074074074074075e-08, + "logits/chosen": -2.243190288543701, + "logits/rejected": -2.2032723426818848, + "logps/chosen": -0.294159471988678, + "logps/rejected": -0.28763648867607117, + "loss": 0.12273113429546356, + "loss/core": 0.10542932897806168, + "loss/preference_sft": 0.2917981445789337, + "loss/reference_anchor": 0.201510950922966, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.434004068374634, + "rewards/margins": 1.2225492000579834, + "rewards/rejected": 1.2114547491073608, + "ronpo/logit": 0.12225492298603058, + "ronpo/residual": 0.1237180233001709, + "ronpo/residual_abs": 0.15315257012844086, + "ronpo/target": -0.0014631092781201005, + "ronpo/target_abs": 0.04341963678598404, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 110 + }, + { + "drift/chosen_abs": 0.2751477062702179, + "drift/rejected_abs": 0.13712719082832336, + "epoch": 0.03655145013905443, + "grad_norm": 2.046875, + "learning_rate": 8.444444444444444e-08, + "logits/chosen": -1.8840402364730835, + "logits/rejected": -1.7751556634902954, + "logps/chosen": -0.2792755961418152, + "logps/rejected": -0.28249382972717285, + "loss": 0.15595600008964539, + "loss/core": 0.1379595398902893, + "loss/preference_sft": 0.2846413254737854, + "loss/reference_anchor": 0.21148864924907684, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.7486186027526855, + "rewards/margins": 1.3827894926071167, + "rewards/rejected": 1.3658287525177002, + "ronpo/logit": 0.13827894628047943, + "ronpo/residual": 0.14020434021949768, + "ronpo/residual_abs": 0.15965864062309265, + "ronpo/target": -0.0019253784557804465, + "ronpo/target_abs": 0.03954272344708443, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 115 + }, + { + "drift/chosen_abs": 0.25264251232147217, + "drift/rejected_abs": 0.11706644296646118, + "epoch": 0.03814064362336114, + "grad_norm": 1.2578125, + "learning_rate": 8.814814814814815e-08, + "logits/chosen": -2.016573905944824, + "logits/rejected": -2.064659357070923, + "logps/chosen": -0.3026151955127716, + "logps/rejected": -0.30011817812919617, + "loss": 0.3021138608455658, + "loss/core": 0.2761771082878113, + "loss/preference_sft": 0.3143635392189026, + "loss/reference_anchor": 0.3143865764141083, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.5251619815826416, + "rewards/margins": 1.356215000152588, + "rewards/rejected": 1.1689469814300537, + "ronpo/logit": 0.13562151789665222, + "ronpo/residual": 0.13547687232494354, + "ronpo/residual_abs": 0.1778053343296051, + "ronpo/target": 0.00014464984997175634, + "ronpo/target_abs": 0.03622669726610184, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 120 + }, + { + "drift/chosen_abs": 0.12724295258522034, + "drift/rejected_abs": 0.07093048840761185, + "epoch": 0.03972983710766786, + "grad_norm": 1.109375, + "learning_rate": 9.185185185185185e-08, + "logits/chosen": -2.2724061012268066, + "logits/rejected": -2.1935276985168457, + "logps/chosen": -0.27425864338874817, + "logps/rejected": -0.2812421917915344, + "loss": 0.04385209083557129, + "loss/core": 0.0365266427397728, + "loss/preference_sft": 0.27121520042419434, + "loss/reference_anchor": 0.07055105268955231, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.2717318534851074, + "rewards/margins": 0.5637460947036743, + "rewards/rejected": 0.7079858779907227, + "ronpo/logit": 0.056374602019786835, + "ronpo/residual": 0.060826338827610016, + "ronpo/residual_abs": 0.08193003386259079, + "ronpo/target": -0.004451734013855457, + "ronpo/target_abs": 0.03687246888875961, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 125 + }, + { + "drift/chosen_abs": 0.36936086416244507, + "drift/rejected_abs": 0.15099933743476868, + "epoch": 0.04131903059197457, + "grad_norm": 16.125, + "learning_rate": 9.555555555555556e-08, + "logits/chosen": -2.049341917037964, + "logits/rejected": -1.9780795574188232, + "logps/chosen": -0.28084278106689453, + "logps/rejected": -0.302662193775177, + "loss": 0.52703458070755, + "loss/core": 0.4721059203147888, + "loss/preference_sft": 0.30051907896995544, + "loss/reference_anchor": 0.7023288011550903, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.693608522415161, + "rewards/margins": 2.1844735145568848, + "rewards/rejected": 1.5091348886489868, + "ronpo/logit": 0.21844735741615295, + "ronpo/residual": 0.2173984795808792, + "ronpo/residual_abs": 0.24278369545936584, + "ronpo/target": 0.0010489022824913263, + "ronpo/target_abs": 0.03291699290275574, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 130 + }, + { + "drift/chosen_abs": 0.22806978225708008, + "drift/rejected_abs": 0.17581363022327423, + "epoch": 0.04290822407628129, + "grad_norm": 93.5, + "learning_rate": 9.925925925925925e-08, + "logits/chosen": -2.155839681625366, + "logits/rejected": -2.1424169540405273, + "logps/chosen": -0.25515803694725037, + "logps/rejected": -0.2733699679374695, + "loss": 0.38603073358535767, + "loss/core": 0.3621215224266052, + "loss/preference_sft": 0.26424866914749146, + "loss/reference_anchor": 0.2923644185066223, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.2795376777648926, + "rewards/margins": 0.5223998427391052, + "rewards/rejected": 1.7571378946304321, + "ronpo/logit": 0.052239976823329926, + "ronpo/residual": 0.05240260437130928, + "ronpo/residual_abs": 0.18414601683616638, + "ronpo/target": -0.0001626206503715366, + "ronpo/target_abs": 0.03484975919127464, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 135 + }, + { + "drift/chosen_abs": 0.21248528361320496, + "drift/rejected_abs": 0.1548730880022049, + "epoch": 0.044497417560588004, + "grad_norm": 37.25, + "learning_rate": 9.999325429879214e-08, + "logits/chosen": -2.066507339477539, + "logits/rejected": -2.0359199047088623, + "logps/chosen": -0.2865599989891052, + "logps/rejected": -0.28677845001220703, + "loss": 0.2560182213783264, + "loss/core": 0.237252876162529, + "loss/preference_sft": 0.2824710011482239, + "loss/reference_anchor": 0.2219577133655548, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.1243197917938232, + "rewards/margins": 0.5769208669662476, + "rewards/rejected": 1.5473989248275757, + "ronpo/logit": 0.05769209936261177, + "ronpo/residual": 0.05259509012103081, + "ronpo/residual_abs": 0.1705760955810547, + "ronpo/target": 0.005097000394016504, + "ronpo/target_abs": 0.0383358970284462, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 140 + }, + { + "drift/chosen_abs": 0.31215378642082214, + "drift/rejected_abs": 0.13634580373764038, + "epoch": 0.04608661104489471, + "grad_norm": 233.0, + "learning_rate": 9.996585300715115e-08, + "logits/chosen": -2.1033005714416504, + "logits/rejected": -2.1405441761016846, + "logps/chosen": -0.2735254764556885, + "logps/rejected": -0.26641449332237244, + "loss": 0.22382652759552002, + "loss/core": 0.1932472586631775, + "loss/preference_sft": 0.2740625739097595, + "loss/reference_anchor": 0.38031694293022156, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.120529890060425, + "rewards/margins": 1.759221076965332, + "rewards/rejected": 1.3613090515136719, + "ronpo/logit": 0.17592211067676544, + "ronpo/residual": 0.18040737509727478, + "ronpo/residual_abs": 0.19332192838191986, + "ronpo/target": -0.004485286772251129, + "ronpo/target_abs": 0.038091517984867096, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 145 + }, + { + "drift/chosen_abs": 0.40075913071632385, + "drift/rejected_abs": 0.11532590538263321, + "epoch": 0.04767580452920143, + "grad_norm": 2.671875, + "learning_rate": 9.991738606227535e-08, + "logits/chosen": -2.0292041301727295, + "logits/rejected": -2.1317694187164307, + "logps/chosen": -0.2969673275947571, + "logps/rejected": -0.2732444405555725, + "loss": 1.0155136585235596, + "loss/core": 0.962049126625061, + "loss/preference_sft": 0.29812875390052795, + "loss/reference_anchor": 0.6830486059188843, + "rewards/accuracies": 0.9375, + "rewards/chosen": 4.007282257080078, + "rewards/margins": 2.855675220489502, + "rewards/rejected": 1.1516071557998657, + "ronpo/logit": 0.2855675220489502, + "ronpo/residual": 0.2750109136104584, + "ronpo/residual_abs": 0.29854530096054077, + "ronpo/target": 0.010556596331298351, + "ronpo/target_abs": 0.040050022304058075, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 150 + }, + { + "drift/chosen_abs": 0.31126388907432556, + "drift/rejected_abs": 0.10443758964538574, + "epoch": 0.04926499801350814, + "grad_norm": 4.46875, + "learning_rate": 9.984787389787688e-08, + "logits/chosen": -2.1672780513763428, + "logits/rejected": -2.172288417816162, + "logps/chosen": -0.26607850193977356, + "logps/rejected": -0.26191627979278564, + "loss": 0.6859047412872314, + "loss/core": 0.644756555557251, + "loss/preference_sft": 0.2656945586204529, + "loss/reference_anchor": 0.5220734477043152, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.1122288703918457, + "rewards/margins": 2.06986665725708, + "rewards/rejected": 1.0423619747161865, + "ronpo/logit": 0.206986665725708, + "ronpo/residual": 0.20458738505840302, + "ronpo/residual_abs": 0.2233704775571823, + "ronpo/target": 0.002399270888417959, + "ronpo/target_abs": 0.03868597000837326, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 155 + }, + { + "drift/chosen_abs": 0.19957342743873596, + "drift/rejected_abs": 0.06797181069850922, + "epoch": 0.05085419149781486, + "grad_norm": 836.0, + "learning_rate": 9.975734582035323e-08, + "logits/chosen": -2.2061939239501953, + "logits/rejected": -2.2242698669433594, + "logps/chosen": -0.32043007016181946, + "logps/rejected": -0.273697167634964, + "loss": 0.5350174307823181, + "loss/core": 0.5092377066612244, + "loss/preference_sft": 0.3236948847770691, + "loss/reference_anchor": 0.31135934591293335, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.9948062896728516, + "rewards/margins": 1.3165847063064575, + "rewards/rejected": 0.6782217025756836, + "ronpo/logit": 0.13165846467018127, + "ronpo/residual": 0.12871192395687103, + "ronpo/residual_abs": 0.15134011209011078, + "ronpo/target": 0.00294652977026999, + "ronpo/target_abs": 0.0420367605984211, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 160 + }, + { + "drift/chosen_abs": 0.13104529678821564, + "drift/rejected_abs": 0.051272135227918625, + "epoch": 0.052443384982121574, + "grad_norm": 3.875, + "learning_rate": 9.964583999643173e-08, + "logits/chosen": -2.1314780712127686, + "logits/rejected": -2.0094780921936035, + "logps/chosen": -0.2844526171684265, + "logps/rejected": -0.2935580015182495, + "loss": 0.10935626924037933, + "loss/core": 0.1020495668053627, + "loss/preference_sft": 0.2882479131221771, + "loss/reference_anchor": 0.06859799474477768, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.3102877140045166, + "rewards/margins": 0.8030896186828613, + "rewards/rejected": 0.5071979761123657, + "ronpo/logit": 0.08030896633863449, + "ronpo/residual": 0.0773935616016388, + "ronpo/residual_abs": 0.10444708168506622, + "ronpo/target": 0.002915406133979559, + "ronpo/target_abs": 0.03609101101756096, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 165 + }, + { + "drift/chosen_abs": 0.31687039136886597, + "drift/rejected_abs": 0.16696619987487793, + "epoch": 0.05403257846642829, + "grad_norm": 1.25, + "learning_rate": 9.951340343707851e-08, + "logits/chosen": -2.027431011199951, + "logits/rejected": -2.020197868347168, + "logps/chosen": -0.28902456164360046, + "logps/rejected": -0.2761145830154419, + "loss": 0.4374803602695465, + "loss/core": 0.4037446975708008, + "loss/preference_sft": 0.28865548968315125, + "loss/reference_anchor": 0.42094317078590393, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.1682214736938477, + "rewards/margins": 1.5027532577514648, + "rewards/rejected": 1.6654682159423828, + "ronpo/logit": 0.150275319814682, + "ronpo/residual": 0.14720943570137024, + "ronpo/residual_abs": 0.23024269938468933, + "ronpo/target": 0.003065910190343857, + "ronpo/target_abs": 0.039906926453113556, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 170 + }, + { + "drift/chosen_abs": 0.19880618155002594, + "drift/rejected_abs": 0.11131361871957779, + "epoch": 0.055621771950735005, + "grad_norm": 30.875, + "learning_rate": 9.936009197767845e-08, + "logits/chosen": -2.263164520263672, + "logits/rejected": -2.1978354454040527, + "logps/chosen": -0.2936115860939026, + "logps/rejected": -0.30503323674201965, + "loss": 0.06118471547961235, + "loss/core": 0.0469072088599205, + "loss/preference_sft": 0.2960822582244873, + "loss/reference_anchor": 0.16075855493545532, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9874756336212158, + "rewards/margins": 0.8753183484077454, + "rewards/rejected": 1.1121572256088257, + "ronpo/logit": 0.08753182739019394, + "ronpo/residual": 0.09231725335121155, + "ronpo/residual_abs": 0.11050446331501007, + "ronpo/target": -0.0047854213044047356, + "ronpo/target_abs": 0.040785495191812515, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 175 + }, + { + "drift/chosen_abs": 0.47293806076049805, + "drift/rejected_abs": 0.2267259806394577, + "epoch": 0.057210965435041714, + "grad_norm": 39.75, + "learning_rate": 9.918597025449504e-08, + "logits/chosen": -2.073373317718506, + "logits/rejected": -2.0509047508239746, + "logps/chosen": -0.2899821400642395, + "logps/rejected": -0.281298965215683, + "loss": 0.952112078666687, + "loss/core": 0.8648629188537598, + "loss/preference_sft": 0.2834712862968445, + "loss/reference_anchor": 1.1349718570709229, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.7293806076049805, + "rewards/margins": 2.462536334991455, + "rewards/rejected": 2.2668442726135254, + "ronpo/logit": 0.24625368416309357, + "ronpo/residual": 0.2527698874473572, + "ronpo/residual_abs": 0.27635571360588074, + "ronpo/target": -0.006516219116747379, + "ronpo/target_abs": 0.03632596880197525, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 180 + }, + { + "drift/chosen_abs": 0.12262760102748871, + "drift/rejected_abs": 0.06452114135026932, + "epoch": 0.05880015891934843, + "grad_norm": 4.6875, + "learning_rate": 9.899111167741965e-08, + "logits/chosen": -2.2256319522857666, + "logits/rejected": -2.3286309242248535, + "logps/chosen": -0.29475218057632446, + "logps/rejected": -0.29190289974212646, + "loss": 0.022488895803689957, + "loss/core": 0.017534026876091957, + "loss/preference_sft": 0.29180967807769775, + "loss/reference_anchor": 0.03688395395874977, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.2256981134414673, + "rewards/margins": 0.583113968372345, + "rewards/rejected": 0.6425840854644775, + "ronpo/logit": 0.05831141024827957, + "ronpo/residual": 0.06310184299945831, + "ronpo/residual_abs": 0.08577104657888412, + "ronpo/target": -0.004790434148162603, + "ronpo/target_abs": 0.04436204954981804, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 185 + }, + { + "drift/chosen_abs": 0.12513764202594757, + "drift/rejected_abs": 0.0551786944270134, + "epoch": 0.060389352403655144, + "grad_norm": 0.8359375, + "learning_rate": 9.877559839902184e-08, + "logits/chosen": -2.314668655395508, + "logits/rejected": -2.410252332687378, + "logps/chosen": -0.305042028427124, + "logps/rejected": -0.30319944024086, + "loss": 0.0467093251645565, + "loss/core": 0.039730604737997055, + "loss/preference_sft": 0.30456751585006714, + "loss/reference_anchor": 0.06259283423423767, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.249873399734497, + "rewards/margins": 0.6990891098976135, + "rewards/rejected": 0.5507842302322388, + "ronpo/logit": 0.06990890949964523, + "ronpo/residual": 0.06662610918283463, + "ronpo/residual_abs": 0.0962677150964737, + "ronpo/target": 0.003282807767391205, + "ronpo/target_abs": 0.04486598074436188, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 190 + }, + { + "drift/chosen_abs": 0.21243241429328918, + "drift/rejected_abs": 0.11867997795343399, + "epoch": 0.06197854588796186, + "grad_norm": 6.1875, + "learning_rate": 9.853952127991371e-08, + "logits/chosen": -2.0548830032348633, + "logits/rejected": -2.0303750038146973, + "logps/chosen": -0.27820366621017456, + "logps/rejected": -0.27188804745674133, + "loss": 0.056257784366607666, + "loss/core": 0.04541615769267082, + "loss/preference_sft": 0.2762068510055542, + "loss/reference_anchor": 0.11693432182073593, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.123030185699463, + "rewards/margins": 0.9386847615242004, + "rewards/rejected": 1.1843453645706177, + "ronpo/logit": 0.09386847168207169, + "ronpo/residual": 0.09398861974477768, + "ronpo/residual_abs": 0.11155357211828232, + "ronpo/target": -0.00012014620006084442, + "ronpo/target_abs": 0.03941798955202103, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 195 + }, + { + "drift/chosen_abs": 0.2837020754814148, + "drift/rejected_abs": 0.0895429477095604, + "epoch": 0.06356773937226858, + "grad_norm": 9.5625, + "learning_rate": 9.828297985044313e-08, + "logits/chosen": -2.0397047996520996, + "logits/rejected": -2.1584408283233643, + "logps/chosen": -0.2889028787612915, + "logps/rejected": -0.2791716456413269, + "loss": 0.4928821623325348, + "loss/core": 0.4666885435581207, + "loss/preference_sft": 0.28681278228759766, + "loss/reference_anchor": 0.3205677270889282, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.836252212524414, + "rewards/margins": 1.9445428848266602, + "rewards/rejected": 0.8917096257209778, + "ronpo/logit": 0.19445426762104034, + "ronpo/residual": 0.19480250775814056, + "ronpo/residual_abs": 0.21554896235466003, + "ronpo/target": -0.000348227855283767, + "ronpo/target_abs": 0.04315657541155815, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 200 + }, + { + "drift/chosen_abs": 0.2943119406700134, + "drift/rejected_abs": 0.10832907259464264, + "epoch": 0.06515693285657528, + "grad_norm": 101.0, + "learning_rate": 9.800608226873141e-08, + "logits/chosen": -2.000119686126709, + "logits/rejected": -2.0814411640167236, + "logps/chosen": -0.28162431716918945, + "logps/rejected": -0.28688597679138184, + "loss": 0.7683490514755249, + "loss/core": 0.725469708442688, + "loss/preference_sft": 0.2873081564903259, + "loss/reference_anchor": 0.5429925322532654, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.9406843185424805, + "rewards/margins": 1.858192801475525, + "rewards/rejected": 1.0824917554855347, + "ronpo/logit": 0.18581928312778473, + "ronpo/residual": 0.20173266530036926, + "ronpo/residual_abs": 0.210129052400589, + "ronpo/target": -0.015913408249616623, + "ronpo/target_abs": 0.03376617655158043, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 205 + }, + { + "drift/chosen_abs": 0.21819961071014404, + "drift/rejected_abs": 0.05788310617208481, + "epoch": 0.066746126340882, + "grad_norm": 3.0625, + "learning_rate": 9.770894527507391e-08, + "logits/chosen": -2.2487425804138184, + "logits/rejected": -2.167056083679199, + "logps/chosen": -0.3088724613189697, + "logps/rejected": -0.315146267414093, + "loss": 1.0223528146743774, + "loss/core": 0.9793564081192017, + "loss/preference_sft": 0.31073203682899475, + "loss/reference_anchor": 0.5422115325927734, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.181572675704956, + "rewards/margins": 1.6049134731292725, + "rewards/rejected": 0.5766594409942627, + "ronpo/logit": 0.16049133241176605, + "ronpo/residual": 0.16226737201213837, + "ronpo/residual_abs": 0.1834581196308136, + "ronpo/target": -0.001776007004082203, + "ronpo/target_abs": 0.03983345627784729, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 210 + }, + { + "drift/chosen_abs": 0.3269013464450836, + "drift/rejected_abs": 0.164812833070755, + "epoch": 0.06833531982518871, + "grad_norm": 1.09375, + "learning_rate": 9.739169414272217e-08, + "logits/chosen": -2.167433500289917, + "logits/rejected": -2.1554653644561768, + "logps/chosen": -0.26557686924934387, + "logps/rejected": -0.27201777696609497, + "loss": 0.30894580483436584, + "loss/core": 0.2645125985145569, + "loss/preference_sft": 0.26722875237464905, + "loss/reference_anchor": 0.5657202005386353, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 3.2687904834747314, + "rewards/margins": 1.621460199356079, + "rewards/rejected": 1.6473299264907837, + "ronpo/logit": 0.16214601695537567, + "ronpo/residual": 0.16244594752788544, + "ronpo/residual_abs": 0.18437254428863525, + "ronpo/target": -0.0002999236749019474, + "ronpo/target_abs": 0.03696250170469284, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 215 + }, + { + "drift/chosen_abs": 0.1197463721036911, + "drift/rejected_abs": 0.08149294555187225, + "epoch": 0.06992451330949544, + "grad_norm": 0.423828125, + "learning_rate": 9.705446262506858e-08, + "logits/chosen": -2.2162628173828125, + "logits/rejected": -2.0838351249694824, + "logps/chosen": -0.30454206466674805, + "logps/rejected": -0.2994697093963623, + "loss": 0.053737662732601166, + "loss/core": 0.04793507978320122, + "loss/preference_sft": 0.3021828234195709, + "loss/reference_anchor": 0.04714947193861008, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.1946014165878296, + "rewards/margins": 0.3831804096698761, + "rewards/rejected": 0.8114210367202759, + "ronpo/logit": 0.03831804171204567, + "ronpo/residual": 0.04171460121870041, + "ronpo/residual_abs": 0.09978470951318741, + "ronpo/target": -0.0033965595066547394, + "ronpo/target_abs": 0.03821447864174843, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 220 + }, + { + "drift/chosen_abs": 0.23339900374412537, + "drift/rejected_abs": 0.10263761132955551, + "epoch": 0.07151370679380215, + "grad_norm": 5.84375, + "learning_rate": 9.669739289925576e-08, + "logits/chosen": -2.178617000579834, + "logits/rejected": -2.1953907012939453, + "logps/chosen": -0.28538942337036133, + "logps/rejected": -0.28581511974334717, + "loss": 0.20499324798583984, + "loss/core": 0.18968087434768677, + "loss/preference_sft": 0.2798437476158142, + "loss/reference_anchor": 0.17618073523044586, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.3339900970458984, + "rewards/margins": 1.3076139688491821, + "rewards/rejected": 1.0263762474060059, + "ronpo/logit": 0.13076139986515045, + "ronpo/residual": 0.12642139196395874, + "ronpo/residual_abs": 0.14748407900333405, + "ronpo/target": 0.004340013489127159, + "ronpo/target_abs": 0.03319591283798218, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 225 + }, + { + "drift/chosen_abs": 0.2666410207748413, + "drift/rejected_abs": 0.09672902524471283, + "epoch": 0.07310290027810885, + "grad_norm": 2.625, + "learning_rate": 9.632063550623463e-08, + "logits/chosen": -2.241774082183838, + "logits/rejected": -2.2601799964904785, + "logps/chosen": -0.26070520281791687, + "logps/rejected": -0.2603910565376282, + "loss": 0.405656635761261, + "loss/core": 0.3820309042930603, + "loss/preference_sft": 0.2593463361263275, + "loss/reference_anchor": 0.2890748381614685, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.666410207748413, + "rewards/margins": 1.7003774642944336, + "rewards/rejected": 0.9660326838493347, + "ronpo/logit": 0.17003774642944336, + "ronpo/residual": 0.17019498348236084, + "ronpo/residual_abs": 0.18751393258571625, + "ronpo/target": -0.00015722606622148305, + "ronpo/target_abs": 0.038429904729127884, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 230 + }, + { + "drift/chosen_abs": 0.45942026376724243, + "drift/rejected_abs": 0.23705634474754333, + "epoch": 0.07469209376241558, + "grad_norm": 70.0, + "learning_rate": 9.592434928729615e-08, + "logits/chosen": -2.114424467086792, + "logits/rejected": -2.1161932945251465, + "logps/chosen": -0.28261953592300415, + "logps/rejected": -0.28544551134109497, + "loss": 1.219696283340454, + "loss/core": 1.1286277770996094, + "loss/preference_sft": 0.28843197226524353, + "loss/reference_anchor": 1.1854050159454346, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.592301368713379, + "rewards/margins": 2.304565191268921, + "rewards/rejected": 2.2877368927001953, + "ronpo/logit": 0.23045650124549866, + "ronpo/residual": 0.23253145813941956, + "ronpo/residual_abs": 0.3629453778266907, + "ronpo/target": -0.0020749385003000498, + "ronpo/target_abs": 0.032184846699237823, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 235 + }, + { + "drift/chosen_abs": 0.29112711548805237, + "drift/rejected_abs": 0.16287891566753387, + "epoch": 0.07628128724672228, + "grad_norm": 192.0, + "learning_rate": 9.550870131710364e-08, + "logits/chosen": -2.141789674758911, + "logits/rejected": -2.2175676822662354, + "logps/chosen": -0.2665143609046936, + "logps/rejected": -0.2892531454563141, + "loss": 0.23649156093597412, + "loss/core": 0.19637542963027954, + "loss/preference_sft": 0.28473249077796936, + "loss/reference_anchor": 0.5064085721969604, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.9112708568573, + "rewards/margins": 1.2845176458358765, + "rewards/rejected": 1.6267534494400024, + "ronpo/logit": 0.12845174968242645, + "ronpo/residual": 0.118413046002388, + "ronpo/residual_abs": 0.14850665628910065, + "ronpo/target": 0.010038711130619049, + "ronpo/target_abs": 0.03962286561727524, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 240 + }, + { + "drift/chosen_abs": 0.18798431754112244, + "drift/rejected_abs": 0.07243553549051285, + "epoch": 0.07787048073102901, + "grad_norm": 1.609375, + "learning_rate": 9.507386683325402e-08, + "logits/chosen": -2.1808266639709473, + "logits/rejected": -2.199118137359619, + "logps/chosen": -0.2991955280303955, + "logps/rejected": -0.30441370606422424, + "loss": 0.27557721734046936, + "loss/core": 0.26069125533103943, + "loss/preference_sft": 0.30375024676322937, + "loss/reference_anchor": 0.16810381412506104, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.878678560256958, + "rewards/margins": 1.1664841175079346, + "rewards/rejected": 0.712194561958313, + "ronpo/logit": 0.11664841324090958, + "ronpo/residual": 0.1208236962556839, + "ronpo/residual_abs": 0.14474524557590485, + "ronpo/target": -0.004175290931016207, + "ronpo/target_abs": 0.04190593212842941, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 245 + }, + { + "drift/chosen_abs": 0.362449586391449, + "drift/rejected_abs": 0.08568783104419708, + "epoch": 0.07945967421533572, + "grad_norm": 7.59375, + "learning_rate": 9.462002916239739e-08, + "logits/chosen": -2.11047625541687, + "logits/rejected": -2.0861949920654297, + "logps/chosen": -0.3092321753501892, + "logps/rejected": -0.30976372957229614, + "loss": 1.2317886352539062, + "loss/core": 1.1781948804855347, + "loss/preference_sft": 0.3098042607307434, + "loss/reference_anchor": 0.6836021542549133, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.62377667427063, + "rewards/margins": 2.7679665088653564, + "rewards/rejected": 0.8558100461959839, + "ronpo/logit": 0.2767966389656067, + "ronpo/residual": 0.2732112407684326, + "ronpo/residual_abs": 0.2978476881980896, + "ronpo/target": 0.003585411235690117, + "ronpo/target_abs": 0.035868819802999496, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 250 + }, + { + "drift/chosen_abs": 0.44660884141921997, + "drift/rejected_abs": 0.20091009140014648, + "epoch": 0.08104886769964244, + "grad_norm": 10.125, + "learning_rate": 9.414737964294634e-08, + "logits/chosen": -2.101787567138672, + "logits/rejected": -2.1054489612579346, + "logps/chosen": -0.26226210594177246, + "logps/rejected": -0.27092769742012024, + "loss": 0.7605728507041931, + "loss/core": 0.6867702603340149, + "loss/preference_sft": 0.2730024456977844, + "loss/reference_anchor": 0.9567340612411499, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 4.464694976806641, + "rewards/margins": 2.549316883087158, + "rewards/rejected": 1.9153783321380615, + "ronpo/logit": 0.25493165850639343, + "ronpo/residual": 0.2584943175315857, + "ronpo/residual_abs": 0.2778705954551697, + "ronpo/target": -0.003562638070434332, + "ronpo/target_abs": 0.04188482090830803, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 255 + }, + { + "drift/chosen_abs": 0.3281967043876648, + "drift/rejected_abs": 0.0860821008682251, + "epoch": 0.08263806118394915, + "grad_norm": 960.0, + "learning_rate": 9.365611754440755e-08, + "logits/chosen": -2.194645881652832, + "logits/rejected": -2.1680915355682373, + "logps/chosen": -0.28327953815460205, + "logps/rejected": -0.2808752655982971, + "loss": 0.8503106236457825, + "loss/core": 0.8106344938278198, + "loss/preference_sft": 0.29711759090423584, + "loss/reference_anchor": 0.4993039667606354, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.2812724113464355, + "rewards/margins": 2.437526226043701, + "rewards/rejected": 0.8437461853027344, + "ronpo/logit": 0.2437526285648346, + "ronpo/residual": 0.24408352375030518, + "ronpo/residual_abs": 0.26183831691741943, + "ronpo/target": -0.0003309231251478195, + "ronpo/target_abs": 0.038510847836732864, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 260 + }, + { + "drift/chosen_abs": 0.1818397492170334, + "drift/rejected_abs": 0.08128567785024643, + "epoch": 0.08422725466825585, + "grad_norm": 16.125, + "learning_rate": 9.314644998336948e-08, + "logits/chosen": -2.153672695159912, + "logits/rejected": -2.1532981395721436, + "logps/chosen": -0.28503522276878357, + "logps/rejected": -0.2909298837184906, + "loss": 0.07522353529930115, + "loss/core": 0.06629156321287155, + "loss/preference_sft": 0.2900465726852417, + "loss/reference_anchor": 0.09008826315402985, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.8183977603912354, + "rewards/margins": 1.012382984161377, + "rewards/rejected": 0.8060146570205688, + "ronpo/logit": 0.10123831033706665, + "ronpo/residual": 0.0954081267118454, + "ronpo/residual_abs": 0.12208437919616699, + "ronpo/target": 0.005830181296914816, + "ronpo/target_abs": 0.03874696046113968, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 265 + }, + { + "drift/chosen_abs": 0.2921431064605713, + "drift/rejected_abs": 0.16023942828178406, + "epoch": 0.08581644815256258, + "grad_norm": 5.90625, + "learning_rate": 9.261859183618188e-08, + "logits/chosen": -2.220456123352051, + "logits/rejected": -2.1795496940612793, + "logps/chosen": -0.2800024747848511, + "logps/rejected": -0.2899700403213501, + "loss": 0.36583656072616577, + "loss/core": 0.3111600875854492, + "loss/preference_sft": 0.2803427577018738, + "loss/reference_anchor": 0.700985312461853, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.9204893112182617, + "rewards/margins": 1.3201406002044678, + "rewards/rejected": 1.6003488302230835, + "ronpo/logit": 0.13201405107975006, + "ronpo/residual": 0.12298605591058731, + "ronpo/residual_abs": 0.15286676585674286, + "ronpo/target": 0.009028008207678795, + "ronpo/target_abs": 0.044766865670681, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 270 + }, + { + "drift/chosen_abs": 0.17901456356048584, + "drift/rejected_abs": 0.10373431444168091, + "epoch": 0.08740564163686929, + "grad_norm": 7.75, + "learning_rate": 9.207276564836366e-08, + "logits/chosen": -2.273167133331299, + "logits/rejected": -2.3467493057250977, + "logps/chosen": -0.2789549231529236, + "logps/rejected": -0.26800718903541565, + "loss": 0.04449104890227318, + "loss/core": 0.034896623343229294, + "loss/preference_sft": 0.27731361985206604, + "loss/reference_anchor": 0.10019437968730927, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.7901455163955688, + "rewards/margins": 0.7541426420211792, + "rewards/rejected": 1.0360027551651, + "ronpo/logit": 0.0754142552614212, + "ronpo/residual": 0.06837241351604462, + "ronpo/residual_abs": 0.09739400446414948, + "ronpo/target": 0.007041858043521643, + "ronpo/target_abs": 0.038143135607242584, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 275 + }, + { + "drift/chosen_abs": 0.16873064637184143, + "drift/rejected_abs": 0.11091496795415878, + "epoch": 0.08899483512117601, + "grad_norm": 48.0, + "learning_rate": 9.150920154077752e-08, + "logits/chosen": -2.183501720428467, + "logits/rejected": -2.1855404376983643, + "logps/chosen": -0.2852725386619568, + "logps/rejected": -0.2904953360557556, + "loss": 0.09437291324138641, + "loss/core": 0.08497822284698486, + "loss/preference_sft": 0.2888687252998352, + "loss/reference_anchor": 0.09637565910816193, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.686948537826538, + "rewards/margins": 0.5791365504264832, + "rewards/rejected": 1.1078121662139893, + "ronpo/logit": 0.0579136498272419, + "ronpo/residual": 0.05210072919726372, + "ronpo/residual_abs": 0.12251182645559311, + "ronpo/target": 0.0058129276148974895, + "ronpo/target_abs": 0.0337434858083725, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 280 + }, + { + "drift/chosen_abs": 0.4374244809150696, + "drift/rejected_abs": 0.13985301554203033, + "epoch": 0.09058402860548272, + "grad_norm": 16.75, + "learning_rate": 9.092813711261074e-08, + "logits/chosen": -2.2230849266052246, + "logits/rejected": -2.194876194000244, + "logps/chosen": -0.2806859612464905, + "logps/rejected": -0.29192352294921875, + "loss": 1.5361340045928955, + "loss/core": 1.4417104721069336, + "loss/preference_sft": 0.2899562418460846, + "loss/reference_anchor": 1.229984998703003, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.373291015625, + "rewards/margins": 2.9773240089416504, + "rewards/rejected": 1.3959667682647705, + "ronpo/logit": 0.2977324426174164, + "ronpo/residual": 0.30149295926094055, + "ronpo/residual_abs": 0.32409870624542236, + "ronpo/target": -0.0037605254910886288, + "ronpo/target_abs": 0.03826469928026199, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 285 + }, + { + "drift/chosen_abs": 0.2992640733718872, + "drift/rejected_abs": 0.14071711897850037, + "epoch": 0.09217322208978942, + "grad_norm": 101.0, + "learning_rate": 9.032981734120312e-08, + "logits/chosen": -2.015190839767456, + "logits/rejected": -2.089470624923706, + "logps/chosen": -0.2724992632865906, + "logps/rejected": -0.2815447449684143, + "loss": 0.2769699692726135, + "loss/core": 0.2548522353172302, + "loss/preference_sft": 0.2765698730945587, + "loss/reference_anchor": 0.26724615693092346, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.989751100540161, + "rewards/margins": 1.5834410190582275, + "rewards/rejected": 1.4063100814819336, + "ronpo/logit": 0.1583441197872162, + "ronpo/residual": 0.15625953674316406, + "ronpo/residual_abs": 0.2027464210987091, + "ronpo/target": 0.0020845707040280104, + "ronpo/target_abs": 0.042959146201610565, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 290 + }, + { + "drift/chosen_abs": 0.19468748569488525, + "drift/rejected_abs": 0.07145005464553833, + "epoch": 0.09376241557409615, + "grad_norm": 193.0, + "learning_rate": 8.97144944787643e-08, + "logits/chosen": -2.1931087970733643, + "logits/rejected": -2.186386823654175, + "logps/chosen": -0.3015713095664978, + "logps/rejected": -0.30587688088417053, + "loss": 0.24030208587646484, + "loss/core": 0.224404975771904, + "loss/preference_sft": 0.29630112648010254, + "loss/reference_anchor": 0.18233135342597961, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9454704523086548, + "rewards/margins": 1.2360841035842896, + "rewards/rejected": 0.70938640832901, + "ronpo/logit": 0.12360841035842896, + "ronpo/residual": 0.12222124636173248, + "ronpo/residual_abs": 0.14251907169818878, + "ronpo/target": 0.0013871697010472417, + "ronpo/target_abs": 0.03497045487165451, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 295 + }, + { + "drift/chosen_abs": 0.136820450425148, + "drift/rejected_abs": 0.08163662254810333, + "epoch": 0.09535160905840286, + "grad_norm": 6.125, + "learning_rate": 8.9082427946024e-08, + "logits/chosen": -2.403992176055908, + "logits/rejected": -2.408097743988037, + "logps/chosen": -0.2954743504524231, + "logps/rejected": -0.2864408791065216, + "loss": 0.027770619839429855, + "loss/core": 0.021785762161016464, + "loss/preference_sft": 0.2909078299999237, + "loss/reference_anchor": 0.05070732906460762, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.3670862913131714, + "rewards/margins": 0.553402841091156, + "rewards/rejected": 0.8136835098266602, + "ronpo/logit": 0.05534028261899948, + "ronpo/residual": 0.05457191914319992, + "ronpo/residual_abs": 0.07968275249004364, + "ronpo/target": 0.0007683668518438935, + "ronpo/target_abs": 0.03892510011792183, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 300 + }, + { + "drift/chosen_abs": 0.22264614701271057, + "drift/rejected_abs": 0.10277211666107178, + "epoch": 0.09694080254270958, + "grad_norm": 2.34375, + "learning_rate": 8.843388422285994e-08, + "logits/chosen": -2.075179100036621, + "logits/rejected": -2.102503538131714, + "logps/chosen": -0.30451059341430664, + "logps/rejected": -0.2927056550979614, + "loss": 0.1892203986644745, + "loss/core": 0.16778506338596344, + "loss/preference_sft": 0.3040909171104431, + "loss/reference_anchor": 0.2553955018520355, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.2247345447540283, + "rewards/margins": 1.1996135711669922, + "rewards/rejected": 1.0251209735870361, + "ronpo/logit": 0.11996134370565414, + "ronpo/residual": 0.1212671548128128, + "ronpo/residual_abs": 0.14174291491508484, + "ronpo/target": -0.0013058021431788802, + "ronpo/target_abs": 0.038282014429569244, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 305 + }, + { + "drift/chosen_abs": 0.23042087256908417, + "drift/rejected_abs": 0.1503649353981018, + "epoch": 0.09852999602701629, + "grad_norm": 211.0, + "learning_rate": 8.776913673594967e-08, + "logits/chosen": -2.0958151817321777, + "logits/rejected": -1.9892704486846924, + "logps/chosen": -0.30442753434181213, + "logps/rejected": -0.30017727613449097, + "loss": 0.08808089792728424, + "loss/core": 0.0615558996796608, + "loss/preference_sft": 0.3082145154476166, + "loss/reference_anchor": 0.3228452503681183, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.3031153678894043, + "rewards/margins": 0.8013383150100708, + "rewards/rejected": 1.501776933670044, + "ronpo/logit": 0.0801338404417038, + "ronpo/residual": 0.08253569155931473, + "ronpo/residual_abs": 0.11766223609447479, + "ronpo/target": -0.0024018441326916218, + "ronpo/target_abs": 0.03481123223900795, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 310 + }, + { + "drift/chosen_abs": 0.21849361062049866, + "drift/rejected_abs": 0.1704569309949875, + "epoch": 0.10011918951132301, + "grad_norm": 1.9609375, + "learning_rate": 8.708846574349372e-08, + "logits/chosen": -2.206904888153076, + "logits/rejected": -2.2400927543640137, + "logps/chosen": -0.28868699073791504, + "logps/rejected": -0.2845986485481262, + "loss": 0.1418466717004776, + "loss/core": 0.08006531000137329, + "loss/preference_sft": 0.28893551230430603, + "loss/reference_anchor": 0.794857919216156, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.184345245361328, + "rewards/margins": 0.48178085684776306, + "rewards/rejected": 1.7025642395019531, + "ronpo/logit": 0.04817808419466019, + "ronpo/residual": 0.044129565358161926, + "ronpo/residual_abs": 0.11013004928827286, + "ronpo/target": 0.004048521164804697, + "ronpo/target_abs": 0.03925003856420517, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 315 + }, + { + "drift/chosen_abs": 0.3064383864402771, + "drift/rejected_abs": 0.15249067544937134, + "epoch": 0.10170838299562972, + "grad_norm": 9.625, + "learning_rate": 8.63921582170582e-08, + "logits/chosen": -2.0485007762908936, + "logits/rejected": -2.04766583442688, + "logps/chosen": -0.2673102021217346, + "logps/rejected": -0.2666773200035095, + "loss": 0.43610453605651855, + "loss/core": 0.3816451132297516, + "loss/preference_sft": 0.2681375741958618, + "loss/reference_anchor": 0.6993125081062317, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.062673807144165, + "rewards/margins": 1.5397114753723145, + "rewards/rejected": 1.5229620933532715, + "ronpo/logit": 0.15397116541862488, + "ronpo/residual": 0.15312956273555756, + "ronpo/residual_abs": 0.18270666897296906, + "ronpo/target": 0.0008415899937972426, + "ronpo/target_abs": 0.0406792089343071, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 320 + }, + { + "drift/chosen_abs": 0.2704039514064789, + "drift/rejected_abs": 0.06453654170036316, + "epoch": 0.10329757647993643, + "grad_norm": 23.0, + "learning_rate": 8.568050772058761e-08, + "logits/chosen": -2.214332103729248, + "logits/rejected": -2.2380669116973877, + "logps/chosen": -0.3050305247306824, + "logps/rejected": -0.2898855209350586, + "loss": 1.148066759109497, + "loss/core": 1.099792718887329, + "loss/preference_sft": 0.30017027258872986, + "loss/reference_anchor": 0.6136360168457031, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.703939437866211, + "rewards/margins": 2.0673062801361084, + "rewards/rejected": 0.6366328001022339, + "ronpo/logit": 0.20673063397407532, + "ronpo/residual": 0.20851190388202667, + "ronpo/residual_abs": 0.2224424183368683, + "ronpo/target": -0.001781263155862689, + "ronpo/target_abs": 0.03680066391825676, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 325 + }, + { + "drift/chosen_abs": 0.2501479387283325, + "drift/rejected_abs": 0.12387964874505997, + "epoch": 0.10488676996424315, + "grad_norm": 12.5625, + "learning_rate": 8.49538142866378e-08, + "logits/chosen": -2.323371648788452, + "logits/rejected": -2.3236114978790283, + "logps/chosen": -0.26736950874328613, + "logps/rejected": -0.2633174955844879, + "loss": 0.5597383379936218, + "loss/core": 0.532612681388855, + "loss/preference_sft": 0.2744181752204895, + "loss/reference_anchor": 0.3342326879501343, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.501479387283325, + "rewards/margins": 1.2703591585159302, + "rewards/rejected": 1.2311203479766846, + "ronpo/logit": 0.12703590095043182, + "ronpo/residual": 0.1348484754562378, + "ronpo/residual_abs": 0.23397144675254822, + "ronpo/target": -0.007812569849193096, + "ronpo/target_abs": 0.04597053676843643, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 330 + }, + { + "drift/chosen_abs": 0.44480544328689575, + "drift/rejected_abs": 0.20965024828910828, + "epoch": 0.10647596344854986, + "grad_norm": 12.125, + "learning_rate": 8.421238428988198e-08, + "logits/chosen": -1.9913667440414429, + "logits/rejected": -1.9402954578399658, + "logps/chosen": -0.2954614460468292, + "logps/rejected": -0.29534387588500977, + "loss": 0.8044527173042297, + "loss/core": 0.697630763053894, + "loss/preference_sft": 0.2934873104095459, + "loss/reference_anchor": 1.394944429397583, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 4.447211742401123, + "rewards/margins": 2.354290246963501, + "rewards/rejected": 2.0929219722747803, + "ronpo/logit": 0.235429048538208, + "ronpo/residual": 0.2442881315946579, + "ronpo/residual_abs": 0.2787492871284485, + "ronpo/target": -0.008859092369675636, + "ronpo/target_abs": 0.036721840500831604, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 335 + }, + { + "drift/chosen_abs": 0.4596560597419739, + "drift/rejected_abs": 0.16454848647117615, + "epoch": 0.10806515693285658, + "grad_norm": 82.5, + "learning_rate": 8.34565303179429e-08, + "logits/chosen": -1.884810447692871, + "logits/rejected": -1.9432003498077393, + "logps/chosen": -0.28668686747550964, + "logps/rejected": -0.2740832567214966, + "loss": 1.0088319778442383, + "loss/core": 0.9366035461425781, + "loss/preference_sft": 0.28669866919517517, + "loss/reference_anchor": 0.9343746304512024, + "rewards/accuracies": 0.875, + "rewards/chosen": 4.593608856201172, + "rewards/margins": 2.9580533504486084, + "rewards/rejected": 1.6355552673339844, + "ronpo/logit": 0.29580533504486084, + "ronpo/residual": 0.29322350025177, + "ronpo/residual_abs": 0.311967670917511, + "ronpo/target": 0.0025818462017923594, + "ronpo/target_abs": 0.033389072865247726, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 340 + }, + { + "drift/chosen_abs": 0.24590358138084412, + "drift/rejected_abs": 0.11953750997781754, + "epoch": 0.10965435041716329, + "grad_norm": 59.5, + "learning_rate": 8.268657103960556e-08, + "logits/chosen": -2.178544521331787, + "logits/rejected": -2.251857280731201, + "logps/chosen": -0.272917777299881, + "logps/rejected": -0.2786032259464264, + "loss": 0.12808948755264282, + "loss/core": 0.10189362615346909, + "loss/preference_sft": 0.28778940439224243, + "loss/reference_anchor": 0.3204990029335022, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.4582910537719727, + "rewards/margins": 1.286404013633728, + "rewards/rejected": 1.171886920928955, + "ronpo/logit": 0.12864042818546295, + "ronpo/residual": 0.13276901841163635, + "ronpo/residual_abs": 0.1446714699268341, + "ronpo/target": -0.004128606058657169, + "ronpo/target_abs": 0.030540451407432556, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 345 + }, + { + "drift/chosen_abs": 0.4136156439781189, + "drift/rejected_abs": 0.16546812653541565, + "epoch": 0.11124354390147001, + "grad_norm": 1.4296875, + "learning_rate": 8.190283107046611e-08, + "logits/chosen": -2.2700629234313965, + "logits/rejected": -2.299020767211914, + "logps/chosen": -0.23404893279075623, + "logps/rejected": -0.24618962407112122, + "loss": 0.9944799542427063, + "loss/core": 0.9109455347061157, + "loss/preference_sft": 0.23421046137809753, + "loss/reference_anchor": 1.0903706550598145, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.136050224304199, + "rewards/margins": 2.4818317890167236, + "rewards/rejected": 1.6542189121246338, + "ronpo/logit": 0.2481832057237625, + "ronpo/residual": 0.24511413276195526, + "ronpo/residual_abs": 0.266287237405777, + "ronpo/target": 0.0030690638814121485, + "ronpo/target_abs": 0.03295261040329933, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 350 + }, + { + "drift/chosen_abs": 0.28086286783218384, + "drift/rejected_abs": 0.1028003841638565, + "epoch": 0.11283273738577672, + "grad_norm": 17.0, + "learning_rate": 8.11056408360737e-08, + "logits/chosen": -2.126852512359619, + "logits/rejected": -2.0258126258850098, + "logps/chosen": -0.2836364507675171, + "logps/rejected": -0.27664485573768616, + "loss": 0.4193928837776184, + "loss/core": 0.39380452036857605, + "loss/preference_sft": 0.2735845446586609, + "loss/reference_anchor": 0.3138180375099182, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.8082804679870605, + "rewards/margins": 1.7823970317840576, + "rewards/rejected": 1.0258831977844238, + "ronpo/logit": 0.17823971807956696, + "ronpo/residual": 0.17218564450740814, + "ronpo/residual_abs": 0.20453758537769318, + "ronpo/target": 0.0060540675185620785, + "ronpo/target_abs": 0.0407588966190815, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 355 + }, + { + "drift/chosen_abs": 0.2176329344511032, + "drift/rejected_abs": 0.08809185028076172, + "epoch": 0.11442193087008343, + "grad_norm": 10.875, + "learning_rate": 8.029533643262255e-08, + "logits/chosen": -2.2074317932128906, + "logits/rejected": -2.2725961208343506, + "logps/chosen": -0.267316609621048, + "logps/rejected": -0.25893837213516235, + "loss": 0.08880702406167984, + "loss/core": 0.07977307587862015, + "loss/preference_sft": 0.26773780584335327, + "loss/reference_anchor": 0.09367872774600983, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.175424098968506, + "rewards/margins": 1.2949514389038086, + "rewards/rejected": 0.8804727792739868, + "ronpo/logit": 0.12949514389038086, + "ronpo/residual": 0.13094380497932434, + "ronpo/residual_abs": 0.15374919772148132, + "ronpo/target": -0.0014486557338386774, + "ronpo/target_abs": 0.03519626706838608, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 360 + }, + { + "drift/chosen_abs": 0.20784783363342285, + "drift/rejected_abs": 0.12744882702827454, + "epoch": 0.11601112435439015, + "grad_norm": 1.9140625, + "learning_rate": 7.947225948525369e-08, + "logits/chosen": -2.0954489707946777, + "logits/rejected": -2.1332972049713135, + "logps/chosen": -0.28586500883102417, + "logps/rejected": -0.32812875509262085, + "loss": 0.07003101706504822, + "loss/core": 0.04061419516801834, + "loss/preference_sft": 0.2947535812854767, + "loss/reference_anchor": 0.36274877190589905, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.0766704082489014, + "rewards/margins": 0.8259242177009583, + "rewards/rejected": 1.2507461309432983, + "ronpo/logit": 0.0825924351811409, + "ronpo/residual": 0.07970172166824341, + "ronpo/residual_abs": 0.10139290988445282, + "ronpo/target": 0.002890714444220066, + "ronpo/target_abs": 0.04324514418840408, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 365 + }, + { + "drift/chosen_abs": 0.3182205855846405, + "drift/rejected_abs": 0.21468524634838104, + "epoch": 0.11760031783869686, + "grad_norm": 39.25, + "learning_rate": 7.863675700402526e-08, + "logits/chosen": -2.134063720703125, + "logits/rejected": -2.1670172214508057, + "logps/chosen": -0.278795450925827, + "logps/rejected": -0.28225570917129517, + "loss": 0.19065257906913757, + "loss/core": 0.10301892459392548, + "loss/preference_sft": 0.28197628259658813, + "loss/reference_anchor": 1.1402509212493896, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.1817617416381836, + "rewards/margins": 1.0355478525161743, + "rewards/rejected": 2.146214008331299, + "ronpo/logit": 0.10355480015277863, + "ronpo/residual": 0.09973715245723724, + "ronpo/residual_abs": 0.13304369151592255, + "ronpo/target": 0.003817648161202669, + "ronpo/target_abs": 0.03572530299425125, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 370 + }, + { + "drift/chosen_abs": 0.21340015530586243, + "drift/rejected_abs": 0.0985214114189148, + "epoch": 0.11918951132300358, + "grad_norm": 21.125, + "learning_rate": 7.778918123761286e-08, + "logits/chosen": -2.27836012840271, + "logits/rejected": -2.302485942840576, + "logps/chosen": -0.29381102323532104, + "logps/rejected": -0.2883267104625702, + "loss": 0.1361909955739975, + "loss/core": 0.12311957031488419, + "loss/preference_sft": 0.2917711138725281, + "loss/reference_anchor": 0.14510871469974518, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.133848190307617, + "rewards/margins": 1.1486339569091797, + "rewards/rejected": 0.9852142333984375, + "ronpo/logit": 0.11486339569091797, + "ronpo/residual": 0.1210290789604187, + "ronpo/residual_abs": 0.14276602864265442, + "ronpo/target": -0.006165677681565285, + "ronpo/target_abs": 0.04001092165708542, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 375 + }, + { + "drift/chosen_abs": 0.2751646339893341, + "drift/rejected_abs": 0.09928236156702042, + "epoch": 0.12077870480731029, + "grad_norm": 0.63671875, + "learning_rate": 7.692988952480112e-08, + "logits/chosen": -2.1607887744903564, + "logits/rejected": -2.1772239208221436, + "logps/chosen": -0.2890273928642273, + "logps/rejected": -0.2840579152107239, + "loss": 0.5159703493118286, + "loss/core": 0.4876245856285095, + "loss/preference_sft": 0.2878762185573578, + "loss/reference_anchor": 0.3491542637348175, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.7504799365997314, + "rewards/margins": 1.7696754932403564, + "rewards/rejected": 0.9808047413825989, + "ronpo/logit": 0.1769675463438034, + "ronpo/residual": 0.16951511800289154, + "ronpo/residual_abs": 0.19311194121837616, + "ronpo/target": 0.007452424615621567, + "ronpo/target_abs": 0.039471663534641266, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 380 + }, + { + "drift/chosen_abs": 0.25231146812438965, + "drift/rejected_abs": 0.14849232137203217, + "epoch": 0.12236789829161701, + "grad_norm": 60.25, + "learning_rate": 7.605924414382924e-08, + "logits/chosen": -2.277235746383667, + "logits/rejected": -2.2479469776153564, + "logps/chosen": -0.2711496949195862, + "logps/rejected": -0.26407307386398315, + "loss": 0.17193853855133057, + "loss/core": 0.15317845344543457, + "loss/preference_sft": 0.26539701223373413, + "loss/reference_anchor": 0.22359445691108704, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.5228137969970703, + "rewards/margins": 1.0395729541778564, + "rewards/rejected": 1.4832407236099243, + "ronpo/logit": 0.10395729541778564, + "ronpo/residual": 0.09792117774486542, + "ronpo/residual_abs": 0.12750428915023804, + "ronpo/target": 0.006036114878952503, + "ronpo/target_abs": 0.042838118970394135, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 385 + }, + { + "drift/chosen_abs": 0.21072201430797577, + "drift/rejected_abs": 0.12239472568035126, + "epoch": 0.12395709177592372, + "grad_norm": 3.4375, + "learning_rate": 7.517761215965428e-08, + "logits/chosen": -2.078555107116699, + "logits/rejected": -2.1201419830322266, + "logps/chosen": -0.30479007959365845, + "logps/rejected": -0.2949450612068176, + "loss": 0.09117992967367172, + "loss/core": 0.0706782341003418, + "loss/preference_sft": 0.295664519071579, + "loss/reference_anchor": 0.24378947913646698, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.1056594848632812, + "rewards/margins": 0.8848453760147095, + "rewards/rejected": 1.2208143472671509, + "ronpo/logit": 0.08848454058170319, + "ronpo/residual": 0.09790941327810287, + "ronpo/residual_abs": 0.11511196941137314, + "ronpo/target": -0.009424876421689987, + "ronpo/target_abs": 0.038498423993587494, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 390 + }, + { + "drift/chosen_abs": 0.21178297698497772, + "drift/rejected_abs": 0.0658094510436058, + "epoch": 0.12554628526023043, + "grad_norm": 3.046875, + "learning_rate": 7.428536526919602e-08, + "logits/chosen": -2.158111095428467, + "logits/rejected": -2.221972942352295, + "logps/chosen": -0.31177249550819397, + "logps/rejected": -0.30004364252090454, + "loss": 0.3803747296333313, + "loss/core": 0.3620082139968872, + "loss/preference_sft": 0.306712806224823, + "loss/reference_anchor": 0.21421615779399872, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.117314100265503, + "rewards/margins": 1.459604024887085, + "rewards/rejected": 0.657710075378418, + "ronpo/logit": 0.14596042037010193, + "ronpo/residual": 0.1454031765460968, + "ronpo/residual_abs": 0.16442766785621643, + "ronpo/target": 0.0005572382360696793, + "ronpo/target_abs": 0.03522815182805061, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 395 + }, + { + "drift/chosen_abs": 0.19120144844055176, + "drift/rejected_abs": 0.08451732248067856, + "epoch": 0.12713547874453715, + "grad_norm": 9.5, + "learning_rate": 7.338287964462934e-08, + "logits/chosen": -2.2306790351867676, + "logits/rejected": -2.3209800720214844, + "logps/chosen": -0.25742968916893005, + "logps/rejected": -0.2603406012058258, + "loss": 0.07361692190170288, + "loss/core": 0.06260836124420166, + "loss/preference_sft": 0.25700175762176514, + "loss/reference_anchor": 0.12108077108860016, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 1.9112262725830078, + "rewards/margins": 1.0695370435714722, + "rewards/rejected": 0.8416891098022461, + "ronpo/logit": 0.1069537028670311, + "ronpo/residual": 0.10586754232645035, + "ronpo/residual_abs": 0.12485964596271515, + "ronpo/target": 0.0010861654300242662, + "ronpo/target_abs": 0.037036895751953125, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 400 + }, + { + "drift/chosen_abs": 0.17024962604045868, + "drift/rejected_abs": 0.14562097191810608, + "epoch": 0.12872467222884387, + "grad_norm": 1.8125, + "learning_rate": 7.247053577478954e-08, + "logits/chosen": -2.2906746864318848, + "logits/rejected": -2.299346446990967, + "logps/chosen": -0.29554539918899536, + "logps/rejected": -0.2890940010547638, + "loss": 0.2859688401222229, + "loss/core": 0.2667706608772278, + "loss/preference_sft": 0.2862167954444885, + "loss/reference_anchor": 0.22735345363616943, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.6998668909072876, + "rewards/margins": 0.24389216303825378, + "rewards/rejected": 1.4559749364852905, + "ronpo/logit": 0.02438921108841896, + "ronpo/residual": 0.024855410680174828, + "ronpo/residual_abs": 0.1514817178249359, + "ronpo/target": -0.0004661986604332924, + "ronpo/target_abs": 0.03743233159184456, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 405 + }, + { + "drift/chosen_abs": 0.1870613545179367, + "drift/rejected_abs": 0.05845289304852486, + "epoch": 0.13031386571315057, + "grad_norm": 0.78515625, + "learning_rate": 7.154871830475796e-08, + "logits/chosen": -2.2287261486053467, + "logits/rejected": -2.140763759613037, + "logps/chosen": -0.2894555628299713, + "logps/rejected": -0.287057101726532, + "loss": 0.3428201377391815, + "loss/core": 0.32726067304611206, + "loss/preference_sft": 0.28788700699806213, + "loss/reference_anchor": 0.1786712408065796, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.8684613704681396, + "rewards/margins": 1.3022751808166504, + "rewards/rejected": 0.566186249256134, + "ronpo/logit": 0.1302274912595749, + "ronpo/residual": 0.13139872252941132, + "ronpo/residual_abs": 0.15106776356697083, + "ronpo/target": -0.0011712164850905538, + "ronpo/target_abs": 0.035140570253133774, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 410 + }, + { + "drift/chosen_abs": 0.12820783257484436, + "drift/rejected_abs": 0.059625618159770966, + "epoch": 0.1319030591974573, + "grad_norm": 18.875, + "learning_rate": 7.061781587369517e-08, + "logits/chosen": -2.267620086669922, + "logits/rejected": -2.297600269317627, + "logps/chosen": -0.2848357558250427, + "logps/rejected": -0.2741553485393524, + "loss": 0.03822881355881691, + "loss/core": 0.03276274353265762, + "loss/preference_sft": 0.2775842547416687, + "loss/reference_anchor": 0.04512251168489456, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.2819339036941528, + "rewards/margins": 0.6913110017776489, + "rewards/rejected": 0.5906227827072144, + "ronpo/logit": 0.06913109868764877, + "ronpo/residual": 0.06746384501457214, + "ronpo/residual_abs": 0.09230469167232513, + "ronpo/target": 0.0016672604251652956, + "ronpo/target_abs": 0.03702988475561142, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 415 + }, + { + "drift/chosen_abs": 0.39186540246009827, + "drift/rejected_abs": 0.12389961630105972, + "epoch": 0.133492252681764, + "grad_norm": 2.9375, + "learning_rate": 6.96782209509903e-08, + "logits/chosen": -2.114809513092041, + "logits/rejected": -2.2321271896362305, + "logps/chosen": -0.2693604826927185, + "logps/rejected": -0.2715781331062317, + "loss": 0.8839341998100281, + "loss/core": 0.8307901620864868, + "loss/preference_sft": 0.2737378180027008, + "loss/reference_anchor": 0.6812156438827515, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.918318271636963, + "rewards/margins": 2.7466630935668945, + "rewards/rejected": 1.1716548204421997, + "ronpo/logit": 0.27466633915901184, + "ronpo/residual": 0.26872462034225464, + "ronpo/residual_abs": 0.286480188369751, + "ronpo/target": 0.005941734649240971, + "ronpo/target_abs": 0.039312396198511124, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 420 + }, + { + "drift/chosen_abs": 0.4238465428352356, + "drift/rejected_abs": 0.1388479471206665, + "epoch": 0.1350814461660707, + "grad_norm": 306.0, + "learning_rate": 6.87303296707956e-08, + "logits/chosen": -2.164503574371338, + "logits/rejected": -2.157972812652588, + "logps/chosen": -0.2988634407520294, + "logps/rejected": -0.29640263319015503, + "loss": 1.2289950847625732, + "loss/core": 1.1507900953292847, + "loss/preference_sft": 0.2907932698726654, + "loss/reference_anchor": 1.0136525630950928, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.237428188323975, + "rewards/margins": 2.8503975868225098, + "rewards/rejected": 1.3870306015014648, + "ronpo/logit": 0.2850397527217865, + "ronpo/residual": 0.27526021003723145, + "ronpo/residual_abs": 0.308517187833786, + "ronpo/target": 0.009779555723071098, + "ronpo/target_abs": 0.04520432651042938, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 425 + }, + { + "drift/chosen_abs": 0.18785476684570312, + "drift/rejected_abs": 0.09160666167736053, + "epoch": 0.13667063965037743, + "grad_norm": 11.9375, + "learning_rate": 6.777454166501589e-08, + "logits/chosen": -2.4303414821624756, + "logits/rejected": -2.4165592193603516, + "logps/chosen": -0.27914959192276, + "logps/rejected": -0.27462783455848694, + "loss": 0.09819485992193222, + "loss/core": 0.08552583307027817, + "loss/preference_sft": 0.2734072804450989, + "loss/reference_anchor": 0.1415795087814331, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.8771387338638306, + "rewards/margins": 0.9617449641227722, + "rewards/rejected": 0.9153939485549927, + "ronpo/logit": 0.09617450088262558, + "ronpo/residual": 0.0931866317987442, + "ronpo/residual_abs": 0.11439953744411469, + "ronpo/target": 0.002987860469147563, + "ronpo/target_abs": 0.033472005277872086, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 430 + }, + { + "drift/chosen_abs": 0.17324425280094147, + "drift/rejected_abs": 0.09321893751621246, + "epoch": 0.13825983313468415, + "grad_norm": 11.875, + "learning_rate": 6.681125989482337e-08, + "logits/chosen": -2.1825103759765625, + "logits/rejected": -2.1760473251342773, + "logps/chosen": -0.27965161204338074, + "logps/rejected": -0.2805442810058594, + "loss": 0.04018530249595642, + "loss/core": 0.03218042477965355, + "loss/preference_sft": 0.2764553427696228, + "loss/reference_anchor": 0.07908622920513153, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.7314059734344482, + "rewards/margins": 0.7999973297119141, + "rewards/rejected": 0.9314085841178894, + "ronpo/logit": 0.07999972999095917, + "ronpo/residual": 0.0743856281042099, + "ronpo/residual_abs": 0.1003790944814682, + "ronpo/target": 0.005614105612039566, + "ronpo/target_abs": 0.033698342740535736, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 435 + }, + { + "drift/chosen_abs": 0.31887176632881165, + "drift/rejected_abs": 0.1319849044084549, + "epoch": 0.13984902661899087, + "grad_norm": 3.984375, + "learning_rate": 6.584089048076865e-08, + "logits/chosen": -2.2736406326293945, + "logits/rejected": -2.299166440963745, + "logps/chosen": -0.2946208119392395, + "logps/rejected": -0.2924998998641968, + "loss": 0.7576056718826294, + "loss/core": 0.6892427206039429, + "loss/preference_sft": 0.299308180809021, + "loss/reference_anchor": 0.8815743327140808, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.1881179809570312, + "rewards/margins": 1.878112554550171, + "rewards/rejected": 1.3100051879882812, + "ronpo/logit": 0.1878112554550171, + "ronpo/residual": 0.18988080322742462, + "ronpo/residual_abs": 0.21202602982521057, + "ronpo/target": -0.002069511916488409, + "ronpo/target_abs": 0.04406188428401947, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 440 + }, + { + "drift/chosen_abs": 0.19031694531440735, + "drift/rejected_abs": 0.16893598437309265, + "epoch": 0.14143822010329757, + "grad_norm": 264.0, + "learning_rate": 6.486384253156013e-08, + "logits/chosen": -2.2305800914764404, + "logits/rejected": -2.154871702194214, + "logps/chosen": -0.28044503927230835, + "logps/rejected": -0.2716797888278961, + "loss": 0.24689729511737823, + "loss/core": 0.22590681910514832, + "loss/preference_sft": 0.2807045876979828, + "loss/reference_anchor": 0.2518024444580078, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.9031444787979126, + "rewards/margins": 0.2139008492231369, + "rewards/rejected": 1.6892436742782593, + "ronpo/logit": 0.021390080451965332, + "ronpo/residual": 0.023730376735329628, + "ronpo/residual_abs": 0.16236847639083862, + "ronpo/target": -0.0023402937222272158, + "ronpo/target_abs": 0.0374097041785717, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 445 + }, + { + "drift/chosen_abs": 0.3165225088596344, + "drift/rejected_abs": 0.10332963615655899, + "epoch": 0.1430274135876043, + "grad_norm": 32.25, + "learning_rate": 6.388052797158323e-08, + "logits/chosen": -2.2609755992889404, + "logits/rejected": -2.325429677963257, + "logps/chosen": -0.271506130695343, + "logps/rejected": -0.2823403477668762, + "loss": 0.7201793193817139, + "loss/core": 0.6770936846733093, + "loss/preference_sft": 0.27659398317337036, + "loss/reference_anchor": 0.5468156933784485, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.1648669242858887, + "rewards/margins": 2.1341538429260254, + "rewards/rejected": 1.030713438987732, + "ronpo/logit": 0.21341535449028015, + "ronpo/residual": 0.21813836693763733, + "ronpo/residual_abs": 0.23593993484973907, + "ronpo/target": -0.004723007790744305, + "ronpo/target_abs": 0.03459266573190689, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 450 + }, + { + "drift/chosen_abs": 0.20675233006477356, + "drift/rejected_abs": 0.09977269917726517, + "epoch": 0.144616607071911, + "grad_norm": 10.1875, + "learning_rate": 6.289136136723267e-08, + "logits/chosen": -2.3178977966308594, + "logits/rejected": -2.3515288829803467, + "logps/chosen": -0.28541308641433716, + "logps/rejected": -0.2820412218570709, + "loss": 0.10540688037872314, + "loss/core": 0.09275539964437485, + "loss/preference_sft": 0.2944668233394623, + "loss/reference_anchor": 0.13923971354961395, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.067516326904297, + "rewards/margins": 1.0707690715789795, + "rewards/rejected": 0.9967470169067383, + "ronpo/logit": 0.10707690566778183, + "ronpo/residual": 0.10755382478237152, + "ronpo/residual_abs": 0.13240396976470947, + "ronpo/target": -0.0004769214428961277, + "ronpo/target_abs": 0.03637883812189102, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 455 + }, + { + "drift/chosen_abs": 0.37812691926956177, + "drift/rejected_abs": 0.16481459140777588, + "epoch": 0.1462058005562177, + "grad_norm": 117.0, + "learning_rate": 6.189675975213094e-08, + "logits/chosen": -1.8068339824676514, + "logits/rejected": -1.666780710220337, + "logps/chosen": -0.28048020601272583, + "logps/rejected": -0.29942306876182556, + "loss": 0.5667847394943237, + "loss/core": 0.5201860666275024, + "loss/preference_sft": 0.29059094190597534, + "loss/reference_anchor": 0.5922571420669556, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.775576114654541, + "rewards/margins": 2.1349546909332275, + "rewards/rejected": 1.6406214237213135, + "ronpo/logit": 0.21349544823169708, + "ronpo/residual": 0.21162322163581848, + "ronpo/residual_abs": 0.234365776181221, + "ronpo/target": 0.0018722775857895613, + "ronpo/target_abs": 0.03759382292628288, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 460 + }, + { + "drift/chosen_abs": 0.34445223212242126, + "drift/rejected_abs": 0.23401422798633575, + "epoch": 0.14779499404052443, + "grad_norm": 253.0, + "learning_rate": 6.089714245130639e-08, + "logits/chosen": -2.095383405685425, + "logits/rejected": -2.1152687072753906, + "logps/chosen": -0.2696693539619446, + "logps/rejected": -0.2734582722187042, + "loss": 0.82310551404953, + "loss/core": 0.7762866020202637, + "loss/preference_sft": 0.2763036787509918, + "loss/reference_anchor": 0.5966211557388306, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.4437789916992188, + "rewards/margins": 1.104857087135315, + "rewards/rejected": 2.3389220237731934, + "ronpo/logit": 0.11048571765422821, + "ronpo/residual": 0.1046980768442154, + "ronpo/residual_abs": 0.27617347240448, + "ronpo/target": 0.005787642206996679, + "ronpo/target_abs": 0.03396866098046303, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 465 + }, + { + "drift/chosen_abs": 0.2995480000972748, + "drift/rejected_abs": 0.11703862994909286, + "epoch": 0.14938418752483115, + "grad_norm": 3.09375, + "learning_rate": 5.98929309044055e-08, + "logits/chosen": -2.0731537342071533, + "logits/rejected": -2.0429816246032715, + "logps/chosen": -0.25429534912109375, + "logps/rejected": -0.2596786320209503, + "loss": 0.7873652577400208, + "loss/core": 0.7461782693862915, + "loss/preference_sft": 0.25899848341941833, + "loss/reference_anchor": 0.5232595205307007, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.9948370456695557, + "rewards/margins": 1.825717568397522, + "rewards/rejected": 1.1691195964813232, + "ronpo/logit": 0.18257175385951996, + "ronpo/residual": 0.18372610211372375, + "ronpo/residual_abs": 0.22811272740364075, + "ronpo/target": -0.0011543564032763243, + "ronpo/target_abs": 0.039658524096012115, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 470 + }, + { + "drift/chosen_abs": 0.2512029707431793, + "drift/rejected_abs": 0.09701138734817505, + "epoch": 0.15097338100913787, + "grad_norm": 7.75, + "learning_rate": 5.888454848801344e-08, + "logits/chosen": -2.195828676223755, + "logits/rejected": -2.1313068866729736, + "logps/chosen": -0.30464741587638855, + "logps/rejected": -0.2989063858985901, + "loss": 0.21697983145713806, + "loss/core": 0.20200279355049133, + "loss/preference_sft": 0.3056924343109131, + "loss/reference_anchor": 0.169124573469162, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.510868787765503, + "rewards/margins": 1.5423170328140259, + "rewards/rejected": 0.9685519933700562, + "ronpo/logit": 0.1542317122220993, + "ronpo/residual": 0.1569804847240448, + "ronpo/residual_abs": 0.1833471953868866, + "ronpo/target": -0.0027487764600664377, + "ronpo/target_abs": 0.035739701241254807, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 475 + }, + { + "drift/chosen_abs": 0.221854567527771, + "drift/rejected_abs": 0.10892774909734726, + "epoch": 0.15256257449344457, + "grad_norm": 1.4921875, + "learning_rate": 5.787242033715808e-08, + "logits/chosen": -2.1198408603668213, + "logits/rejected": -2.14213490486145, + "logps/chosen": -0.2731625437736511, + "logps/rejected": -0.2686478793621063, + "loss": 0.0852787047624588, + "loss/core": 0.07367287576198578, + "loss/preference_sft": 0.26938527822494507, + "loss/reference_anchor": 0.12780579924583435, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.216320753097534, + "rewards/margins": 1.1356593370437622, + "rewards/rejected": 1.0806615352630615, + "ronpo/logit": 0.11356594413518906, + "ronpo/residual": 0.12406400591135025, + "ronpo/residual_abs": 0.13756707310676575, + "ronpo/target": -0.010498052462935448, + "ronpo/target_abs": 0.03645600751042366, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 480 + }, + { + "drift/chosen_abs": 0.29766806960105896, + "drift/rejected_abs": 0.1601838320493698, + "epoch": 0.1541517679777513, + "grad_norm": 6.1875, + "learning_rate": 5.685697316607273e-08, + "logits/chosen": -1.901150107383728, + "logits/rejected": -2.023740530014038, + "logps/chosen": -0.2827039659023285, + "logps/rejected": -0.28020596504211426, + "loss": 0.13552992045879364, + "loss/core": 0.10949452966451645, + "loss/preference_sft": 0.2870265245437622, + "loss/reference_anchor": 0.31843602657318115, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.975731134414673, + "rewards/margins": 1.3740638494491577, + "rewards/rejected": 1.601667046546936, + "ronpo/logit": 0.1374063938856125, + "ronpo/residual": 0.13112321496009827, + "ronpo/residual_abs": 0.15812107920646667, + "ronpo/target": 0.006283201277256012, + "ronpo/target_abs": 0.04036698862910271, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 485 + }, + { + "drift/chosen_abs": 0.3679271936416626, + "drift/rejected_abs": 0.15137556195259094, + "epoch": 0.15574096146205801, + "grad_norm": 362.0, + "learning_rate": 5.583863508829281e-08, + "logits/chosen": -2.1041316986083984, + "logits/rejected": -2.0852389335632324, + "logps/chosen": -0.28241246938705444, + "logps/rejected": -0.282585084438324, + "loss": 0.681788980960846, + "loss/core": 0.6289718747138977, + "loss/preference_sft": 0.2821527421474457, + "loss/reference_anchor": 0.6760126948356628, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.6771881580352783, + "rewards/margins": 2.1641111373901367, + "rewards/rejected": 1.5130772590637207, + "ronpo/logit": 0.21641111373901367, + "ronpo/residual": 0.21629562973976135, + "ronpo/residual_abs": 0.23846352100372314, + "ronpo/target": 0.00011549964256118983, + "ronpo/target_abs": 0.039183907210826874, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 490 + }, + { + "drift/chosen_abs": 0.22022119164466858, + "drift/rejected_abs": 0.18502850830554962, + "epoch": 0.1573301549463647, + "grad_norm": 264.0, + "learning_rate": 5.4817835436163e-08, + "logits/chosen": -2.1266822814941406, + "logits/rejected": -2.09769344329834, + "logps/chosen": -0.286654531955719, + "logps/rejected": -0.27344611287117004, + "loss": 1.2056705951690674, + "loss/core": 1.1531113386154175, + "loss/preference_sft": 0.28491732478141785, + "loss/reference_anchor": 0.672301709651947, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.2011239528656006, + "rewards/margins": 0.3535767197608948, + "rewards/rejected": 1.8475472927093506, + "ronpo/logit": 0.03535768389701843, + "ronpo/residual": 0.03558214753866196, + "ronpo/residual_abs": 0.2800794839859009, + "ronpo/target": -0.00022447053925134242, + "ronpo/target_abs": 0.04289742931723595, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 495 + }, + { + "drift/chosen_abs": 0.24428431689739227, + "drift/rejected_abs": 0.14757463335990906, + "epoch": 0.15891934843067143, + "grad_norm": 4.09375, + "learning_rate": 5.379500457983005e-08, + "logits/chosen": -2.233088493347168, + "logits/rejected": -2.089561939239502, + "logps/chosen": -0.28068411350250244, + "logps/rejected": -0.27640584111213684, + "loss": 0.2265290468931198, + "loss/core": 0.203072190284729, + "loss/preference_sft": 0.2757698893547058, + "loss/reference_anchor": 0.2851807475090027, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.442843437194824, + "rewards/margins": 0.9873887896537781, + "rewards/rejected": 1.4554545879364014, + "ronpo/logit": 0.09873887896537781, + "ronpo/residual": 0.10049834102392197, + "ronpo/residual_abs": 0.17442360520362854, + "ronpo/target": -0.0017594577511772513, + "ronpo/target_abs": 0.03850649669766426, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 500 + }, + { + "drift/chosen_abs": 0.1610635221004486, + "drift/rejected_abs": 0.1057731881737709, + "epoch": 0.16050854191497815, + "grad_norm": 4.28125, + "learning_rate": 5.27705737457985e-08, + "logits/chosen": -2.282769203186035, + "logits/rejected": -2.203336715698242, + "logps/chosen": -0.29518648982048035, + "logps/rejected": -0.28152212500572205, + "loss": 0.027087271213531494, + "loss/core": 0.01963396742939949, + "loss/preference_sft": 0.2921666204929352, + "loss/reference_anchor": 0.07016074657440186, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.6077709197998047, + "rewards/margins": 0.5503221154212952, + "rewards/rejected": 1.0574488639831543, + "ronpo/logit": 0.055032216012477875, + "ronpo/residual": 0.04476800560951233, + "ronpo/residual_abs": 0.08265403658151627, + "ronpo/target": 0.010264214128255844, + "ronpo/target_abs": 0.03978971391916275, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 505 + }, + { + "drift/chosen_abs": 0.31371009349823, + "drift/rejected_abs": 0.16116222739219666, + "epoch": 0.16209773539928488, + "grad_norm": 2.484375, + "learning_rate": 5.1744974835125056e-08, + "logits/chosen": -1.9395443201065063, + "logits/rejected": -1.9623817205429077, + "logps/chosen": -0.263325572013855, + "logps/rejected": -0.26283055543899536, + "loss": 0.43391937017440796, + "loss/core": 0.3860779404640198, + "loss/preference_sft": 0.26411715149879456, + "loss/reference_anchor": 0.6114745140075684, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.135848045349121, + "rewards/margins": 1.5270367860794067, + "rewards/rejected": 1.608811378479004, + "ronpo/logit": 0.1527036726474762, + "ronpo/residual": 0.1507427990436554, + "ronpo/residual_abs": 0.1738193929195404, + "ronpo/target": 0.00196089968085289, + "ronpo/target_abs": 0.034039102494716644, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 510 + }, + { + "drift/chosen_abs": 0.2592666745185852, + "drift/rejected_abs": 0.1546706259250641, + "epoch": 0.16368692888359157, + "grad_norm": 5.0, + "learning_rate": 5.071864024132867e-08, + "logits/chosen": -2.1409928798675537, + "logits/rejected": -2.2105953693389893, + "logps/chosen": -0.2879509925842285, + "logps/rejected": -0.2745327353477478, + "loss": 0.34892356395721436, + "loss/core": 0.3256063461303711, + "loss/preference_sft": 0.2804018259048462, + "loss/reference_anchor": 0.2828558087348938, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.5926671028137207, + "rewards/margins": 1.0476982593536377, + "rewards/rejected": 1.544968843460083, + "ronpo/logit": 0.10476982593536377, + "ronpo/residual": 0.10065307468175888, + "ronpo/residual_abs": 0.19120723009109497, + "ronpo/target": 0.0041167414747178555, + "ronpo/target_abs": 0.037326134741306305, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 515 + }, + { + "drift/chosen_abs": 0.21726028621196747, + "drift/rejected_abs": 0.10538153350353241, + "epoch": 0.1652761223678983, + "grad_norm": 2.859375, + "learning_rate": 4.969200266809307e-08, + "logits/chosen": -2.1490025520324707, + "logits/rejected": -2.16178560256958, + "logps/chosen": -0.29243892431259155, + "logps/rejected": -0.30090299248695374, + "loss": 0.26550185680389404, + "loss/core": 0.22479858994483948, + "loss/preference_sft": 0.2937026917934418, + "loss/reference_anchor": 0.5133408904075623, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.171968460083008, + "rewards/margins": 1.124643087387085, + "rewards/rejected": 1.0473252534866333, + "ronpo/logit": 0.1124643087387085, + "ronpo/residual": 0.10653400421142578, + "ronpo/residual_abs": 0.131363645195961, + "ronpo/target": 0.0059303115122020245, + "ronpo/target_abs": 0.04164915904402733, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 520 + }, + { + "drift/chosen_abs": 0.21974170207977295, + "drift/rejected_abs": 0.06504969298839569, + "epoch": 0.16686531585220502, + "grad_norm": 7.75, + "learning_rate": 4.866549494683838e-08, + "logits/chosen": -1.9057044982910156, + "logits/rejected": -2.0494656562805176, + "logps/chosen": -0.31208372116088867, + "logps/rejected": -0.3202230632305145, + "loss": 0.267322301864624, + "loss/core": 0.25114768743515015, + "loss/preference_sft": 0.3229205012321472, + "loss/reference_anchor": 0.18336977064609528, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.1963610649108887, + "rewards/margins": 1.5482251644134521, + "rewards/rejected": 0.6481360197067261, + "ronpo/logit": 0.15482252836227417, + "ronpo/residual": 0.16023744642734528, + "ronpo/residual_abs": 0.17627668380737305, + "ronpo/target": -0.005414927843958139, + "ronpo/target_abs": 0.04029358923435211, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 525 + }, + { + "drift/chosen_abs": 0.15339070558547974, + "drift/rejected_abs": 0.06500252336263657, + "epoch": 0.1684545093365117, + "grad_norm": 1.15625, + "learning_rate": 4.763954985423904e-08, + "logits/chosen": -2.323418378829956, + "logits/rejected": -2.318234920501709, + "logps/chosen": -0.2666859030723572, + "logps/rejected": -0.2610042989253998, + "loss": 0.056935809552669525, + "loss/core": 0.05055410414934158, + "loss/preference_sft": 0.2673361599445343, + "loss/reference_anchor": 0.0583559088408947, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 1.533581256866455, + "rewards/margins": 0.8862428665161133, + "rewards/rejected": 0.6473384499549866, + "ronpo/logit": 0.08862429112195969, + "ronpo/residual": 0.09020157158374786, + "ronpo/residual_abs": 0.1063581109046936, + "ronpo/target": -0.001577280811034143, + "ronpo/target_abs": 0.03666796162724495, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 530 + }, + { + "drift/chosen_abs": 0.3485838770866394, + "drift/rejected_abs": 0.14534269273281097, + "epoch": 0.17004370282081843, + "grad_norm": 2.875, + "learning_rate": 4.6614599929764626e-08, + "logits/chosen": -2.2419562339782715, + "logits/rejected": -2.2564449310302734, + "logps/chosen": -0.3141189515590668, + "logps/rejected": -0.32258644700050354, + "loss": 1.5565026998519897, + "loss/core": 1.473420262336731, + "loss/preference_sft": 0.31737521290779114, + "loss/reference_anchor": 1.0760281085968018, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.48555326461792, + "rewards/margins": 2.219147205352783, + "rewards/rejected": 1.2664062976837158, + "ronpo/logit": 0.22191467881202698, + "ronpo/residual": 0.2292250096797943, + "ronpo/residual_abs": 0.3092333674430847, + "ronpo/target": -0.007310344371944666, + "ronpo/target_abs": 0.04150359705090523, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 535 + }, + { + "drift/chosen_abs": 0.24426977336406708, + "drift/rejected_abs": 0.10654947906732559, + "epoch": 0.17163289630512515, + "grad_norm": 8.375, + "learning_rate": 4.5591077293320914e-08, + "logits/chosen": -2.2026126384735107, + "logits/rejected": -2.239305019378662, + "logps/chosen": -0.2799997925758362, + "logps/rejected": -0.2913915812969208, + "loss": 0.22537103295326233, + "loss/core": 0.19563640654087067, + "loss/preference_sft": 0.28852710127830505, + "loss/reference_anchor": 0.36760905385017395, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.4418580532073975, + "rewards/margins": 1.3771570920944214, + "rewards/rejected": 1.0647010803222656, + "ronpo/logit": 0.13771569728851318, + "ronpo/residual": 0.1375226229429245, + "ronpo/residual_abs": 0.1619495302438736, + "ronpo/target": 0.00019309911294840276, + "ronpo/target_abs": 0.03357410058379173, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 540 + }, + { + "drift/chosen_abs": 0.39887771010398865, + "drift/rejected_abs": 0.2444618195295334, + "epoch": 0.17322208978943188, + "grad_norm": 5.25, + "learning_rate": 4.45694134630676e-08, + "logits/chosen": -1.9924676418304443, + "logits/rejected": -2.0167553424835205, + "logps/chosen": -0.2603081166744232, + "logps/rejected": -0.2806777060031891, + "loss": 0.48962488770484924, + "loss/core": 0.42624610662460327, + "loss/preference_sft": 0.27903109788894653, + "loss/reference_anchor": 0.8171467781066895, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.987790584564209, + "rewards/margins": 1.5475997924804688, + "rewards/rejected": 2.4401907920837402, + "ronpo/logit": 0.1547599732875824, + "ronpo/residual": 0.15789082646369934, + "ronpo/residual_abs": 0.23713842034339905, + "ronpo/target": -0.003130866913124919, + "ronpo/target_abs": 0.04018576815724373, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 545 + }, + { + "drift/chosen_abs": 0.2760270833969116, + "drift/rejected_abs": 0.1387709677219391, + "epoch": 0.17481128327373857, + "grad_norm": 1.25, + "learning_rate": 4.355003917348984e-08, + "logits/chosen": -2.1436705589294434, + "logits/rejected": -2.1003220081329346, + "logps/chosen": -0.25462886691093445, + "logps/rejected": -0.2643481194972992, + "loss": 0.3101690411567688, + "loss/core": 0.28291812539100647, + "loss/preference_sft": 0.26238495111465454, + "loss/reference_anchor": 0.3371073007583618, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.759854793548584, + "rewards/margins": 1.3732728958129883, + "rewards/rejected": 1.3865820169448853, + "ronpo/logit": 0.13732729852199554, + "ronpo/residual": 0.1491035521030426, + "ronpo/residual_abs": 0.17806217074394226, + "ronpo/target": -0.011776268482208252, + "ronpo/target_abs": 0.04159698635339737, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 550 + }, + { + "drift/chosen_abs": 0.12518154084682465, + "drift/rejected_abs": 0.05519000440835953, + "epoch": 0.1764004767580453, + "grad_norm": 0.412109375, + "learning_rate": 4.253338419380016e-08, + "logits/chosen": -2.2161974906921387, + "logits/rejected": -2.2592287063598633, + "logps/chosen": -0.3115261197090149, + "logps/rejected": -0.3088235855102539, + "loss": 0.06282468140125275, + "loss/core": 0.05494839698076248, + "loss/preference_sft": 0.30944591760635376, + "loss/reference_anchor": 0.07407257705926895, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.2518155574798584, + "rewards/margins": 0.7002865076065063, + "rewards/rejected": 0.551529049873352, + "ronpo/logit": 0.0700286477804184, + "ronpo/residual": 0.0636088103055954, + "ronpo/residual_abs": 0.09568959474563599, + "ronpo/target": 0.006419846322387457, + "ronpo/target_abs": 0.04170956462621689, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 555 + }, + { + "drift/chosen_abs": 0.2650905251502991, + "drift/rejected_abs": 0.1225300058722496, + "epoch": 0.17798967024235202, + "grad_norm": 17.0, + "learning_rate": 4.151987714674736e-08, + "logits/chosen": -2.084272861480713, + "logits/rejected": -2.0913736820220947, + "logps/chosen": -0.2723826766014099, + "logps/rejected": -0.27883923053741455, + "loss": 0.2584993243217468, + "loss/core": 0.23772597312927246, + "loss/preference_sft": 0.27814561128616333, + "loss/reference_anchor": 0.24916362762451172, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.6494510173797607, + "rewards/margins": 1.4253307580947876, + "rewards/rejected": 1.224120020866394, + "ronpo/logit": 0.1425330638885498, + "ronpo/residual": 0.14170929789543152, + "ronpo/residual_abs": 0.15773284435272217, + "ronpo/target": 0.0008237810689024627, + "ronpo/target_abs": 0.032286323606967926, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 560 + }, + { + "drift/chosen_abs": 0.2599169611930847, + "drift/rejected_abs": 0.11472652852535248, + "epoch": 0.1795788637266587, + "grad_norm": 29.875, + "learning_rate": 4.0509945327908704e-08, + "logits/chosen": -2.0401737689971924, + "logits/rejected": -2.0661861896514893, + "logps/chosen": -0.34084010124206543, + "logps/rejected": -0.3358941674232483, + "loss": 0.36931371688842773, + "loss/core": 0.33606022596359253, + "loss/preference_sft": 0.34215277433395386, + "loss/reference_anchor": 0.40916499495506287, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.597136974334717, + "rewards/margins": 1.4537303447723389, + "rewards/rejected": 1.1434067487716675, + "ronpo/logit": 0.14537303149700165, + "ronpo/residual": 0.14311483502388, + "ronpo/residual_abs": 0.16634781658649445, + "ronpo/target": 0.0022582041565328836, + "ronpo/target_abs": 0.036671485751867294, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 565 + }, + { + "drift/chosen_abs": 0.4108946919441223, + "drift/rejected_abs": 0.10000451654195786, + "epoch": 0.18116805721096543, + "grad_norm": 12.375, + "learning_rate": 3.950401452554171e-08, + "logits/chosen": -2.1480000019073486, + "logits/rejected": -2.1182470321655273, + "logps/chosen": -0.2765151560306549, + "logps/rejected": -0.2645667493343353, + "loss": 1.3812847137451172, + "loss/core": 1.3190586566925049, + "loss/preference_sft": 0.27181077003479004, + "loss/reference_anchor": 0.8025016784667969, + "rewards/accuracies": 0.875, + "rewards/chosen": 4.108946800231934, + "rewards/margins": 3.11163330078125, + "rewards/rejected": 0.9973136186599731, + "ronpo/logit": 0.3111633360385895, + "ronpo/residual": 0.30616316199302673, + "ronpo/residual_abs": 0.3383898138999939, + "ronpo/target": 0.005000181496143341, + "ronpo/target_abs": 0.046630989760160446, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 570 + }, + { + "drift/chosen_abs": 0.19218239188194275, + "drift/rejected_abs": 0.09658966213464737, + "epoch": 0.18275725069527216, + "grad_norm": 3.421875, + "learning_rate": 3.850250884107133e-08, + "logits/chosen": -2.0075910091400146, + "logits/rejected": -2.094909429550171, + "logps/chosen": -0.28321409225463867, + "logps/rejected": -0.27906012535095215, + "loss": 0.05575397610664368, + "loss/core": 0.04724646359682083, + "loss/preference_sft": 0.2887270748615265, + "loss/reference_anchor": 0.0845608115196228, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.9186960458755493, + "rewards/margins": 0.9540292620658875, + "rewards/rejected": 0.9646669626235962, + "ronpo/logit": 0.09540291875600815, + "ronpo/residual": 0.0999816432595253, + "ronpo/residual_abs": 0.1167878657579422, + "ronpo/target": -0.004578724503517151, + "ronpo/target_abs": 0.03717956319451332, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 575 + }, + { + "drift/chosen_abs": 0.36307868361473083, + "drift/rejected_abs": 0.16689428687095642, + "epoch": 0.18434644417957885, + "grad_norm": 2.125, + "learning_rate": 3.750585051028845e-08, + "logits/chosen": -2.0031397342681885, + "logits/rejected": -2.033116102218628, + "logps/chosen": -0.29011231660842896, + "logps/rejected": -0.2679058611392975, + "loss": 0.5739777684211731, + "loss/core": 0.5245698690414429, + "loss/preference_sft": 0.2851426303386688, + "loss/reference_anchor": 0.630257248878479, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.629528045654297, + "rewards/margins": 1.9614536762237549, + "rewards/rejected": 1.6680740118026733, + "ronpo/logit": 0.19614538550376892, + "ronpo/residual": 0.19045701622962952, + "ronpo/residual_abs": 0.23201608657836914, + "ronpo/target": 0.005688379053026438, + "ronpo/target_abs": 0.03336022049188614, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 580 + }, + { + "drift/chosen_abs": 0.23189207911491394, + "drift/rejected_abs": 0.09468026459217072, + "epoch": 0.18593563766388557, + "grad_norm": 1.671875, + "learning_rate": 3.651445972533487e-08, + "logits/chosen": -1.9576046466827393, + "logits/rejected": -1.9823410511016846, + "logps/chosen": -0.2863709330558777, + "logps/rejected": -0.2869415879249573, + "loss": 0.1575651466846466, + "loss/core": 0.14273057878017426, + "loss/preference_sft": 0.295357882976532, + "loss/reference_anchor": 0.1682581901550293, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.3186957836151123, + "rewards/margins": 1.3729966878890991, + "rewards/rejected": 0.9456988573074341, + "ronpo/logit": 0.13729968667030334, + "ronpo/residual": 0.1349247694015503, + "ronpo/residual_abs": 0.15127669274806976, + "ronpo/target": 0.0023749221581965685, + "ronpo/target_abs": 0.03486529737710953, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 585 + }, + { + "drift/chosen_abs": 0.2609372138977051, + "drift/rejected_abs": 0.12208591401576996, + "epoch": 0.1875248311481923, + "grad_norm": 1.328125, + "learning_rate": 3.552875445754975e-08, + "logits/chosen": -2.116716146469116, + "logits/rejected": -2.0834579467773438, + "logps/chosen": -0.3057488203048706, + "logps/rejected": -0.3151412904262543, + "loss": 0.6112656593322754, + "loss/core": 0.5733734965324402, + "loss/preference_sft": 0.31387200951576233, + "loss/reference_anchor": 0.47384127974510193, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.6066231727600098, + "rewards/margins": 1.3866499662399292, + "rewards/rejected": 1.219973087310791, + "ronpo/logit": 0.13866500556468964, + "ronpo/residual": 0.14339695870876312, + "ronpo/residual_abs": 0.17332182824611664, + "ronpo/target": -0.004731960128992796, + "ronpo/target_abs": 0.03508470207452774, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 590 + }, + { + "drift/chosen_abs": 0.23165372014045715, + "drift/rejected_abs": 0.11231007426977158, + "epoch": 0.18911402463249902, + "grad_norm": 5.34375, + "learning_rate": 3.4549150281252633e-08, + "logits/chosen": -1.9744529724121094, + "logits/rejected": -1.9331066608428955, + "logps/chosen": -0.2898443341255188, + "logps/rejected": -0.29342323541641235, + "loss": 0.20584657788276672, + "loss/core": 0.18938076496124268, + "loss/preference_sft": 0.2938799262046814, + "loss/reference_anchor": 0.1901559680700302, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.314818859100342, + "rewards/margins": 1.1970219612121582, + "rewards/rejected": 1.1177968978881836, + "ronpo/logit": 0.11970218271017075, + "ronpo/residual": 0.1114286333322525, + "ronpo/residual_abs": 0.1585887372493744, + "ronpo/target": 0.008273562416434288, + "ronpo/target_abs": 0.04056030511856079, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 595 + }, + { + "drift/chosen_abs": 0.2051103115081787, + "drift/rejected_abs": 0.1440320909023285, + "epoch": 0.1907032181168057, + "grad_norm": 145.0, + "learning_rate": 3.3576060198536815e-08, + "logits/chosen": -2.132131576538086, + "logits/rejected": -2.1050548553466797, + "logps/chosen": -0.29798927903175354, + "logps/rejected": -0.318352609872818, + "loss": 0.4632243514060974, + "loss/core": 0.43989643454551697, + "loss/preference_sft": 0.3055892586708069, + "loss/reference_anchor": 0.28048065304756165, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.049271821975708, + "rewards/margins": 0.6105486154556274, + "rewards/rejected": 1.4387229681015015, + "ronpo/logit": 0.06105484813451767, + "ronpo/residual": 0.062055326998233795, + "ronpo/residual_abs": 0.2071889191865921, + "ronpo/target": -0.0010004760697484016, + "ronpo/target_abs": 0.03393607214093208, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 600 + }, + { + "drift/chosen_abs": 0.34726980328559875, + "drift/rejected_abs": 0.15757516026496887, + "epoch": 0.19229241160111243, + "grad_norm": 1.3515625, + "learning_rate": 3.260989446514726e-08, + "logits/chosen": -2.0277671813964844, + "logits/rejected": -1.95284104347229, + "logps/chosen": -0.28237101435661316, + "logps/rejected": -0.2795073688030243, + "loss": 0.4946117401123047, + "loss/core": 0.4488951563835144, + "loss/preference_sft": 0.2740669250488281, + "loss/reference_anchor": 0.5821473598480225, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.4638190269470215, + "rewards/margins": 1.898168921470642, + "rewards/rejected": 1.565650224685669, + "ronpo/logit": 0.1898169070482254, + "ronpo/residual": 0.18911583721637726, + "ronpo/residual_abs": 0.21403014659881592, + "ronpo/target": 0.0007010673289187253, + "ronpo/target_abs": 0.0408640094101429, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 605 + }, + { + "drift/chosen_abs": 0.3361540138721466, + "drift/rejected_abs": 0.1532098352909088, + "epoch": 0.19388160508541916, + "grad_norm": 93.0, + "learning_rate": 3.165106041751639e-08, + "logits/chosen": -2.0205764770507812, + "logits/rejected": -1.9713436365127563, + "logps/chosen": -0.30869030952453613, + "logps/rejected": -0.2928310036659241, + "loss": 0.36041101813316345, + "loss/core": 0.3217954933643341, + "loss/preference_sft": 0.314534455537796, + "loss/reference_anchor": 0.4834206998348236, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.3594353199005127, + "rewards/margins": 1.8287744522094727, + "rewards/rejected": 1.5306605100631714, + "ronpo/logit": 0.18287745118141174, + "ronpo/residual": 0.18577490746974945, + "ronpo/residual_abs": 0.20601661503314972, + "ronpo/target": -0.0028974597807973623, + "ronpo/target_abs": 0.04099640995264053, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 610 + }, + { + "drift/chosen_abs": 0.2551515996456146, + "drift/rejected_abs": 0.11366691440343857, + "epoch": 0.19547079856972585, + "grad_norm": 128.0, + "learning_rate": 3.069996230103066e-08, + "logits/chosen": -2.1977291107177734, + "logits/rejected": -2.159012794494629, + "logps/chosen": -0.2876770496368408, + "logps/rejected": -0.28747281432151794, + "loss": 0.3087400197982788, + "loss/core": 0.2813792824745178, + "loss/preference_sft": 0.2934519648551941, + "loss/reference_anchor": 0.3354647755622864, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.551334857940674, + "rewards/margins": 1.4261997938156128, + "rewards/rejected": 1.1251349449157715, + "ronpo/logit": 0.1426199972629547, + "ronpo/residual": 0.14335501194000244, + "ronpo/residual_abs": 0.15863969922065735, + "ronpo/target": -0.0007350404048338532, + "ronpo/target_abs": 0.03268890082836151, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 615 + }, + { + "drift/chosen_abs": 0.22230342030525208, + "drift/rejected_abs": 0.10894560813903809, + "epoch": 0.19705999205403257, + "grad_norm": 12.75, + "learning_rate": 2.9757001099600225e-08, + "logits/chosen": -2.0835347175598145, + "logits/rejected": -2.018907070159912, + "logps/chosen": -0.29156553745269775, + "logps/rejected": -0.28759321570396423, + "loss": 0.10223954916000366, + "loss/core": 0.08924255520105362, + "loss/preference_sft": 0.2917032241821289, + "loss/reference_anchor": 0.1441229283809662, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.2221884727478027, + "rewards/margins": 1.1332981586456299, + "rewards/rejected": 1.0888900756835938, + "ronpo/logit": 0.11332981288433075, + "ronpo/residual": 0.11472058296203613, + "ronpo/residual_abs": 0.13595080375671387, + "ronpo/target": -0.0013907629763707519, + "ronpo/target_abs": 0.03300943225622177, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 620 + }, + { + "drift/chosen_abs": 0.2458353340625763, + "drift/rejected_abs": 0.09283015877008438, + "epoch": 0.1986491855383393, + "grad_norm": 44.5, + "learning_rate": 2.8822574366603803e-08, + "logits/chosen": -2.0361926555633545, + "logits/rejected": -2.0415730476379395, + "logps/chosen": -0.2964986562728882, + "logps/rejected": -0.298392653465271, + "loss": 0.33931902050971985, + "loss/core": 0.3200526833534241, + "loss/preference_sft": 0.298342764377594, + "loss/reference_anchor": 0.22704991698265076, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.4581804275512695, + "rewards/margins": 1.5353305339813232, + "rewards/rejected": 0.9228495359420776, + "ronpo/logit": 0.15353310108184814, + "ronpo/residual": 0.1616130769252777, + "ronpo/residual_abs": 0.1806202232837677, + "ronpo/target": -0.008080018684267998, + "ronpo/target_abs": 0.03993826359510422, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 625 + }, + { + "drift/chosen_abs": 0.23397347331047058, + "drift/rejected_abs": 0.08942471444606781, + "epoch": 0.20023837902264602, + "grad_norm": 5.6875, + "learning_rate": 2.78970760572797e-08, + "logits/chosen": -2.3254988193511963, + "logits/rejected": -2.4149765968322754, + "logps/chosen": -0.2674204409122467, + "logps/rejected": -0.26369109749794006, + "loss": 0.31197839975357056, + "loss/core": 0.2868371903896332, + "loss/preference_sft": 0.266465961933136, + "loss/reference_anchor": 0.30856937170028687, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 2.3387513160705566, + "rewards/margins": 1.4476630687713623, + "rewards/rejected": 0.8910881876945496, + "ronpo/logit": 0.14476630091667175, + "ronpo/residual": 0.13926377892494202, + "ronpo/residual_abs": 0.16467837989330292, + "ronpo/target": 0.005502545274794102, + "ronpo/target_abs": 0.03496618941426277, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 630 + }, + { + "drift/chosen_abs": 0.2901550531387329, + "drift/rejected_abs": 0.10847359895706177, + "epoch": 0.2018275725069527, + "grad_norm": 16.625, + "learning_rate": 2.6980896362634047e-08, + "logits/chosen": -2.1083121299743652, + "logits/rejected": -2.1480813026428223, + "logps/chosen": -0.26491206884384155, + "logps/rejected": -0.2721540033817291, + "loss": 0.3880026042461395, + "loss/core": 0.36257725954055786, + "loss/preference_sft": 0.26688578724861145, + "loss/reference_anchor": 0.3123162090778351, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.9005064964294434, + "rewards/margins": 1.8258981704711914, + "rewards/rejected": 1.074608325958252, + "ronpo/logit": 0.1825898140668869, + "ronpo/residual": 0.17556194961071014, + "ronpo/residual_abs": 0.20522348582744598, + "ronpo/target": 0.007027854211628437, + "ronpo/target_abs": 0.03924590349197388, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 635 + }, + { + "drift/chosen_abs": 0.3232152760028839, + "drift/rejected_abs": 0.09826619923114777, + "epoch": 0.20341676599125944, + "grad_norm": 1.40625, + "learning_rate": 2.6074421544935676e-08, + "logits/chosen": -2.015664577484131, + "logits/rejected": -2.0078439712524414, + "logps/chosen": -0.31468555331230164, + "logps/rejected": -0.3170844614505768, + "loss": 0.7409719824790955, + "loss/core": 0.6974954605102539, + "loss/preference_sft": 0.31186458468437195, + "loss/reference_anchor": 0.5485001802444458, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.2312824726104736, + "rewards/margins": 2.2840328216552734, + "rewards/rejected": 0.9472495913505554, + "ronpo/logit": 0.2284032553434372, + "ronpo/residual": 0.2323722094297409, + "ronpo/residual_abs": 0.2483997792005539, + "ronpo/target": -0.003968966659158468, + "ronpo/target_abs": 0.038829077035188675, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 640 + }, + { + "drift/chosen_abs": 0.2661309540271759, + "drift/rejected_abs": 0.11417138576507568, + "epoch": 0.20500595947556616, + "grad_norm": 8.1875, + "learning_rate": 2.5178033774867692e-08, + "logits/chosen": -2.1209521293640137, + "logits/rejected": -2.1270642280578613, + "logps/chosen": -0.30802121758461, + "logps/rejected": -0.2950921952724457, + "loss": 0.33442920446395874, + "loss/core": 0.30818670988082886, + "loss/preference_sft": 0.3144807517528534, + "loss/reference_anchor": 0.31845149397850037, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.659245014190674, + "rewards/margins": 1.5209836959838867, + "rewards/rejected": 1.1382615566253662, + "ronpo/logit": 0.1520983725786209, + "ronpo/residual": 0.14826013147830963, + "ronpo/residual_abs": 0.1766631305217743, + "ronpo/target": 0.0038382462225854397, + "ronpo/target_abs": 0.04023699089884758, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 645 + }, + { + "drift/chosen_abs": 0.1514950692653656, + "drift/rejected_abs": 0.0910639688372612, + "epoch": 0.20659515295987285, + "grad_norm": 1.4765625, + "learning_rate": 2.4292110970403884e-08, + "logits/chosen": -2.2066166400909424, + "logits/rejected": -2.1343297958374023, + "logps/chosen": -0.3011590242385864, + "logps/rejected": -0.29367610812187195, + "loss": 0.05876779556274414, + "loss/core": 0.05132526159286499, + "loss/preference_sft": 0.3029544949531555, + "loss/reference_anchor": 0.06893852353096008, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.5139214992523193, + "rewards/margins": 0.6618244051933289, + "rewards/rejected": 0.85209721326828, + "ronpo/logit": 0.066182442009449, + "ronpo/residual": 0.06901034712791443, + "ronpo/residual_abs": 0.09437278658151627, + "ronpo/target": -0.002827908843755722, + "ronpo/target_abs": 0.037444792687892914, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 650 + }, + { + "drift/chosen_abs": 0.20499472320079803, + "drift/rejected_abs": 0.064644455909729, + "epoch": 0.20818434644417957, + "grad_norm": 127.0, + "learning_rate": 2.3417026637478186e-08, + "logits/chosen": -2.277536392211914, + "logits/rejected": -2.283423900604248, + "logps/chosen": -0.2831169366836548, + "logps/rejected": -0.2986695170402527, + "loss": 0.38664403557777405, + "loss/core": 0.367914617061615, + "loss/preference_sft": 0.29018551111221313, + "loss/reference_anchor": 0.2207069844007492, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.0499470233917236, + "rewards/margins": 1.4070502519607544, + "rewards/rejected": 0.6428970098495483, + "ronpo/logit": 0.14070503413677216, + "ronpo/residual": 0.1532699167728424, + "ronpo/residual_abs": 0.1704498678445816, + "ronpo/target": -0.012564892880618572, + "ronpo/target_abs": 0.04341302812099457, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 655 + }, + { + "drift/chosen_abs": 0.17548860609531403, + "drift/rejected_abs": 0.07433202862739563, + "epoch": 0.2097735399284863, + "grad_norm": 6.71875, + "learning_rate": 2.255314971251419e-08, + "logits/chosen": -2.251873731613159, + "logits/rejected": -2.295532703399658, + "logps/chosen": -0.29541832208633423, + "logps/rejected": -0.2891073524951935, + "loss": 0.1282232701778412, + "loss/core": 0.11798721551895142, + "loss/preference_sft": 0.2941397428512573, + "loss/reference_anchor": 0.10706689208745956, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 1.754166841506958, + "rewards/margins": 1.0147831439971924, + "rewards/rejected": 0.7393836975097656, + "ronpo/logit": 0.10147831588983536, + "ronpo/residual": 0.10077782720327377, + "ronpo/residual_abs": 0.1172768622636795, + "ronpo/target": 0.0007004888029769063, + "ronpo/target_abs": 0.03470372408628464, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 660 + }, + { + "drift/chosen_abs": 0.2351798266172409, + "drift/rejected_abs": 0.1356552243232727, + "epoch": 0.21136273341279302, + "grad_norm": 0.65625, + "learning_rate": 2.1700844406881107e-08, + "logits/chosen": -2.0752432346343994, + "logits/rejected": -2.019031524658203, + "logps/chosen": -0.28208786249160767, + "logps/rejected": -0.28667864203453064, + "loss": 0.23971502482891083, + "loss/core": 0.21660105884075165, + "loss/preference_sft": 0.28722888231277466, + "loss/reference_anchor": 0.27946338057518005, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.348977565765381, + "rewards/margins": 0.9931402206420898, + "rewards/rejected": 1.3558372259140015, + "ronpo/logit": 0.09931402653455734, + "ronpo/residual": 0.10590650886297226, + "ronpo/residual_abs": 0.13825678825378418, + "ronpo/target": -0.006592473480850458, + "ronpo/target_abs": 0.037365201860666275, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 665 + }, + { + "drift/chosen_abs": 0.20990140736103058, + "drift/rejected_abs": 0.07856923341751099, + "epoch": 0.2129519268970997, + "grad_norm": 5.5, + "learning_rate": 2.0860470053341957e-08, + "logits/chosen": -2.3458199501037598, + "logits/rejected": -2.3533999919891357, + "logps/chosen": -0.2676357328891754, + "logps/rejected": -0.27186718583106995, + "loss": 0.21298956871032715, + "loss/core": 0.19874659180641174, + "loss/preference_sft": 0.2663695812225342, + "loss/reference_anchor": 0.16326895356178284, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.097968101501465, + "rewards/margins": 1.3122758865356445, + "rewards/rejected": 0.7856923341751099, + "ronpo/logit": 0.13122756779193878, + "ronpo/residual": 0.1469726860523224, + "ronpo/residual_abs": 0.15434160828590393, + "ronpo/target": -0.01574511080980301, + "ronpo/target_abs": 0.03544539958238602, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 670 + }, + { + "drift/chosen_abs": 0.4016236364841461, + "drift/rejected_abs": 0.07960326224565506, + "epoch": 0.21454112038140644, + "grad_norm": 2.234375, + "learning_rate": 2.0032380954558548e-08, + "logits/chosen": -1.986501693725586, + "logits/rejected": -2.0995402336120605, + "logps/chosen": -0.27848881483078003, + "logps/rejected": -0.271128386259079, + "loss": 1.4639503955841064, + "loss/core": 1.401967167854309, + "loss/preference_sft": 0.2735333740711212, + "loss/reference_anchor": 0.7990926504135132, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 4.014946460723877, + "rewards/margins": 3.2203209400177, + "rewards/rejected": 0.7946258783340454, + "ronpo/logit": 0.32203203439712524, + "ronpo/residual": 0.31797724962234497, + "ronpo/residual_abs": 0.3429466485977173, + "ronpo/target": 0.0040548210963606834, + "ronpo/target_abs": 0.04477760195732117, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 675 + }, + { + "drift/chosen_abs": 0.1950986236333847, + "drift/rejected_abs": 0.1172579675912857, + "epoch": 0.21613031386571316, + "grad_norm": 2.203125, + "learning_rate": 1.9216926233717085e-08, + "logits/chosen": -1.9547698497772217, + "logits/rejected": -1.949459433555603, + "logps/chosen": -0.28763753175735474, + "logps/rejected": -0.2842276394367218, + "loss": 0.06276907026767731, + "loss/core": 0.04688558354973793, + "loss/preference_sft": 0.2863856256008148, + "loss/reference_anchor": 0.18314120173454285, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9503726959228516, + "rewards/margins": 0.7782739400863647, + "rewards/rejected": 1.172098994255066, + "ronpo/logit": 0.07782740890979767, + "ronpo/residual": 0.07493593543767929, + "ronpo/residual_abs": 0.10520467907190323, + "ronpo/target": 0.0028914690483361483, + "ronpo/target_abs": 0.042150795459747314, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 680 + }, + { + "drift/chosen_abs": 0.1819314956665039, + "drift/rejected_abs": 0.09456787258386612, + "epoch": 0.21771950735001985, + "grad_norm": 3.0, + "learning_rate": 1.8414449687337464e-08, + "logits/chosen": -1.972158432006836, + "logits/rejected": -2.0605552196502686, + "logps/chosen": -0.2929169535636902, + "logps/rejected": -0.2997816503047943, + "loss": 0.052760761231184006, + "loss/core": 0.044637665152549744, + "loss/preference_sft": 0.3041604459285736, + "loss/reference_anchor": 0.07789192348718643, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.8181995153427124, + "rewards/margins": 0.875436007976532, + "rewards/rejected": 0.9427635073661804, + "ronpo/logit": 0.08754359930753708, + "ronpo/residual": 0.08810020983219147, + "ronpo/residual_abs": 0.10490294545888901, + "ronpo/target": -0.0005566048203036189, + "ronpo/target_abs": 0.038404613733291626, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 685 + }, + { + "drift/chosen_abs": 0.45035824179649353, + "drift/rejected_abs": 0.18560758233070374, + "epoch": 0.21930870083432658, + "grad_norm": 440.0, + "learning_rate": 1.7625289640328318e-08, + "logits/chosen": -1.9293177127838135, + "logits/rejected": -1.9729528427124023, + "logps/chosen": -0.2784627079963684, + "logps/rejected": -0.27157917618751526, + "loss": 0.891852080821991, + "loss/core": 0.807598888874054, + "loss/preference_sft": 0.2851676642894745, + "loss/reference_anchor": 1.0948574542999268, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 4.503391265869141, + "rewards/margins": 2.649190664291382, + "rewards/rejected": 1.8542003631591797, + "ronpo/logit": 0.26491910219192505, + "ronpo/residual": 0.2634193003177643, + "ronpo/residual_abs": 0.2890183627605438, + "ronpo/target": 0.001499784062616527, + "ronpo/target_abs": 0.0398874506354332, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 690 + }, + { + "drift/chosen_abs": 0.1834675371646881, + "drift/rejected_abs": 0.11008524894714355, + "epoch": 0.2208978943186333, + "grad_norm": 12.375, + "learning_rate": 1.6849778803349007e-08, + "logits/chosen": -2.160452127456665, + "logits/rejected": -2.1469032764434814, + "logps/chosen": -0.29817405343055725, + "logps/rejected": -0.3005991578102112, + "loss": 0.10696051269769669, + "loss/core": 0.09560181200504303, + "loss/preference_sft": 0.29798537492752075, + "loss/reference_anchor": 0.12165065854787827, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.8344100713729858, + "rewards/margins": 0.7348702549934387, + "rewards/rejected": 1.099540114402771, + "ronpo/logit": 0.07348702102899551, + "ronpo/residual": 0.06854233145713806, + "ronpo/residual_abs": 0.13825374841690063, + "ronpo/target": 0.004944689106196165, + "ronpo/target_abs": 0.03506334498524666, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 695 + }, + { + "drift/chosen_abs": 0.22251728177070618, + "drift/rejected_abs": 0.139739990234375, + "epoch": 0.22248708780294002, + "grad_norm": 6.21875, + "learning_rate": 1.60882441325383e-08, + "logits/chosen": -2.216174364089966, + "logits/rejected": -2.175872564315796, + "logps/chosen": -0.3221741318702698, + "logps/rejected": -0.3115740418434143, + "loss": 0.43813127279281616, + "loss/core": 0.4061037003993988, + "loss/preference_sft": 0.3150758147239685, + "loss/reference_anchor": 0.3955269455909729, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.2245700359344482, + "rewards/margins": 0.8280360102653503, + "rewards/rejected": 1.3965343236923218, + "ronpo/logit": 0.08280360698699951, + "ronpo/residual": 0.08249805122613907, + "ronpo/residual_abs": 0.17446820437908173, + "ronpo/target": 0.0003055560518987477, + "ronpo/target_abs": 0.0334026925265789, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 700 + }, + { + "drift/chosen_abs": 0.184719055891037, + "drift/rejected_abs": 0.08408048748970032, + "epoch": 0.22407628128724671, + "grad_norm": 3.109375, + "learning_rate": 1.5341006691669512e-08, + "logits/chosen": -2.2611207962036133, + "logits/rejected": -2.366459369659424, + "logps/chosen": -0.2667378783226013, + "logps/rejected": -0.2706339359283447, + "loss": 0.07560595124959946, + "loss/core": 0.06591415405273438, + "loss/preference_sft": 0.27826228737831116, + "loss/reference_anchor": 0.10139782726764679, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8469181060791016, + "rewards/margins": 1.010367512702942, + "rewards/rejected": 0.8365505933761597, + "ronpo/logit": 0.10103676468133926, + "ronpo/residual": 0.10294114053249359, + "ronpo/residual_abs": 0.12307702004909515, + "ronpo/target": -0.0019043807405978441, + "ronpo/target_abs": 0.041786063462495804, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 705 + }, + { + "drift/chosen_abs": 0.2891550660133362, + "drift/rejected_abs": 0.10379669815301895, + "epoch": 0.22566547477155344, + "grad_norm": 101.0, + "learning_rate": 1.4608381516789548e-08, + "logits/chosen": -2.1673738956451416, + "logits/rejected": -2.180238962173462, + "logps/chosen": -0.29929423332214355, + "logps/rejected": -0.2966969609260559, + "loss": 0.7068883180618286, + "loss/core": 0.6746903657913208, + "loss/preference_sft": 0.29829683899879456, + "loss/reference_anchor": 0.3994755446910858, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.891120433807373, + "rewards/margins": 1.9227606058120728, + "rewards/rejected": 0.9683598279953003, + "ronpo/logit": 0.19227604568004608, + "ronpo/residual": 0.1905229389667511, + "ronpo/residual_abs": 0.2223322093486786, + "ronpo/target": 0.0017531251069158316, + "ronpo/target_abs": 0.03450788930058479, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 710 + }, + { + "drift/chosen_abs": 0.1664462685585022, + "drift/rejected_abs": 0.09171423316001892, + "epoch": 0.22725466825586016, + "grad_norm": 1.1796875, + "learning_rate": 1.3890677483399537e-08, + "logits/chosen": -2.047231912612915, + "logits/rejected": -1.9673421382904053, + "logps/chosen": -0.30263659358024597, + "logps/rejected": -0.3006385564804077, + "loss": 0.06795786321163177, + "loss/core": 0.05951591208577156, + "loss/preference_sft": 0.3053247034549713, + "loss/reference_anchor": 0.08202681690454483, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.6637260913848877, + "rewards/margins": 0.747805118560791, + "rewards/rejected": 0.9159210324287415, + "ronpo/logit": 0.07478051632642746, + "ronpo/residual": 0.06953813135623932, + "ronpo/residual_abs": 0.11027568578720093, + "ronpo/target": 0.0052423798479139805, + "ronpo/target_abs": 0.0369865819811821, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 715 + }, + { + "drift/chosen_abs": 0.3428126871585846, + "drift/rejected_abs": 0.14502094686031342, + "epoch": 0.22884386174016685, + "grad_norm": 524.0, + "learning_rate": 1.3188197176232486e-08, + "logits/chosen": -1.9479389190673828, + "logits/rejected": -2.027177095413208, + "logps/chosen": -0.2941284775733948, + "logps/rejected": -0.285388320684433, + "loss": 0.583694577217102, + "loss/core": 0.5399834513664246, + "loss/preference_sft": 0.2799127697944641, + "loss/reference_anchor": 0.5548235774040222, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.4279160499572754, + "rewards/margins": 2.029214859008789, + "rewards/rejected": 1.3987010717391968, + "ronpo/logit": 0.20292147994041443, + "ronpo/residual": 0.19507944583892822, + "ronpo/residual_abs": 0.21936511993408203, + "ronpo/target": 0.007842039689421654, + "ronpo/target_abs": 0.037268415093421936, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 720 + }, + { + "drift/chosen_abs": 0.2962958812713623, + "drift/rejected_abs": 0.06851255893707275, + "epoch": 0.23043305522447358, + "grad_norm": 3.765625, + "learning_rate": 1.250123676168332e-08, + "logits/chosen": -2.0833048820495605, + "logits/rejected": -2.0245532989501953, + "logps/chosen": -0.28989580273628235, + "logps/rejected": -0.2917415499687195, + "loss": 1.0986053943634033, + "loss/core": 1.0533392429351807, + "loss/preference_sft": 0.2916197180747986, + "loss/reference_anchor": 0.5743886828422546, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.961975574493408, + "rewards/margins": 2.2812411785125732, + "rewards/rejected": 0.6807340383529663, + "ronpo/logit": 0.22812414169311523, + "ronpo/residual": 0.21930313110351562, + "ronpo/residual_abs": 0.2511158585548401, + "ronpo/target": 0.008821023628115654, + "ronpo/target_abs": 0.042353756725788116, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 725 + }, + { + "drift/chosen_abs": 0.31698840856552124, + "drift/rejected_abs": 0.09349516034126282, + "epoch": 0.2320222487087803, + "grad_norm": 2416.0, + "learning_rate": 1.183008586294485e-08, + "logits/chosen": -2.178093910217285, + "logits/rejected": -2.21374249458313, + "logps/chosen": -0.29720231890678406, + "logps/rejected": -0.2978862226009369, + "loss": 0.9873377084732056, + "loss/core": 0.9382851719856262, + "loss/preference_sft": 0.30639463663101196, + "loss/reference_anchor": 0.6233946084976196, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.169236183166504, + "rewards/margins": 2.236426830291748, + "rewards/rejected": 0.9328093528747559, + "ronpo/logit": 0.22364270687103271, + "ronpo/residual": 0.21640853583812714, + "ronpo/residual_abs": 0.24321117997169495, + "ronpo/target": 0.0072341784834861755, + "ronpo/target_abs": 0.039287909865379333, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 730 + }, + { + "drift/chosen_abs": 0.4120539724826813, + "drift/rejected_abs": 0.15305277705192566, + "epoch": 0.23361144219308702, + "grad_norm": 10.625, + "learning_rate": 1.1175027437902328e-08, + "logits/chosen": -1.8680089712142944, + "logits/rejected": -1.9213451147079468, + "logps/chosen": -0.31041333079338074, + "logps/rejected": -0.282566100358963, + "loss": 1.3381201028823853, + "loss/core": 1.2714483737945557, + "loss/preference_sft": 0.3085525631904602, + "loss/reference_anchor": 0.8580993413925171, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 4.118987083435059, + "rewards/margins": 2.5983529090881348, + "rewards/rejected": 1.5206340551376343, + "ronpo/logit": 0.25983530282974243, + "ronpo/residual": 0.2567339241504669, + "ronpo/residual_abs": 0.3305421769618988, + "ronpo/target": 0.0031014089472591877, + "ronpo/target_abs": 0.046786196529865265, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 735 + }, + { + "drift/chosen_abs": 0.4067060947418213, + "drift/rejected_abs": 0.1747940182685852, + "epoch": 0.23520063567739372, + "grad_norm": 6.90625, + "learning_rate": 1.0536337659838091e-08, + "logits/chosen": -2.0639398097991943, + "logits/rejected": -2.054687976837158, + "logps/chosen": -0.2756916880607605, + "logps/rejected": -0.27250200510025024, + "loss": 0.8732901811599731, + "loss/core": 0.7945246696472168, + "loss/preference_sft": 0.2825769782066345, + "loss/reference_anchor": 1.0219491720199585, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 4.066915988922119, + "rewards/margins": 2.3204641342163086, + "rewards/rejected": 1.746451735496521, + "ronpo/logit": 0.23204641044139862, + "ronpo/residual": 0.2248552292585373, + "ronpo/residual_abs": 0.265652596950531, + "ronpo/target": 0.00719115138053894, + "ronpo/target_abs": 0.04038388282060623, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 740 + }, + { + "drift/chosen_abs": 0.279170423746109, + "drift/rejected_abs": 0.14095792174339294, + "epoch": 0.23678982916170044, + "grad_norm": 1.8046875, + "learning_rate": 9.914285800996669e-09, + "logits/chosen": -1.965667486190796, + "logits/rejected": -1.9407310485839844, + "logps/chosen": -0.3105481266975403, + "logps/rejected": -0.29601940512657166, + "loss": 0.42376190423965454, + "loss/core": 0.39586013555526733, + "loss/preference_sft": 0.3031192421913147, + "loss/reference_anchor": 0.34171098470687866, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.7881577014923096, + "rewards/margins": 1.4106953144073486, + "rewards/rejected": 1.3774619102478027, + "ronpo/logit": 0.14106956124305725, + "ronpo/residual": 0.14506323635578156, + "ronpo/residual_abs": 0.18387925624847412, + "ronpo/target": -0.003993672784417868, + "ronpo/target_abs": 0.03346020728349686, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 745 + }, + { + "drift/chosen_abs": 0.4928775727748871, + "drift/rejected_abs": 0.08850506693124771, + "epoch": 0.23837902264600716, + "grad_norm": 27.375, + "learning_rate": 9.309134119059337e-09, + "logits/chosen": -1.8723657131195068, + "logits/rejected": -2.0245261192321777, + "logps/chosen": -0.3265787661075592, + "logps/rejected": -0.3039904534816742, + "loss": 1.7299203872680664, + "loss/core": 1.652858018875122, + "loss/preference_sft": 0.308999240398407, + "loss/reference_anchor": 0.9966007471084595, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.92842960357666, + "rewards/margins": 4.047325134277344, + "rewards/rejected": 0.8811044692993164, + "ronpo/logit": 0.40473252534866333, + "ronpo/residual": 0.40347546339035034, + "ronpo/residual_abs": 0.4258064329624176, + "ronpo/target": 0.0012570355320349336, + "ronpo/target_abs": 0.04142013564705849, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 750 + }, + { + "drift/chosen_abs": 0.5538117289543152, + "drift/rejected_abs": 0.2532739043235779, + "epoch": 0.23996821613031386, + "grad_norm": 169.0, + "learning_rate": 8.721137746576019e-09, + "logits/chosen": -1.7783946990966797, + "logits/rejected": -1.7093887329101562, + "logps/chosen": -0.2701883912086487, + "logps/rejected": -0.2783645987510681, + "loss": 1.8371918201446533, + "loss/core": 1.743137001991272, + "loss/preference_sft": 0.2584080994129181, + "loss/reference_anchor": 1.2282252311706543, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 5.537991046905518, + "rewards/margins": 3.011918544769287, + "rewards/rejected": 2.5260722637176514, + "ronpo/logit": 0.30119186639785767, + "ronpo/residual": 0.29457274079322815, + "ronpo/residual_abs": 0.46518081426620483, + "ronpo/target": 0.006619105581194162, + "ronpo/target_abs": 0.036873988807201385, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 755 + }, + { + "drift/chosen_abs": 0.2569332718849182, + "drift/rejected_abs": 0.10224013030529022, + "epoch": 0.24155740961462058, + "grad_norm": 1.2421875, + "learning_rate": 8.150544583401114e-09, + "logits/chosen": -2.202519655227661, + "logits/rejected": -2.2522640228271484, + "logps/chosen": -0.2607022821903229, + "logps/rejected": -0.26129353046417236, + "loss": 0.41069111227989197, + "loss/core": 0.38572731614112854, + "loss/preference_sft": 0.2723042368888855, + "loss/reference_anchor": 0.3056199550628662, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.5683581829071045, + "rewards/margins": 1.5499927997589111, + "rewards/rejected": 1.0183653831481934, + "ronpo/logit": 0.1549992859363556, + "ronpo/residual": 0.1515064835548401, + "ronpo/residual_abs": 0.17911091446876526, + "ronpo/target": 0.0034928210079669952, + "ronpo/target_abs": 0.0378664955496788, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 760 + }, + { + "drift/chosen_abs": 0.16791877150535583, + "drift/rejected_abs": 0.09147568047046661, + "epoch": 0.2431466030989273, + "grad_norm": 0.97265625, + "learning_rate": 7.597595192178703e-09, + "logits/chosen": -2.397317409515381, + "logits/rejected": -2.3352389335632324, + "logps/chosen": -0.28569984436035156, + "logps/rejected": -0.28979137539863586, + "loss": 0.044909216463565826, + "loss/core": 0.03475845605134964, + "loss/preference_sft": 0.28664320707321167, + "loss/reference_anchor": 0.1066790446639061, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.6786880493164062, + "rewards/margins": 0.7643090486526489, + "rewards/rejected": 0.9143790006637573, + "ronpo/logit": 0.07643090933561325, + "ronpo/residual": 0.08657925575971603, + "ronpo/residual_abs": 0.10033098608255386, + "ronpo/target": -0.010148352012038231, + "ronpo/target_abs": 0.03190162032842636, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 765 + }, + { + "drift/chosen_abs": 0.32959985733032227, + "drift/rejected_abs": 0.10112398862838745, + "epoch": 0.24473579658323402, + "grad_norm": 55.5, + "learning_rate": 7.062522696921108e-09, + "logits/chosen": -1.964592695236206, + "logits/rejected": -2.0949795246124268, + "logps/chosen": -0.2564390003681183, + "logps/rejected": -0.27129215002059937, + "loss": 0.5951908826828003, + "loss/core": 0.565562903881073, + "loss/preference_sft": 0.26717469096183777, + "loss/reference_anchor": 0.36832180619239807, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.2949090003967285, + "rewards/margins": 2.3579647541046143, + "rewards/rejected": 0.9369438290596008, + "ronpo/logit": 0.2357964962720871, + "ronpo/residual": 0.24124550819396973, + "ronpo/residual_abs": 0.25939813256263733, + "ronpo/target": -0.005449019372463226, + "ronpo/target_abs": 0.04333696514368057, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 770 + }, + { + "drift/chosen_abs": 0.3441210985183716, + "drift/rejected_abs": 0.1433735191822052, + "epoch": 0.24632499006754072, + "grad_norm": 1.078125, + "learning_rate": 6.545552684723582e-09, + "logits/chosen": -1.8267816305160522, + "logits/rejected": -1.8162485361099243, + "logps/chosen": -0.2950648367404938, + "logps/rejected": -0.2963963449001312, + "loss": 0.5573657155036926, + "loss/core": 0.5240105390548706, + "loss/preference_sft": 0.29786160588264465, + "loss/reference_anchor": 0.4149499833583832, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.4393134117126465, + "rewards/margins": 2.0451102256774902, + "rewards/rejected": 1.3942031860351562, + "ronpo/logit": 0.20451104640960693, + "ronpo/residual": 0.2089034765958786, + "ronpo/residual_abs": 0.2280438393354416, + "ronpo/target": -0.004392450209707022, + "ronpo/target_abs": 0.04173804819583893, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 775 + }, + { + "drift/chosen_abs": 0.349498987197876, + "drift/rejected_abs": 0.1552140712738037, + "epoch": 0.24791418355184744, + "grad_norm": 3.828125, + "learning_rate": 6.046903110656504e-09, + "logits/chosen": -2.1567647457122803, + "logits/rejected": -2.180453062057495, + "logps/chosen": -0.2706982493400574, + "logps/rejected": -0.2793269753456116, + "loss": 0.35834938287734985, + "loss/core": 0.32353121042251587, + "loss/preference_sft": 0.27363473176956177, + "loss/reference_anchor": 0.436878502368927, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.4948108196258545, + "rewards/margins": 1.943900465965271, + "rewards/rejected": 1.5509105920791626, + "ronpo/logit": 0.19439007341861725, + "ronpo/residual": 0.19187742471694946, + "ronpo/residual_abs": 0.2496909350156784, + "ronpo/target": 0.0025126230902969837, + "ronpo/target_abs": 0.04215142875909805, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 780 + }, + { + "drift/chosen_abs": 0.47929421067237854, + "drift/rejected_abs": 0.1080431342124939, + "epoch": 0.24950337703615416, + "grad_norm": 3.71875, + "learning_rate": 5.566784205875363e-09, + "logits/chosen": -2.1040244102478027, + "logits/rejected": -2.2148659229278564, + "logps/chosen": -0.26970845460891724, + "logps/rejected": -0.2620154917240143, + "loss": 1.8097639083862305, + "loss/core": 1.7370777130126953, + "loss/preference_sft": 0.26922696828842163, + "loss/reference_anchor": 0.9422265887260437, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.785176753997803, + "rewards/margins": 3.7156035900115967, + "rewards/rejected": 1.0695734024047852, + "ronpo/logit": 0.37156039476394653, + "ronpo/residual": 0.3729218542575836, + "ronpo/residual_abs": 0.39052265882492065, + "ronpo/target": -0.0013614622876048088, + "ronpo/target_abs": 0.04210304841399193, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 785 + }, + { + "drift/chosen_abs": 0.35931769013404846, + "drift/rejected_abs": 0.12135148048400879, + "epoch": 0.25109257052046086, + "grad_norm": 199.0, + "learning_rate": 5.105398388987098e-09, + "logits/chosen": -2.007981538772583, + "logits/rejected": -2.027435302734375, + "logps/chosen": -0.31240516901016235, + "logps/rejected": -0.28974097967147827, + "loss": 0.5043392181396484, + "loss/core": 0.4675632119178772, + "loss/preference_sft": 0.30789914727211, + "loss/reference_anchor": 0.45955753326416016, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.5931477546691895, + "rewards/margins": 2.408783435821533, + "rewards/rejected": 1.1843643188476562, + "ronpo/logit": 0.2408783733844757, + "ronpo/residual": 0.24351993203163147, + "ronpo/residual_abs": 0.27939122915267944, + "ronpo/target": -0.002641576575115323, + "ronpo/target_abs": 0.039345476776361465, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 790 + }, + { + "drift/chosen_abs": 0.24088767170906067, + "drift/rejected_abs": 0.1129360944032669, + "epoch": 0.2526817640047676, + "grad_norm": 1.171875, + "learning_rate": 4.662940180710167e-09, + "logits/chosen": -2.1340980529785156, + "logits/rejected": -2.16754150390625, + "logps/chosen": -0.2590358257293701, + "logps/rejected": -0.2528657913208008, + "loss": 0.1206575408577919, + "loss/core": 0.10890237987041473, + "loss/preference_sft": 0.2507992386817932, + "loss/reference_anchor": 0.13165560364723206, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.4075381755828857, + "rewards/margins": 1.279028296470642, + "rewards/rejected": 1.128509759902954, + "ronpo/logit": 0.1279028356075287, + "ronpo/residual": 0.13108345866203308, + "ronpo/residual_abs": 0.1609242856502533, + "ronpo/target": -0.0031806365586817265, + "ronpo/target_abs": 0.03893408179283142, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 795 + }, + { + "drift/chosen_abs": 0.28383442759513855, + "drift/rejected_abs": 0.1589200645685196, + "epoch": 0.2542709574890743, + "grad_norm": 20.125, + "learning_rate": 4.23959612186448e-09, + "logits/chosen": -2.1699929237365723, + "logits/rejected": -2.2424261569976807, + "logps/chosen": -0.26959216594696045, + "logps/rejected": -0.27034133672714233, + "loss": 0.2392566204071045, + "loss/core": 0.17778518795967102, + "loss/preference_sft": 0.25929346680641174, + "loss/reference_anchor": 0.7936893701553345, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.8376173973083496, + "rewards/margins": 1.2496715784072876, + "rewards/rejected": 1.5879456996917725, + "ronpo/logit": 0.12496715784072876, + "ronpo/residual": 0.1267654150724411, + "ronpo/residual_abs": 0.14537087082862854, + "ronpo/target": -0.0017982644494622946, + "ronpo/target_abs": 0.03246055170893669, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 800 + }, + { + "drift/chosen_abs": 0.31993216276168823, + "drift/rejected_abs": 0.13845376670360565, + "epoch": 0.255860150973381, + "grad_norm": 195.0, + "learning_rate": 3.835544694725529e-09, + "logits/chosen": -2.2753825187683105, + "logits/rejected": -2.2379002571105957, + "logps/chosen": -0.3026265501976013, + "logps/rejected": -0.30721110105514526, + "loss": 0.9451104998588562, + "loss/core": 0.8910640478134155, + "loss/preference_sft": 0.29863858222961426, + "loss/reference_anchor": 0.6907570958137512, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.199321746826172, + "rewards/margins": 1.8681728839874268, + "rewards/rejected": 1.3311488628387451, + "ronpo/logit": 0.1868172585964203, + "ronpo/residual": 0.18839898705482483, + "ronpo/residual_abs": 0.2753552794456482, + "ronpo/target": -0.0015817094827070832, + "ronpo/target_abs": 0.036414168775081635, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 805 + }, + { + "drift/chosen_abs": 0.2113758623600006, + "drift/rejected_abs": 0.11016102135181427, + "epoch": 0.25744934445768775, + "grad_norm": 7.9375, + "learning_rate": 3.4509562477761658e-09, + "logits/chosen": -2.0627362728118896, + "logits/rejected": -2.113102912902832, + "logps/chosen": -0.283230185508728, + "logps/rejected": -0.287577748298645, + "loss": 0.04678564518690109, + "loss/core": 0.03707611560821533, + "loss/preference_sft": 0.2837401032447815, + "loss/reference_anchor": 0.10108641535043716, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.1131529808044434, + "rewards/margins": 1.0126079320907593, + "rewards/rejected": 1.1005451679229736, + "ronpo/logit": 0.10126079618930817, + "ronpo/residual": 0.08919169753789902, + "ronpo/residual_abs": 0.11383958160877228, + "ronpo/target": 0.012069085612893105, + "ronpo/target_abs": 0.03552011400461197, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 810 + }, + { + "drift/chosen_abs": 0.2591978907585144, + "drift/rejected_abs": 0.1618344634771347, + "epoch": 0.25903853794199444, + "grad_norm": 12.4375, + "learning_rate": 3.0859929238874314e-09, + "logits/chosen": -2.1082515716552734, + "logits/rejected": -2.030113697052002, + "logps/chosen": -0.26747792959213257, + "logps/rejected": -0.27874407172203064, + "loss": 0.21905355155467987, + "loss/core": 0.2024170458316803, + "loss/preference_sft": 0.26561182737350464, + "loss/reference_anchor": 0.19525930285453796, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.5913424491882324, + "rewards/margins": 0.9835828542709351, + "rewards/rejected": 1.6077591180801392, + "ronpo/logit": 0.09835829585790634, + "ronpo/residual": 0.08990249782800674, + "ronpo/residual_abs": 0.19450879096984863, + "ronpo/target": 0.008455811068415642, + "ronpo/target_abs": 0.04190158098936081, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 815 + }, + { + "drift/chosen_abs": 0.19986757636070251, + "drift/rejected_abs": 0.11452219635248184, + "epoch": 0.26062773142630113, + "grad_norm": 2.359375, + "learning_rate": 2.7408085919590262e-09, + "logits/chosen": -2.022050380706787, + "logits/rejected": -1.9783365726470947, + "logps/chosen": -0.2758370041847229, + "logps/rejected": -0.28798961639404297, + "loss": 0.06202256679534912, + "loss/core": 0.047978322952985764, + "loss/preference_sft": 0.2731260657310486, + "loss/reference_anchor": 0.1599440574645996, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.9943485260009766, + "rewards/margins": 0.8497527241706848, + "rewards/rejected": 1.1445958614349365, + "ronpo/logit": 0.08497526496648788, + "ronpo/residual": 0.08352629095315933, + "ronpo/residual_abs": 0.11770425736904144, + "ronpo/target": 0.0014489861205220222, + "ronpo/target_abs": 0.03912581130862236, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 820 + }, + { + "drift/chosen_abs": 0.287466824054718, + "drift/rejected_abs": 0.18906255066394806, + "epoch": 0.2622169249106079, + "grad_norm": 80.5, + "learning_rate": 2.415548782047999e-09, + "logits/chosen": -2.213947057723999, + "logits/rejected": -2.160534620285034, + "logps/chosen": -0.299244225025177, + "logps/rejected": -0.3014959692955017, + "loss": 0.3660229444503784, + "loss/core": 0.301073282957077, + "loss/preference_sft": 0.29996412992477417, + "loss/reference_anchor": 0.8359990119934082, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.8729865550994873, + "rewards/margins": 0.9903990030288696, + "rewards/rejected": 1.8825876712799072, + "ronpo/logit": 0.09903990477323532, + "ronpo/residual": 0.09887038916349411, + "ronpo/residual_abs": 0.1854637861251831, + "ronpo/target": 0.00016951440193224698, + "ronpo/target_abs": 0.035706378519535065, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 825 + }, + { + "drift/chosen_abs": 0.18484026193618774, + "drift/rejected_abs": 0.11339914798736572, + "epoch": 0.2638061183949146, + "grad_norm": 31.875, + "learning_rate": 2.110350624013124e-09, + "logits/chosen": -2.233262538909912, + "logits/rejected": -2.234431266784668, + "logps/chosen": -0.277521014213562, + "logps/rejected": -0.2729399502277374, + "loss": 0.12444338947534561, + "loss/core": 0.11208049207925797, + "loss/preference_sft": 0.2777172923088074, + "loss/reference_anchor": 0.1370667666196823, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8484026193618774, + "rewards/margins": 0.7153452038764954, + "rewards/rejected": 1.1330575942993164, + "ronpo/logit": 0.07153451442718506, + "ronpo/residual": 0.06523828208446503, + "ronpo/residual_abs": 0.13810399174690247, + "ronpo/target": 0.006296238861978054, + "ronpo/target_abs": 0.03897542133927345, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 830 + }, + { + "drift/chosen_abs": 0.40631216764450073, + "drift/rejected_abs": 0.1610126495361328, + "epoch": 0.2653953118792213, + "grad_norm": 34.5, + "learning_rate": 1.8253427897008456e-09, + "logits/chosen": -2.191138982772827, + "logits/rejected": -2.1562600135803223, + "logps/chosen": -0.24265797436237335, + "logps/rejected": -0.24613361060619354, + "loss": 1.0934888124465942, + "loss/core": 1.040398359298706, + "loss/preference_sft": 0.24096707999706268, + "loss/reference_anchor": 0.6837762594223022, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 4.062660217285156, + "rewards/margins": 2.45281982421875, + "rewards/rejected": 1.6098406314849854, + "ronpo/logit": 0.24528197944164276, + "ronpo/residual": 0.23903052508831024, + "ronpo/residual_abs": 0.3242782950401306, + "ronpo/target": 0.006251484155654907, + "ronpo/target_abs": 0.036385711282491684, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 835 + }, + { + "drift/chosen_abs": 0.28632503747940063, + "drift/rejected_abs": 0.13661116361618042, + "epoch": 0.266984505363528, + "grad_norm": 0.92578125, + "learning_rate": 1.5606454386970668e-09, + "logits/chosen": -2.362611770629883, + "logits/rejected": -2.4255595207214355, + "logps/chosen": -0.2671854496002197, + "logps/rejected": -0.2649426758289337, + "loss": 0.36431366205215454, + "loss/core": 0.3328004479408264, + "loss/preference_sft": 0.2735097408294678, + "loss/reference_anchor": 0.3928253948688507, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.863193988800049, + "rewards/margins": 1.4984618425369263, + "rewards/rejected": 1.364732027053833, + "ronpo/logit": 0.1498461812734604, + "ronpo/residual": 0.1546889841556549, + "ronpo/residual_abs": 0.17629854381084442, + "ronpo/target": -0.0048428005538880825, + "ronpo/target_abs": 0.04114602878689766, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 840 + }, + { + "drift/chosen_abs": 0.29827648401260376, + "drift/rejected_abs": 0.18521063029766083, + "epoch": 0.2685736988478347, + "grad_norm": 4.96875, + "learning_rate": 1.316370167667763e-09, + "logits/chosen": -2.1433889865875244, + "logits/rejected": -2.135906457901001, + "logps/chosen": -0.24772045016288757, + "logps/rejected": -0.23420384526252747, + "loss": 0.09277145564556122, + "loss/core": 0.06800076365470886, + "loss/preference_sft": 0.24155735969543457, + "loss/reference_anchor": 0.3061201870441437, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.98088002204895, + "rewards/margins": 1.129408597946167, + "rewards/rejected": 1.8514715433120728, + "ronpo/logit": 0.11294084787368774, + "ronpo/residual": 0.10954286903142929, + "ronpo/residual_abs": 0.1346508264541626, + "ronpo/target": 0.0033979781437665224, + "ronpo/target_abs": 0.04030502587556839, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 845 + }, + { + "drift/chosen_abs": 0.41069334745407104, + "drift/rejected_abs": 0.14589200913906097, + "epoch": 0.2701628923321414, + "grad_norm": 80.0, + "learning_rate": 1.0926199633097156e-09, + "logits/chosen": -1.9553956985473633, + "logits/rejected": -1.8889896869659424, + "logps/chosen": -0.27275902032852173, + "logps/rejected": -0.28432151675224304, + "loss": 0.9306114315986633, + "loss/core": 0.8736518621444702, + "loss/preference_sft": 0.2677767872810364, + "loss/reference_anchor": 0.7326820492744446, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 4.104775428771973, + "rewards/margins": 2.6476259231567383, + "rewards/rejected": 1.4571493864059448, + "ronpo/logit": 0.2647625803947449, + "ronpo/residual": 0.2660098969936371, + "ronpo/residual_abs": 0.2944030165672302, + "ronpo/target": -0.0012473245151340961, + "ronpo/target_abs": 0.03741658851504326, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 850 + }, + { + "drift/chosen_abs": 0.2068370282649994, + "drift/rejected_abs": 0.07252339273691177, + "epoch": 0.27175208581644816, + "grad_norm": 2.078125, + "learning_rate": 8.894891589312448e-10, + "logits/chosen": -2.378898859024048, + "logits/rejected": -2.3654732704162598, + "logps/chosen": -0.2773938775062561, + "logps/rejected": -0.2757563591003418, + "loss": 0.41834402084350586, + "loss/core": 0.39636608958244324, + "loss/preference_sft": 0.27365297079086304, + "loss/reference_anchor": 0.26567381620407104, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.0683701038360596, + "rewards/margins": 1.3431364297866821, + "rewards/rejected": 0.7252338528633118, + "ronpo/logit": 0.13431362807750702, + "ronpo/residual": 0.13792987167835236, + "ronpo/residual_abs": 0.1564667969942093, + "ronpo/target": -0.0036162561737000942, + "ronpo/target_abs": 0.03731636330485344, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 855 + }, + { + "drift/chosen_abs": 0.18851238489151, + "drift/rejected_abs": 0.10406199842691422, + "epoch": 0.27334127930075486, + "grad_norm": 32.5, + "learning_rate": 7.070633946811345e-10, + "logits/chosen": -2.203418254852295, + "logits/rejected": -2.129629373550415, + "logps/chosen": -0.2803042531013489, + "logps/rejected": -0.2796124815940857, + "loss": 0.12177155166864395, + "loss/core": 0.10930196195840836, + "loss/preference_sft": 0.2833760976791382, + "loss/reference_anchor": 0.13792333006858826, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.8851238489151, + "rewards/margins": 0.8534868359565735, + "rewards/rejected": 1.0316369533538818, + "ronpo/logit": 0.08534868061542511, + "ronpo/residual": 0.08152299374341965, + "ronpo/residual_abs": 0.1145586222410202, + "ronpo/target": 0.0038256910629570484, + "ronpo/target_abs": 0.03399581462144852, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 860 + }, + { + "drift/chosen_abs": 0.24835388362407684, + "drift/rejected_abs": 0.12930510938167572, + "epoch": 0.2749304727850616, + "grad_norm": 0.5703125, + "learning_rate": 5.454195814427021e-10, + "logits/chosen": -2.090212821960449, + "logits/rejected": -2.035008668899536, + "logps/chosen": -0.2799471318721771, + "logps/rejected": -0.2892903685569763, + "loss": 0.11395768821239471, + "loss/core": 0.09873252362012863, + "loss/preference_sft": 0.2899934649467468, + "loss/reference_anchor": 0.1740027666091919, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.4830942153930664, + "rewards/margins": 1.1911389827728271, + "rewards/rejected": 1.2919554710388184, + "ronpo/logit": 0.11911387741565704, + "ronpo/residual": 0.11887941509485245, + "ronpo/residual_abs": 0.15069282054901123, + "ronpo/target": 0.00023447340936399996, + "ronpo/target_abs": 0.03890164569020271, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 865 + }, + { + "drift/chosen_abs": 0.4616221487522125, + "drift/rejected_abs": 0.2202831208705902, + "epoch": 0.2765196662693683, + "grad_norm": 520.0, + "learning_rate": 4.0462586840810387e-10, + "logits/chosen": -2.099531888961792, + "logits/rejected": -2.0872442722320557, + "logps/chosen": -0.27098220586776733, + "logps/rejected": -0.2758156359195709, + "loss": 0.7392908930778503, + "loss/core": 0.6623992919921875, + "loss/preference_sft": 0.2761059105396271, + "loss/reference_anchor": 0.9976102709770203, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.614747047424316, + "rewards/margins": 2.412616014480591, + "rewards/rejected": 2.2021312713623047, + "ronpo/logit": 0.2412615716457367, + "ronpo/residual": 0.2500116527080536, + "ronpo/residual_abs": 0.27789297699928284, + "ronpo/target": -0.008750082924962044, + "ronpo/target_abs": 0.0406760573387146, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 870 + }, + { + "drift/chosen_abs": 0.30633243918418884, + "drift/rejected_abs": 0.15432590246200562, + "epoch": 0.278108859753675, + "grad_norm": 5.1875, + "learning_rate": 2.847416143465342e-10, + "logits/chosen": -1.943389892578125, + "logits/rejected": -1.9119516611099243, + "logps/chosen": -0.2705642282962799, + "logps/rejected": -0.27163758873939514, + "loss": 0.2206476628780365, + "loss/core": 0.19633831083774567, + "loss/preference_sft": 0.27449434995651245, + "loss/reference_anchor": 0.2966754138469696, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.0624802112579346, + "rewards/margins": 1.5204280614852905, + "rewards/rejected": 1.542051911354065, + "ronpo/logit": 0.15204280614852905, + "ronpo/residual": 0.1534896343946457, + "ronpo/residual_abs": 0.17870879173278809, + "ronpo/target": -0.001446827664040029, + "ronpo/target_abs": 0.04036080092191696, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 875 + }, + { + "drift/chosen_abs": 0.244231179356575, + "drift/rejected_abs": 0.09062740206718445, + "epoch": 0.27969805323798175, + "grad_norm": 5.5, + "learning_rate": 1.8581736257852754e-10, + "logits/chosen": -2.1149797439575195, + "logits/rejected": -2.1567158699035645, + "logps/chosen": -0.27833813428878784, + "logps/rejected": -0.28993064165115356, + "loss": 0.40490633249282837, + "loss/core": 0.38430437445640564, + "loss/preference_sft": 0.29224926233291626, + "loss/reference_anchor": 0.24546775221824646, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.4423117637634277, + "rewards/margins": 1.538805365562439, + "rewards/rejected": 0.9035066366195679, + "ronpo/logit": 0.1538805216550827, + "ronpo/residual": 0.14947783946990967, + "ronpo/residual_abs": 0.1738136261701584, + "ronpo/target": 0.004402679391205311, + "ronpo/target_abs": 0.04329409450292587, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 880 + }, + { + "drift/chosen_abs": 0.2656122148036957, + "drift/rejected_abs": 0.10401421785354614, + "epoch": 0.28128724672228844, + "grad_norm": 7.0625, + "learning_rate": 1.0789481966683722e-10, + "logits/chosen": -2.159553050994873, + "logits/rejected": -2.2222096920013428, + "logps/chosen": -0.29716652631759644, + "logps/rejected": -0.2955102324485779, + "loss": 0.41419491171836853, + "loss/core": 0.3724747598171234, + "loss/preference_sft": 0.29870742559432983, + "loss/reference_anchor": 0.5263984203338623, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 2.6553502082824707, + "rewards/margins": 1.617895483970642, + "rewards/rejected": 1.037454605102539, + "ronpo/logit": 0.16178953647613525, + "ronpo/residual": 0.16292984783649445, + "ronpo/residual_abs": 0.17799483239650726, + "ronpo/target": -0.0011402855161577463, + "ronpo/target_abs": 0.03544146940112114, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 885 + }, + { + "drift/chosen_abs": 0.2998327910900116, + "drift/rejected_abs": 0.12200510501861572, + "epoch": 0.28287644020659514, + "grad_norm": 3.515625, + "learning_rate": 5.1006837832928694e-11, + "logits/chosen": -2.3049538135528564, + "logits/rejected": -2.263355255126953, + "logps/chosen": -0.28130239248275757, + "logps/rejected": -0.28048938512802124, + "loss": 0.4184916913509369, + "loss/core": 0.37995871901512146, + "loss/preference_sft": 0.28656134009361267, + "loss/reference_anchor": 0.48511725664138794, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.9957728385925293, + "rewards/margins": 1.780024766921997, + "rewards/rejected": 1.2157481908798218, + "ronpo/logit": 0.1780024766921997, + "ronpo/residual": 0.1812599003314972, + "ronpo/residual_abs": 0.19650830328464508, + "ronpo/target": -0.003257411066442728, + "ronpo/target_abs": 0.035617318004369736, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 890 + }, + { + "drift/chosen_abs": 0.3059307932853699, + "drift/rejected_abs": 0.1232098937034607, + "epoch": 0.2844656336909019, + "grad_norm": 13.5, + "learning_rate": 1.517740110641985e-11, + "logits/chosen": -2.080798625946045, + "logits/rejected": -2.070003032684326, + "logps/chosen": -0.28609567880630493, + "logps/rejected": -0.2946876883506775, + "loss": 0.3638497292995453, + "loss/core": 0.3331601321697235, + "loss/preference_sft": 0.3008359372615814, + "loss/reference_anchor": 0.37911102175712585, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.0580692291259766, + "rewards/margins": 1.9168615341186523, + "rewards/rejected": 1.1412078142166138, + "ronpo/logit": 0.19168615341186523, + "ronpo/residual": 0.18204104900360107, + "ronpo/residual_abs": 0.21228711307048798, + "ronpo/target": 0.009645095095038414, + "ronpo/target_abs": 0.044705357402563095, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 895 + }, + { + "drift/chosen_abs": 0.15993809700012207, + "drift/rejected_abs": 0.0836057960987091, + "epoch": 0.2860548271752086, + "grad_norm": 15.4375, + "learning_rate": 4.2161521340289277e-13, + "logits/chosen": -2.1035187244415283, + "logits/rejected": -2.086210012435913, + "logps/chosen": -0.31812211871147156, + "logps/rejected": -0.3182290196418762, + "loss": 0.13807110488414764, + "loss/core": 0.12665197253227234, + "loss/preference_sft": 0.32046961784362793, + "loss/reference_anchor": 0.1202082484960556, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.598246455192566, + "rewards/margins": 0.88621586561203, + "rewards/rejected": 0.7120307087898254, + "ronpo/logit": 0.0886215791106224, + "ronpo/residual": 0.08629345148801804, + "ronpo/residual_abs": 0.11131948232650757, + "ronpo/target": 0.0023281420581042767, + "ronpo/target_abs": 0.036599740386009216, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 900 + }, + { + "epoch": 0.2860548271752086, + "step": 900, + "total_flos": 0.0, + "train_loss": 0.430257171690464, + "train_runtime": 7318.8237, + "train_samples_per_second": 1.968, + "train_steps_per_second": 0.123 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..392f108 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b3e0757315d2f087a3e8db33e99e8bdcc9d0bbeb039168b1b5d1192fa6509d9 +size 6993 diff --git a/training_status.json b/training_status.json new file mode 100644 index 0000000..244a146 --- /dev/null +++ b/training_status.json @@ -0,0 +1,19 @@ +{ + "status": "completed", + "candidate_id": "ronpo_full_a", + "method": "ronpo_full_expect", + "seed": 42, + "gpu": 1, + "config": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/fair_demo_20260715/sweep/candidates/ronpo_full_a/config.yaml", + "dataset": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/ronpo_full_expect", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "234ea82cbcb3", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/234ea82cbcb3", + "launcher": "supplemental_one_candidate", + "started_at": "2026-07-15T18:02:38+09:00", + "returncode": 0, + "model_complete": true, + "finite_results": true, + "completed_at": "2026-07-15T20:05:18+09:00" +}