commit 2a734bdd2fedbd4b2be1a327d18ac2c2b1c7af01 Author: ModelHub XC Date: Sat Jul 18 12:42:10 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: promotion/ronpo-qwen3-8b-fair-ronpo-k-only-s42 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..1d85c38 --- /dev/null +++ b/README.md @@ -0,0 +1,12 @@ +--- +base_model: Qwen/Qwen3-8B +library_name: transformers +tags: +- preference-optimization +- ronpo +- qwen3 +--- + +# Qwen3-8B fair-demo checkpoint: ronpo_k_only + +Validation-selected candidate: `ronpo_top_a`. Exact evaluator, selection, sweep, and training metadata are stored under `experiment/`. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..c6cc656 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.2860548271752086, + "total_flos": 0.0, + "train_loss": 0.4462373371918996, + "train_runtime": 7238.9345, + "train_samples": 50340, + "train_samples_per_second": 1.989, + "train_steps_per_second": 0.124 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..64965f9 --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +model_name_or_path: /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218 +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/fair_demo_20260715/precomputed/ronpo_k_only_k1: 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: ronpo +eta: 0.0075 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.15 +ronpo_tau: 0.1 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.1 +preference_sft_weight: 0.01 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 5.0e-08 +lr_scheduler_type: cosine +warmup_ratio: 0.2 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 42 +gradient_accumulation_steps: 16 +gradient_checkpointing: true +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 1 +per_device_eval_batch_size: 1 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/fair_demo_20260715/sweep/candidates/ronpo_top_a +run_name: qwen3-8b-fair-demo-ronpo_top_a diff --git a/experiment/evaluator_lock.json b/experiment/evaluator_lock.json new file mode 100644 index 0000000..b406c9a --- /dev/null +++ b/experiment/evaluator_lock.json @@ -0,0 +1,85 @@ +{ + "status": "LOCKED_BEFORE_ANY_NEW_METHOD_RANKING", + "locked_at": "2026-07-15T20:20:29+09:00", + "objective_signals": [ + "skywork", + "armo_safety", + "length_conciseness" + ], + "objective_semantics": [ + "helpfulness", + "safety", + "conciseness" + ], + "primary": { + "name": "open_weight_panel_mean_prompt_worst_vs_base", + "definition": "For each prompt and objective, average win=1/tie=0.5/loss=0 over two A/B positions and two judges; take the minimum objective, then mean over prompts.", + "judges": [ + { + "model": "openai/gpt-oss-120b", + "revision": "b5c939de8f754692c1647ca79fbf85e8c1e70f8a" + }, + { + "model": "Qwen/Qwen3-32B", + "revision": "9216db5781bf21249d130ec9da846c4624c16137" + } + ], + "decode": { + "temperature": 0.0, + "top_p": 1.0, + "max_new_tokens": 512, + "seed": 42 + }, + "position_swap": true, + "validation_selection_rule": "Highest eligible mean prompt-level worst panel score within each method; an exact numeric tie is broken by candidate_id lexical order." + }, + "secondary": { + "normalization": "For each locked signal, divide each evaluation prompt's candidate-minus-base raw delta by the population SD of the diagnostic base and matched-control scores; take the prompt-level minimum across objectives, then the mean. No per-prompt min-max.", + "diagnostic_control_scale": { + "skywork": 22.807301785782975, + "armo_safety": 0.3098083353203591, + "length_conciseness": 1.5891969646897472 + }, + "reward_signal_provenance": { + "skywork": { + "model": "Skywork/Skywork-Reward-V2-Llama-3.1-8B", + "revision": "cba2f842f3f1af2f1b2f0d35e794d789976390c5", + "semantics": "helpfulness" + }, + "armo_safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "revision": "eb2676d20da2f2d41082289d23c59b9f7427f955", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "length_conciseness": { + "deterministic": "-log1p(response_word_count)", + "tokenization": "Python str.split whitespace words" + } + }, + "report_raw_paired_deltas": true + }, + "bootstrap": { + "paired_prompt_resamples": 2000, + "seed": 42, + "interval": "percentile_95" + }, + "power": { + "target_absolute_effect": 0.05, + "paired_sd": 0.13831629563357775, + "required_prompts_80pct_power": 61, + "planned_fresh_test_prompts": 1024, + "alpha_two_sided": 0.05, + "power": 0.8 + }, + "fresh_test_source": { + "dataset": "HuggingFaceH4/ultrachat_200k", + "revision": "8049631c405ae6576f93f445c6b8166f76f5505a", + "split": "test_sft" + }, + "diagnostic_summary_sha256": "e365ed814926ac5c17777e0fc65d0299a45b81bd390542889bbd55b5453d4ffe", + "judge_diagnostic_lock_sha256": "fce2d97806053cabdf3d358806f4baa80df7f89d71d8c1c298883327d4e02635", + "prereg_sha256": "71cbd1ac17c3e807f2ece1cb0a82374bbe74edb654c2845b97c3222ed680f0d0", + "method_ranking_computed": false, + "spent_sealed_split_touched": false +} diff --git a/experiment/selection_lock.json b/experiment/selection_lock.json new file mode 100644 index 0000000..bae8b9c --- /dev/null +++ b/experiment/selection_lock.json @@ -0,0 +1,99 @@ +{ + "status": "VALIDATION_SELECTION_LOCKED_BEFORE_FRESH_TEST", + "locked_at": "2026-07-15T20:37:11+09:00", + "selection_metric": "open_weight_panel_mean_prompt_worst_vs_base", + "tie_break": "candidate_id lexical order", + "selected_by_method": { + "dpo": { + "candidate_id": "dpo_b", + "validation_primary": 0.40625, + "validation_primary_ci95": [ + 0.3837890625, + 0.427734375 + ], + "eligible_candidates": [ + "dpo_a", + "dpo_b" + ] + }, + "ht_mnpo_helpfulness": { + "candidate_id": "ht_help_b", + "validation_primary": 0.4072265625, + "validation_primary_ci95": [ + 0.3828125, + 0.4296875 + ], + "eligible_candidates": [ + "ht_help_b" + ] + }, + "ht_mnpo_safety": { + "candidate_id": "ht_safety_a", + "validation_primary": 0.4150390625, + "validation_primary_ci95": [ + 0.3925537109375, + 0.4384765625 + ], + "eligible_candidates": [ + "ht_safety_a", + "ht_safety_b" + ] + }, + "inpo_avg": { + "candidate_id": "inpo_b", + "validation_primary": 0.40625, + "validation_primary_ci95": [ + 0.3818359375, + 0.427734375 + ], + "eligible_candidates": [ + "inpo_b" + ] + }, + "ronpo_full_expect": { + "candidate_id": "ronpo_full_a", + "validation_primary": 0.4033203125, + "validation_primary_ci95": [ + 0.37890625, + 0.42580566406249987 + ], + "eligible_candidates": [ + "ronpo_full_a", + "ronpo_full_b" + ] + }, + "ronpo_k_only": { + "candidate_id": "ronpo_top_a", + "validation_primary": 0.412109375, + "validation_primary_ci95": [ + 0.3876953125, + 0.43557128906249987 + ], + "eligible_candidates": [ + "ronpo_top_a" + ] + }, + "sppo_avg": { + "candidate_id": "sppo_b", + "validation_primary": 0.4111328125, + "validation_primary_ci95": [ + 0.3866943359375, + 0.4345703125 + ], + "eligible_candidates": [ + "sppo_b" + ] + } + }, + "selected_ronpo_overall": "ronpo_top_a", + "failed_methods": [ + "ht_mnpo_conciseness", + "ipo", + "simpo" + ], + "panel_summary_sha256": "8850f23e16cadf56a99e3d562d224ce5337ab36fa588d8a843382853e6ad17bd", + "evaluator_lock_sha256": "ab7b12c84932ed57acfec7edbcbc1ff1df5e9acd203869b1458343adf6e5b2c8", + "grid_sha256": "abfd982ef10c3bc71364752241150286ebf06be5e97f087596adf80e55d459e0", + "fresh_test_opened": false, + "spent_sealed_split_touched": false +} diff --git a/experiment/sweep_grid.json b/experiment/sweep_grid.json new file mode 100644 index 0000000..1ee759f --- /dev/null +++ b/experiment/sweep_grid.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "status": "frozen_before_training_and_validation_ranking", + "frozen_at_kst": "2026-07-15T09:01:45+09:00", + "seed": 42, + "common": { + "optimizer_steps": 900, + "effective_batch_size": 16, + "per_device_train_batch_size": 1, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "bf16": true, + "attn_implementation": "sdpa", + "cudnn_sdpa": false, + "lr_scheduler_type": "cosine", + "max_length": 2048, + "max_prompt_length": 1800, + "save_total_limit": 1, + "wandb_entity": "promotion-kim", + "wandb_project": "mnpo", + "wandb_group": "qwen3-8b-fair-demo-20260715" + }, + "budget_rule": "Exactly two configs per reported method; no extension after validation rankings are visible.", + "candidates": [ + {"id":"ronpo_full_a","method":"ronpo_full_expect","dataset":"ronpo_full_expect_kall","learning_rate":1e-7,"warmup_ratio":0.15,"ronpo_alpha":0.25,"ronpo_tau":0.10,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075,"expected_support_k":0}, + {"id":"ronpo_full_b","method":"ronpo_full_expect","dataset":"ronpo_full_expect_k6","learning_rate":2.5e-7,"warmup_ratio":0.20,"ronpo_alpha":0.35,"ronpo_tau":0.05,"reference_anchor_weight":0.10,"preference_sft_weight":0.01,"expected_support_k":6}, + {"id":"ronpo_top_a","method":"ronpo_k_only","dataset":"ronpo_k_only_k1","learning_rate":5e-8,"warmup_ratio":0.20,"ronpo_alpha":0.15,"ronpo_tau":0.10,"reference_anchor_weight":0.10,"preference_sft_weight":0.01,"expected_support_k":1}, + {"id":"ronpo_top_b","method":"ronpo_k_only","dataset":"ronpo_k_only_k2","learning_rate":1e-7,"warmup_ratio":0.15,"ronpo_alpha":0.25,"ronpo_tau":0.10,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075,"expected_support_k":2}, + {"id":"dpo_a","method":"dpo","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.10,"dpo_beta":0.05,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"dpo_b","method":"dpo","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.15,"dpo_beta":0.10,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ipo_a","method":"ipo","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.15,"dpo_beta":0.20,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ipo_b","method":"ipo","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.20,"dpo_beta":0.50,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075}, + {"id":"simpo_a","method":"simpo","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.10,"simpo_beta":1.0,"simpo_gamma":0.3,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"simpo_b","method":"simpo","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.15,"simpo_beta":2.0,"simpo_gamma":0.6,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"sppo_a","method":"sppo_avg","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.15,"eta":0.005,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"sppo_b","method":"sppo_avg","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.20,"eta":0.010,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075}, + {"id":"inpo_a","method":"inpo_avg","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.15,"eta":0.050,"ratio":0.3333,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"inpo_b","method":"inpo_avg","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.20,"eta":0.100,"ratio":0.3333,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075}, + {"id":"ht_help_a","method":"ht_mnpo_helpfulness","dataset":"ht_mnpo_helpfulness","learning_rate":2.5e-7,"warmup_ratio":0.10,"eta":0.005,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"ht_help_b","method":"ht_mnpo_helpfulness","dataset":"ht_mnpo_helpfulness","learning_rate":5e-7,"warmup_ratio":0.15,"eta":0.010,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ht_safety_a","method":"ht_mnpo_safety","dataset":"ht_mnpo_safety","learning_rate":2.5e-7,"warmup_ratio":0.10,"eta":0.005,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"ht_safety_b","method":"ht_mnpo_safety","dataset":"ht_mnpo_safety","learning_rate":5e-7,"warmup_ratio":0.15,"eta":0.010,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ht_concise_a","method":"ht_mnpo_conciseness","dataset":"ht_mnpo_conciseness","learning_rate":2.5e-7,"warmup_ratio":0.10,"eta":0.005,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"ht_concise_b","method":"ht_mnpo_conciseness","dataset":"ht_mnpo_conciseness","learning_rate":5e-7,"warmup_ratio":0.15,"eta":0.010,"reference_anchor_weight":0.05,"preference_sft_weight":0.005} + ] +} diff --git a/experiment/training_status.json b/experiment/training_status.json new file mode 100644 index 0000000..d82bb3d --- /dev/null +++ b/experiment/training_status.json @@ -0,0 +1,20 @@ +{ + "status": "completed", + "candidate_id": "ronpo_top_a", + "method": "ronpo_k_only", + "seed": 42, + "gpu": 2, + "pid": 90276, + "config": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/fair_demo_20260715/sweep/candidates/ronpo_top_a/config.yaml", + "dataset": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/fair_demo_20260715/precomputed/ronpo_k_only_k1", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "b8ffed8fc967", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/b8ffed8fc967", + "started_at": "2026-07-15T09:31:12+09:00", + "returncode": 0, + "model_complete": true, + "finite_results": true, + "measured_step": null, + "completed_at": "2026-07-15T11:35:44+09:00" +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..a5e58c2 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b172b0dbe0b263bc63266e239f7c9a93f7d198c63fe65d90d0387d2662620751 +size 16381517208 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..e2cb12d --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": true, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..c6cc656 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.2860548271752086, + "total_flos": 0.0, + "train_loss": 0.4462373371918996, + "train_runtime": 7238.9345, + "train_samples": 50340, + "train_samples_per_second": 1.989, + "train_steps_per_second": 0.124 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..142db30 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,4903 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.2860548271752086, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "drift/chosen_abs": 0.2889330983161926, + "drift/rejected_abs": 0.1752604842185974, + "epoch": 0.0015891934843067143, + "grad_norm": 1.015625, + "learning_rate": 1.111111111111111e-09, + "logits/chosen": -2.2279419898986816, + "logits/rejected": -2.1537013053894043, + "logps/chosen": -0.2503497004508972, + "logps/rejected": -0.25458794832229614, + "loss": 0.2848242223262787, + "loss/core": 0.2454325258731842, + "loss/preference_sft": 0.24757125973701477, + "loss/reference_anchor": 0.36916011571884155, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.8893306255340576, + "rewards/margins": 1.1367456912994385, + "rewards/rejected": 1.7525850534439087, + "ronpo/logit": 0.1136745810508728, + "ronpo/residual": 0.10866384208202362, + "ronpo/residual_abs": 0.2132132351398468, + "ronpo/target": 0.005010743625462055, + "ronpo/target_abs": 0.04510241001844406, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 5 + }, + { + "drift/chosen_abs": 0.251609742641449, + "drift/rejected_abs": 0.17321258783340454, + "epoch": 0.0031783869686134287, + "grad_norm": 5.75, + "learning_rate": 2.5e-09, + "logits/chosen": -1.9813182353973389, + "logits/rejected": -1.9225966930389404, + "logps/chosen": -0.2639704942703247, + "logps/rejected": -0.27192094922065735, + "loss": 0.19277027249336243, + "loss/core": 0.15751448273658752, + "loss/preference_sft": 0.2704908847808838, + "loss/reference_anchor": 0.32550862431526184, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.5153849124908447, + "rewards/margins": 0.7842203974723816, + "rewards/rejected": 1.7311643362045288, + "ronpo/logit": 0.07842205464839935, + "ronpo/residual": 0.07000575214624405, + "ronpo/residual_abs": 0.17156489193439484, + "ronpo/target": 0.008416297845542431, + "ronpo/target_abs": 0.04953983426094055, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 10 + }, + { + "drift/chosen_abs": 0.14724932610988617, + "drift/rejected_abs": 0.07930952310562134, + "epoch": 0.004767580452920143, + "grad_norm": 7.96875, + "learning_rate": 3.8888888888888884e-09, + "logits/chosen": -2.2914788722991943, + "logits/rejected": -2.24648118019104, + "logps/chosen": -0.27627187967300415, + "logps/rejected": -0.2831260859966278, + "loss": 0.05049755424261093, + "loss/core": 0.041736677289009094, + "loss/preference_sft": 0.2777397632598877, + "loss/reference_anchor": 0.05983476713299751, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.4721348285675049, + "rewards/margins": 0.6800366640090942, + "rewards/rejected": 0.7920981645584106, + "ronpo/logit": 0.06800366193056107, + "ronpo/residual": 0.07898763567209244, + "ronpo/residual_abs": 0.12248041480779648, + "ronpo/target": -0.010983972810208797, + "ronpo/target_abs": 0.053494714200496674, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 15 + }, + { + "drift/chosen_abs": 0.14157260954380035, + "drift/rejected_abs": 0.0700564831495285, + "epoch": 0.006356773937226857, + "grad_norm": 1.5859375, + "learning_rate": 5.277777777777778e-09, + "logits/chosen": -2.291292905807495, + "logits/rejected": -2.2172799110412598, + "logps/chosen": -0.275204598903656, + "logps/rejected": -0.2790767550468445, + "loss": 0.03427541255950928, + "loss/core": 0.02626476064324379, + "loss/preference_sft": 0.27796709537506104, + "loss/reference_anchor": 0.05230981111526489, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 1.4151142835617065, + "rewards/margins": 0.718280553817749, + "rewards/rejected": 0.6968337297439575, + "ronpo/logit": 0.07182805240154266, + "ronpo/residual": 0.07484658807516098, + "ronpo/residual_abs": 0.10549838840961456, + "ronpo/target": -0.003018544986844063, + "ronpo/target_abs": 0.050488390028476715, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 20 + }, + { + "drift/chosen_abs": 0.23460185527801514, + "drift/rejected_abs": 0.07755617797374725, + "epoch": 0.007945967421533572, + "grad_norm": 1.375, + "learning_rate": 6.666666666666666e-09, + "logits/chosen": -2.354492664337158, + "logits/rejected": -2.394556999206543, + "logps/chosen": -0.23961284756660461, + "logps/rejected": -0.2479882687330246, + "loss": 0.3154865503311157, + "loss/core": 0.2928386330604553, + "loss/preference_sft": 0.24814143776893616, + "loss/reference_anchor": 0.2016647309064865, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.3450961112976074, + "rewards/margins": 1.571350336074829, + "rewards/rejected": 0.7737457156181335, + "ronpo/logit": 0.1571349948644638, + "ronpo/residual": 0.16688285768032074, + "ronpo/residual_abs": 0.1950080841779709, + "ronpo/target": -0.009747850708663464, + "ronpo/target_abs": 0.05017418786883354, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 25 + }, + { + "drift/chosen_abs": 0.2047700136899948, + "drift/rejected_abs": 0.08747217059135437, + "epoch": 0.009535160905840286, + "grad_norm": 346.0, + "learning_rate": 8.055555555555556e-09, + "logits/chosen": -2.229592800140381, + "logits/rejected": -2.125317096710205, + "logps/chosen": -0.2870704233646393, + "logps/rejected": -0.2829613983631134, + "loss": 0.17466267943382263, + "loss/core": 0.15042664110660553, + "loss/preference_sft": 0.2907489836215973, + "loss/reference_anchor": 0.21328547596931458, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.047576904296875, + "rewards/margins": 1.1739780902862549, + "rewards/rejected": 0.8735988736152649, + "ronpo/logit": 0.11739782243967056, + "ronpo/residual": 0.1012236699461937, + "ronpo/residual_abs": 0.14672869443893433, + "ronpo/target": 0.016174152493476868, + "ronpo/target_abs": 0.05148453637957573, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 30 + }, + { + "drift/chosen_abs": 0.20920784771442413, + "drift/rejected_abs": 0.13015516102313995, + "epoch": 0.011124354390147001, + "grad_norm": 4.375, + "learning_rate": 9.444444444444444e-09, + "logits/chosen": -2.1240811347961426, + "logits/rejected": -2.0613348484039307, + "logps/chosen": -0.27063313126564026, + "logps/rejected": -0.28193506598472595, + "loss": 0.058792613446712494, + "loss/core": 0.03939628228545189, + "loss/preference_sft": 0.2742695212364197, + "loss/reference_anchor": 0.1665363758802414, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.0917978286743164, + "rewards/margins": 0.7923383116722107, + "rewards/rejected": 1.299459457397461, + "ronpo/logit": 0.07923384010791779, + "ronpo/residual": 0.06704284250736237, + "ronpo/residual_abs": 0.1055685505270958, + "ronpo/target": 0.012190988287329674, + "ronpo/target_abs": 0.04947342351078987, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 35 + }, + { + "drift/chosen_abs": 0.13401487469673157, + "drift/rejected_abs": 0.10863766819238663, + "epoch": 0.012713547874453715, + "grad_norm": 452.0, + "learning_rate": 1.0833333333333333e-08, + "logits/chosen": -2.08085298538208, + "logits/rejected": -2.0408239364624023, + "logps/chosen": -0.318928062915802, + "logps/rejected": -0.32496583461761475, + "loss": 0.24572625756263733, + "loss/core": 0.2277563512325287, + "loss/preference_sft": 0.31747809052467346, + "loss/reference_anchor": 0.1479511708021164, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.3394415378570557, + "rewards/margins": 0.2542641758918762, + "rewards/rejected": 1.0851774215698242, + "ronpo/logit": 0.02542642131447792, + "ronpo/residual": 0.011654033325612545, + "ronpo/residual_abs": 0.14836682379245758, + "ronpo/target": 0.01377238892018795, + "ronpo/target_abs": 0.05200153589248657, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 40 + }, + { + "drift/chosen_abs": 0.1999109983444214, + "drift/rejected_abs": 0.10404626280069351, + "epoch": 0.014302741358760428, + "grad_norm": 3.84375, + "learning_rate": 1.2222222222222222e-08, + "logits/chosen": -2.1482348442077637, + "logits/rejected": -2.0771121978759766, + "logps/chosen": -0.3045925498008728, + "logps/rejected": -0.29603320360183716, + "loss": 0.0930178314447403, + "loss/core": 0.07558077573776245, + "loss/preference_sft": 0.30196070671081543, + "loss/reference_anchor": 0.14417441189289093, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9981660842895508, + "rewards/margins": 0.9596048593521118, + "rewards/rejected": 1.0385613441467285, + "ronpo/logit": 0.09596048295497894, + "ronpo/residual": 0.08603992313146591, + "ronpo/residual_abs": 0.13150832056999207, + "ronpo/target": 0.00992056354880333, + "ronpo/target_abs": 0.05012957379221916, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 45 + }, + { + "drift/chosen_abs": 0.21057943999767303, + "drift/rejected_abs": 0.1738637387752533, + "epoch": 0.015891934843067144, + "grad_norm": 14.6875, + "learning_rate": 1.361111111111111e-08, + "logits/chosen": -2.2325432300567627, + "logits/rejected": -2.280646324157715, + "logps/chosen": -0.38519901037216187, + "logps/rejected": -0.3755597174167633, + "loss": 0.520218551158905, + "loss/core": 0.46836939454078674, + "loss/preference_sft": 0.3819790482521057, + "loss/reference_anchor": 0.4802929759025574, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.1057944297790527, + "rewards/margins": 1.4202427864074707, + "rewards/rejected": 0.6855517625808716, + "ronpo/logit": 0.14202429354190826, + "ronpo/residual": 0.14623448252677917, + "ronpo/residual_abs": 0.21223580837249756, + "ronpo/target": -0.004210175015032291, + "ronpo/target_abs": 0.04790406674146652, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 50 + }, + { + "drift/chosen_abs": 0.24303218722343445, + "drift/rejected_abs": 0.11335645616054535, + "epoch": 0.01748112832737386, + "grad_norm": 1.3359375, + "learning_rate": 1.5e-08, + "logits/chosen": -2.134533643722534, + "logits/rejected": -2.106968402862549, + "logps/chosen": -0.3017352223396301, + "logps/rejected": -0.2939698100090027, + "loss": 0.29720133543014526, + "loss/core": 0.27017584443092346, + "loss/preference_sft": 0.29091566801071167, + "loss/reference_anchor": 0.2411632239818573, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.428701877593994, + "rewards/margins": 1.2995102405548096, + "rewards/rejected": 1.1291919946670532, + "ronpo/logit": 0.12995100021362305, + "ronpo/residual": 0.1421266496181488, + "ronpo/residual_abs": 0.17218372225761414, + "ronpo/target": -0.012175637297332287, + "ronpo/target_abs": 0.05182241275906563, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 55 + }, + { + "drift/chosen_abs": 0.286194771528244, + "drift/rejected_abs": 0.07608653604984283, + "epoch": 0.01907032181168057, + "grad_norm": 8.0, + "learning_rate": 1.6388888888888888e-08, + "logits/chosen": -2.018968105316162, + "logits/rejected": -2.0368776321411133, + "logps/chosen": -0.25825974345207214, + "logps/rejected": -0.26028722524642944, + "loss": 0.7146841287612915, + "loss/core": 0.6727110147476196, + "loss/preference_sft": 0.26158055663108826, + "loss/reference_anchor": 0.39357417821884155, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.861429214477539, + "rewards/margins": 2.1107606887817383, + "rewards/rejected": 0.7506682872772217, + "ronpo/logit": 0.2110760658979416, + "ronpo/residual": 0.21510767936706543, + "ronpo/residual_abs": 0.2415662556886673, + "ronpo/target": -0.0040316274389624596, + "ronpo/target_abs": 0.04714924842119217, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 60 + }, + { + "drift/chosen_abs": 0.23433828353881836, + "drift/rejected_abs": 0.084614098072052, + "epoch": 0.020659515295987287, + "grad_norm": 173.0, + "learning_rate": 1.7777777777777777e-08, + "logits/chosen": -2.3905935287475586, + "logits/rejected": -2.4606516361236572, + "logps/chosen": -0.25088340044021606, + "logps/rejected": -0.2440684586763382, + "loss": 0.27581101655960083, + "loss/core": 0.2562420666217804, + "loss/preference_sft": 0.24446015059947968, + "loss/reference_anchor": 0.17124369740486145, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.3433830738067627, + "rewards/margins": 1.4972563982009888, + "rewards/rejected": 0.8461267352104187, + "ronpo/logit": 0.14972564578056335, + "ronpo/residual": 0.13597126305103302, + "ronpo/residual_abs": 0.17988578975200653, + "ronpo/target": 0.013754380866885185, + "ronpo/target_abs": 0.051927365362644196, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 65 + }, + { + "drift/chosen_abs": 0.218926340341568, + "drift/rejected_abs": 0.11191727966070175, + "epoch": 0.022248708780294002, + "grad_norm": 2.734375, + "learning_rate": 1.9166666666666666e-08, + "logits/chosen": -2.131840944290161, + "logits/rejected": -2.131176233291626, + "logps/chosen": -0.2633002996444702, + "logps/rejected": -0.26801466941833496, + "loss": 0.07617291808128357, + "loss/core": 0.062096256762742996, + "loss/preference_sft": 0.2654517590999603, + "loss/reference_anchor": 0.11422140896320343, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.188795804977417, + "rewards/margins": 1.0696232318878174, + "rewards/rejected": 1.1191728115081787, + "ronpo/logit": 0.10696232318878174, + "ronpo/residual": 0.10355587303638458, + "ronpo/residual_abs": 0.12823593616485596, + "ronpo/target": 0.003406452713534236, + "ronpo/target_abs": 0.05231611803174019, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 70 + }, + { + "drift/chosen_abs": 0.2827184796333313, + "drift/rejected_abs": 0.147368922829628, + "epoch": 0.023837902264600714, + "grad_norm": 4.875, + "learning_rate": 2.0555555555555552e-08, + "logits/chosen": -2.1087307929992676, + "logits/rejected": -2.042454242706299, + "logps/chosen": -0.2781427502632141, + "logps/rejected": -0.27138060331344604, + "loss": 0.6821389198303223, + "loss/core": 0.637114942073822, + "loss/preference_sft": 0.270221084356308, + "loss/reference_anchor": 0.4232177734375, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.826986789703369, + "rewards/margins": 1.3705655336380005, + "rewards/rejected": 1.4564214944839478, + "ronpo/logit": 0.13705654442310333, + "ronpo/residual": 0.13803334534168243, + "ronpo/residual_abs": 0.2660602927207947, + "ronpo/target": -0.0009768044110387564, + "ronpo/target_abs": 0.050860919058322906, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 75 + }, + { + "drift/chosen_abs": 0.32659146189689636, + "drift/rejected_abs": 0.13844743371009827, + "epoch": 0.02542709574890743, + "grad_norm": 38.5, + "learning_rate": 2.1944444444444445e-08, + "logits/chosen": -2.052938222885132, + "logits/rejected": -2.021271228790283, + "logps/chosen": -0.270699679851532, + "logps/rejected": -0.2735486626625061, + "loss": 0.4138568341732025, + "loss/core": 0.37115219235420227, + "loss/preference_sft": 0.2724604308605194, + "loss/reference_anchor": 0.39979997277259827, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.2659149169921875, + "rewards/margins": 1.8850905895233154, + "rewards/rejected": 1.3808239698410034, + "ronpo/logit": 0.18850906193256378, + "ronpo/residual": 0.17836426198482513, + "ronpo/residual_abs": 0.21546652913093567, + "ronpo/target": 0.010144800879061222, + "ronpo/target_abs": 0.04853076487779617, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 80 + }, + { + "drift/chosen_abs": 0.2298695147037506, + "drift/rejected_abs": 0.07848798483610153, + "epoch": 0.027016289233214145, + "grad_norm": 1024.0, + "learning_rate": 2.3333333333333334e-08, + "logits/chosen": -2.3688607215881348, + "logits/rejected": -2.3101439476013184, + "logps/chosen": -0.2924603819847107, + "logps/rejected": -0.26657044887542725, + "loss": 0.5983717441558838, + "loss/core": 0.5626398324966431, + "loss/preference_sft": 0.27175357937812805, + "loss/reference_anchor": 0.3301441967487335, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.297607898712158, + "rewards/margins": 1.5177080631256104, + "rewards/rejected": 0.7798997163772583, + "ronpo/logit": 0.15177080035209656, + "ronpo/residual": 0.14059045910835266, + "ronpo/residual_abs": 0.18269233405590057, + "ronpo/target": 0.011180354282259941, + "ronpo/target_abs": 0.053492844104766846, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 85 + }, + { + "drift/chosen_abs": 0.19050000607967377, + "drift/rejected_abs": 0.15556380152702332, + "epoch": 0.028605482717520857, + "grad_norm": 1.3984375, + "learning_rate": 2.4722222222222223e-08, + "logits/chosen": -2.261676788330078, + "logits/rejected": -2.2711334228515625, + "logps/chosen": -0.2823393940925598, + "logps/rejected": -0.2829485237598419, + "loss": 0.24817046523094177, + "loss/core": 0.2262168675661087, + "loss/preference_sft": 0.2772825360298157, + "loss/reference_anchor": 0.1918078511953354, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.9050003290176392, + "rewards/margins": 0.3525739014148712, + "rewards/rejected": 1.5524263381958008, + "ronpo/logit": 0.03525739163160324, + "ronpo/residual": 0.03311719745397568, + "ronpo/residual_abs": 0.1650288850069046, + "ronpo/target": 0.0021401962731033564, + "ronpo/target_abs": 0.05777706578373909, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 90 + }, + { + "drift/chosen_abs": 0.2254537045955658, + "drift/rejected_abs": 0.1015145406126976, + "epoch": 0.030194676201827572, + "grad_norm": 3.328125, + "learning_rate": 2.6111111111111112e-08, + "logits/chosen": -2.3401074409484863, + "logits/rejected": -2.2592806816101074, + "logps/chosen": -0.2731631100177765, + "logps/rejected": -0.2693111002445221, + "loss": 0.3633732199668884, + "loss/core": 0.33972230553627014, + "loss/preference_sft": 0.2640019357204437, + "loss/reference_anchor": 0.21010908484458923, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.2541868686676025, + "rewards/margins": 1.2461910247802734, + "rewards/rejected": 1.007995843887329, + "ronpo/logit": 0.12461910396814346, + "ronpo/residual": 0.12321188300848007, + "ronpo/residual_abs": 0.19888217747211456, + "ronpo/target": 0.0014072132762521505, + "ronpo/target_abs": 0.055929411202669144, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 95 + }, + { + "drift/chosen_abs": 0.2529149651527405, + "drift/rejected_abs": 0.10006153583526611, + "epoch": 0.03178386968613429, + "grad_norm": 9.75, + "learning_rate": 2.75e-08, + "logits/chosen": -1.9696271419525146, + "logits/rejected": -2.041968822479248, + "logps/chosen": -0.2982648015022278, + "logps/rejected": -0.2892451584339142, + "loss": 0.2479362040758133, + "loss/core": 0.22514848411083221, + "loss/preference_sft": 0.28658321499824524, + "loss/reference_anchor": 0.1992187649011612, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.52838134765625, + "rewards/margins": 1.5295547246932983, + "rewards/rejected": 0.9988266229629517, + "ronpo/logit": 0.15295547246932983, + "ronpo/residual": 0.1562143713235855, + "ronpo/residual_abs": 0.18405328691005707, + "ronpo/target": -0.0032588993199169636, + "ronpo/target_abs": 0.04928561672568321, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 100 + }, + { + "drift/chosen_abs": 0.3901594877243042, + "drift/rejected_abs": 0.12727132439613342, + "epoch": 0.033373063170441, + "grad_norm": 96.0, + "learning_rate": 2.8888888888888884e-08, + "logits/chosen": -1.9293874502182007, + "logits/rejected": -1.936822533607483, + "logps/chosen": -0.2803095877170563, + "logps/rejected": -0.283048152923584, + "loss": 0.7342885732650757, + "loss/core": 0.6701232194900513, + "loss/preference_sft": 0.27669522166252136, + "loss/reference_anchor": 0.6139832735061646, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.9012649059295654, + "rewards/margins": 2.641878604888916, + "rewards/rejected": 1.259386420249939, + "ronpo/logit": 0.26418787240982056, + "ronpo/residual": 0.2662436068058014, + "ronpo/residual_abs": 0.29203924536705017, + "ronpo/target": -0.0020557777024805546, + "ronpo/target_abs": 0.053755033761262894, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 105 + }, + { + "drift/chosen_abs": 0.2434467077255249, + "drift/rejected_abs": 0.1235206350684166, + "epoch": 0.03496225665474772, + "grad_norm": 8.1875, + "learning_rate": 3.027777777777777e-08, + "logits/chosen": -2.2449328899383545, + "logits/rejected": -2.2041220664978027, + "logps/chosen": -0.2943647503852844, + "logps/rejected": -0.2864592969417572, + "loss": 0.12976226210594177, + "loss/core": 0.10662249475717545, + "loss/preference_sft": 0.2895354628562927, + "loss/reference_anchor": 0.20244407653808594, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.431950807571411, + "rewards/margins": 1.2087242603302002, + "rewards/rejected": 1.223226547241211, + "ronpo/logit": 0.12087241560220718, + "ronpo/residual": 0.12740285694599152, + "ronpo/residual_abs": 0.15507419407367706, + "ronpo/target": -0.006530435290187597, + "ronpo/target_abs": 0.048901356756687164, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 110 + }, + { + "drift/chosen_abs": 0.27528631687164307, + "drift/rejected_abs": 0.13778968155384064, + "epoch": 0.03655145013905443, + "grad_norm": 1.6640625, + "learning_rate": 3.1666666666666666e-08, + "logits/chosen": -1.8844751119613647, + "logits/rejected": -1.7757047414779663, + "logps/chosen": -0.2790004312992096, + "logps/rejected": -0.28186291456222534, + "loss": 0.16678151488304138, + "loss/core": 0.1427289992570877, + "loss/preference_sft": 0.2831977903842926, + "loss/reference_anchor": 0.2122054547071457, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.7513701915740967, + "rewards/margins": 1.3792320489883423, + "rewards/rejected": 1.3721381425857544, + "ronpo/logit": 0.1379232108592987, + "ronpo/residual": 0.1344049870967865, + "ronpo/residual_abs": 0.1711626797914505, + "ronpo/target": 0.0035182249266654253, + "ronpo/target_abs": 0.05670144036412239, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 115 + }, + { + "drift/chosen_abs": 0.251396119594574, + "drift/rejected_abs": 0.11713407188653946, + "epoch": 0.03814064362336114, + "grad_norm": 1.4765625, + "learning_rate": 3.305555555555555e-08, + "logits/chosen": -2.0189380645751953, + "logits/rejected": -2.066065549850464, + "logps/chosen": -0.3038211464881897, + "logps/rejected": -0.30019038915634155, + "loss": 0.3087940514087677, + "loss/core": 0.27471792697906494, + "loss/preference_sft": 0.3073151111602783, + "loss/reference_anchor": 0.31002968549728394, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.5131027698516846, + "rewards/margins": 1.34487783908844, + "rewards/rejected": 1.168224573135376, + "ronpo/logit": 0.1344877928495407, + "ronpo/residual": 0.13394805788993835, + "ronpo/residual_abs": 0.19293567538261414, + "ronpo/target": 0.0005397285567596555, + "ronpo/target_abs": 0.04854871332645416, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 120 + }, + { + "drift/chosen_abs": 0.12723875045776367, + "drift/rejected_abs": 0.07009278982877731, + "epoch": 0.03972983710766786, + "grad_norm": 0.9375, + "learning_rate": 3.4444444444444444e-08, + "logits/chosen": -2.273139715194702, + "logits/rejected": -2.194185972213745, + "logps/chosen": -0.2742195725440979, + "logps/rejected": -0.2820054292678833, + "loss": 0.04828890785574913, + "loss/core": 0.03853292018175125, + "loss/preference_sft": 0.2750937342643738, + "loss/reference_anchor": 0.07005052268505096, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.2721223831176758, + "rewards/margins": 0.5717689990997314, + "rewards/rejected": 0.7003533840179443, + "ronpo/logit": 0.057176899164915085, + "ronpo/residual": 0.06078102067112923, + "ronpo/residual_abs": 0.09998279809951782, + "ronpo/target": -0.0036041145212948322, + "ronpo/target_abs": 0.05498850345611572, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 125 + }, + { + "drift/chosen_abs": 0.36905351281166077, + "drift/rejected_abs": 0.15415672957897186, + "epoch": 0.04131903059197457, + "grad_norm": 15.1875, + "learning_rate": 3.583333333333333e-08, + "logits/chosen": -2.0486526489257812, + "logits/rejected": -1.976944923400879, + "logps/chosen": -0.28115013241767883, + "logps/rejected": -0.29946258664131165, + "loss": 0.5215417146682739, + "loss/core": 0.4482294023036957, + "loss/preference_sft": 0.27702564001083374, + "loss/reference_anchor": 0.7054204940795898, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.690535068511963, + "rewards/margins": 2.1494040489196777, + "rewards/rejected": 1.541130781173706, + "ronpo/logit": 0.21494042873382568, + "ronpo/residual": 0.21962769329547882, + "ronpo/residual_abs": 0.2594267427921295, + "ronpo/target": -0.004687254782766104, + "ronpo/target_abs": 0.05417006462812424, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 130 + }, + { + "drift/chosen_abs": 0.2299853265285492, + "drift/rejected_abs": 0.1746150255203247, + "epoch": 0.04290822407628129, + "grad_norm": 104.5, + "learning_rate": 3.722222222222222e-08, + "logits/chosen": -2.1584811210632324, + "logits/rejected": -2.1437926292419434, + "logps/chosen": -0.25327831506729126, + "logps/rejected": -0.27471664547920227, + "loss": 0.4051688313484192, + "loss/core": 0.37376266717910767, + "loss/preference_sft": 0.2582192122936249, + "loss/reference_anchor": 0.2882392406463623, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.298335552215576, + "rewards/margins": 0.5546636581420898, + "rewards/rejected": 1.7436714172363281, + "ronpo/logit": 0.05546638369560242, + "ronpo/residual": 0.06129222363233566, + "ronpo/residual_abs": 0.2028910219669342, + "ronpo/target": -0.005825856234878302, + "ronpo/target_abs": 0.05126802995800972, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 135 + }, + { + "drift/chosen_abs": 0.21368154883384705, + "drift/rejected_abs": 0.1560516655445099, + "epoch": 0.044497417560588004, + "grad_norm": 42.5, + "learning_rate": 3.861111111111111e-08, + "logits/chosen": -2.065540075302124, + "logits/rejected": -2.036073684692383, + "logps/chosen": -0.28536534309387207, + "logps/rejected": -0.2856103181838989, + "loss": 0.2617502212524414, + "loss/core": 0.23660394549369812, + "loss/preference_sft": 0.28660038113594055, + "loss/reference_anchor": 0.22280272841453552, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.1362662315368652, + "rewards/margins": 0.5771859884262085, + "rewards/rejected": 1.5590801239013672, + "ronpo/logit": 0.05771859735250473, + "ronpo/residual": 0.0572664849460125, + "ronpo/residual_abs": 0.18445725739002228, + "ronpo/target": 0.0004521068185567856, + "ronpo/target_abs": 0.05590490251779556, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 140 + }, + { + "drift/chosen_abs": 0.3109515607357025, + "drift/rejected_abs": 0.1370420604944229, + "epoch": 0.04608661104489471, + "grad_norm": 242.0, + "learning_rate": 4e-08, + "logits/chosen": -2.100191116333008, + "logits/rejected": -2.1371798515319824, + "logps/chosen": -0.2746989130973816, + "logps/rejected": -0.2657720148563385, + "loss": 0.2308829128742218, + "loss/core": 0.19054999947547913, + "loss/preference_sft": 0.2726476788520813, + "loss/reference_anchor": 0.3760644197463989, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.108795642852783, + "rewards/margins": 1.7410615682601929, + "rewards/rejected": 1.3677339553833008, + "ronpo/logit": 0.174106165766716, + "ronpo/residual": 0.17656587064266205, + "ronpo/residual_abs": 0.20304152369499207, + "ronpo/target": -0.002459695329889655, + "ronpo/target_abs": 0.04878217726945877, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 145 + }, + { + "drift/chosen_abs": 0.4026366174221039, + "drift/rejected_abs": 0.11420154571533203, + "epoch": 0.04767580452920143, + "grad_norm": 3.046875, + "learning_rate": 4.138888888888889e-08, + "logits/chosen": -2.0290703773498535, + "logits/rejected": -2.1320388317108154, + "logps/chosen": -0.2950589656829834, + "logps/rejected": -0.27424511313438416, + "loss": 1.086280107498169, + "loss/core": 1.0141551494598389, + "loss/preference_sft": 0.2878630757331848, + "loss/reference_anchor": 0.692463755607605, + "rewards/accuracies": 0.9375, + "rewards/chosen": 4.026366233825684, + "rewards/margins": 2.884765625, + "rewards/rejected": 1.1416008472442627, + "ronpo/logit": 0.28847652673721313, + "ronpo/residual": 0.2900107502937317, + "ronpo/residual_abs": 0.32643136382102966, + "ronpo/target": -0.0015342000406235456, + "ronpo/target_abs": 0.05635756999254227, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 150 + }, + { + "drift/chosen_abs": 0.31187811493873596, + "drift/rejected_abs": 0.10443463176488876, + "epoch": 0.04926499801350814, + "grad_norm": 3.46875, + "learning_rate": 4.277777777777777e-08, + "logits/chosen": -2.167552947998047, + "logits/rejected": -2.173797607421875, + "logps/chosen": -0.2655718922615051, + "logps/rejected": -0.26215478777885437, + "loss": 0.7132057547569275, + "loss/core": 0.6582164764404297, + "loss/preference_sft": 0.2656114995479584, + "loss/reference_anchor": 0.523330807685852, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.1172947883605957, + "rewards/margins": 2.077317714691162, + "rewards/rejected": 1.0399768352508545, + "ronpo/logit": 0.20773179829120636, + "ronpo/residual": 0.21012091636657715, + "ronpo/residual_abs": 0.2313835173845291, + "ronpo/target": -0.002389120403677225, + "ronpo/target_abs": 0.04883997514843941, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 155 + }, + { + "drift/chosen_abs": 0.2020689994096756, + "drift/rejected_abs": 0.06760167330503464, + "epoch": 0.05085419149781486, + "grad_norm": 900.0, + "learning_rate": 4.4166666666666665e-08, + "logits/chosen": -2.207207202911377, + "logits/rejected": -2.223909378051758, + "logps/chosen": -0.3178792893886566, + "logps/rejected": -0.2741866707801819, + "loss": 0.596080482006073, + "loss/core": 0.5607853531837463, + "loss/preference_sft": 0.2734844386577606, + "loss/reference_anchor": 0.32560402154922485, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.0203135013580322, + "rewards/margins": 1.346987009048462, + "rewards/rejected": 0.6733266115188599, + "ronpo/logit": 0.13469868898391724, + "ronpo/residual": 0.1416359394788742, + "ronpo/residual_abs": 0.1742027997970581, + "ronpo/target": -0.006937255151569843, + "ronpo/target_abs": 0.0489855594933033, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 160 + }, + { + "drift/chosen_abs": 0.13068225979804993, + "drift/rejected_abs": 0.05099822208285332, + "epoch": 0.052443384982121574, + "grad_norm": 2.859375, + "learning_rate": 4.555555555555555e-08, + "logits/chosen": -2.1282591819763184, + "logits/rejected": -2.0065226554870605, + "logps/chosen": -0.2849346101284027, + "logps/rejected": -0.29358747601509094, + "loss": 0.10835715383291245, + "loss/core": 0.09872569143772125, + "loss/preference_sft": 0.28516682982444763, + "loss/reference_anchor": 0.06779788434505463, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.3054677248001099, + "rewards/margins": 0.7985641360282898, + "rewards/rejected": 0.5069035887718201, + "ronpo/logit": 0.07985641807317734, + "ronpo/residual": 0.08017878234386444, + "ronpo/residual_abs": 0.11188485473394394, + "ronpo/target": -0.0003223681706003845, + "ronpo/target_abs": 0.04679277911782265, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 165 + }, + { + "drift/chosen_abs": 0.3152261972427368, + "drift/rejected_abs": 0.16839492321014404, + "epoch": 0.05403257846642829, + "grad_norm": 1.734375, + "learning_rate": 4.6944444444444443e-08, + "logits/chosen": -2.0259292125701904, + "logits/rejected": -2.019106149673462, + "logps/chosen": -0.2906624674797058, + "logps/rejected": -0.2745984196662903, + "loss": 0.4489654004573822, + "loss/core": 0.4040234088897705, + "loss/preference_sft": 0.2755510210990906, + "loss/reference_anchor": 0.4218643307685852, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.1518421173095703, + "rewards/margins": 1.471212387084961, + "rewards/rejected": 1.680630087852478, + "ronpo/logit": 0.14712122082710266, + "ronpo/residual": 0.1320314109325409, + "ronpo/residual_abs": 0.2297447919845581, + "ronpo/target": 0.015089799650013447, + "ronpo/target_abs": 0.05001477152109146, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 170 + }, + { + "drift/chosen_abs": 0.1986486166715622, + "drift/rejected_abs": 0.11206654459238052, + "epoch": 0.055621771950735005, + "grad_norm": 31.5, + "learning_rate": 4.833333333333333e-08, + "logits/chosen": -2.2629666328430176, + "logits/rejected": -2.200096845626831, + "logps/chosen": -0.29371052980422974, + "logps/rejected": -0.3042106628417969, + "loss": 0.06993673741817474, + "loss/core": 0.05088762566447258, + "loss/preference_sft": 0.2960934638977051, + "loss/reference_anchor": 0.16088177263736725, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.9864861965179443, + "rewards/margins": 0.8661035299301147, + "rewards/rejected": 1.1203826665878296, + "ronpo/logit": 0.0866103544831276, + "ronpo/residual": 0.08271974325180054, + "ronpo/residual_abs": 0.11550427973270416, + "ronpo/target": 0.0038906182162463665, + "ronpo/target_abs": 0.049135539680719376, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 175 + }, + { + "drift/chosen_abs": 0.47210416197776794, + "drift/rejected_abs": 0.2260768860578537, + "epoch": 0.057210965435041714, + "grad_norm": 47.0, + "learning_rate": 4.972222222222222e-08, + "logits/chosen": -2.071915864944458, + "logits/rejected": -2.048391103744507, + "logps/chosen": -0.2908160388469696, + "logps/rejected": -0.28190651535987854, + "loss": 0.9919503927230835, + "loss/core": 0.8761954307556152, + "loss/preference_sft": 0.28307628631591797, + "loss/reference_anchor": 1.129240870475769, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.721041679382324, + "rewards/margins": 2.460272789001465, + "rewards/rejected": 2.2607688903808594, + "ronpo/logit": 0.24602727591991425, + "ronpo/residual": 0.250581294298172, + "ronpo/residual_abs": 0.29279235005378723, + "ronpo/target": -0.004554013721644878, + "ronpo/target_abs": 0.043643683195114136, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 180 + }, + { + "drift/chosen_abs": 0.1224537044763565, + "drift/rejected_abs": 0.06466648727655411, + "epoch": 0.05880015891934843, + "grad_norm": 5.75, + "learning_rate": 4.999619237890978e-08, + "logits/chosen": -2.226485013961792, + "logits/rejected": -2.3286421298980713, + "logps/chosen": -0.29493987560272217, + "logps/rejected": -0.29172560572624207, + "loss": 0.03143535554409027, + "loss/core": 0.024793710559606552, + "loss/preference_sft": 0.2961380183696747, + "loss/reference_anchor": 0.03680265694856644, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.2238212823867798, + "rewards/margins": 0.5794644355773926, + "rewards/rejected": 0.6443568468093872, + "ronpo/logit": 0.05794644355773926, + "ronpo/residual": 0.07471181452274323, + "ronpo/residual_abs": 0.10397426784038544, + "ronpo/target": -0.01676536537706852, + "ronpo/target_abs": 0.05582839995622635, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 185 + }, + { + "drift/chosen_abs": 0.12504041194915771, + "drift/rejected_abs": 0.055094581097364426, + "epoch": 0.060389352403655144, + "grad_norm": 0.8984375, + "learning_rate": 4.9980725906018076e-08, + "logits/chosen": -2.314112663269043, + "logits/rejected": -2.4101805686950684, + "logps/chosen": -0.3051087260246277, + "logps/rejected": -0.3032669425010681, + "loss": 0.05326960235834122, + "loss/core": 0.04397609829902649, + "loss/preference_sft": 0.3047112822532654, + "loss/reference_anchor": 0.0624639168381691, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.2492070198059082, + "rewards/margins": 0.6990978717803955, + "rewards/rejected": 0.5501092076301575, + "ronpo/logit": 0.06990978866815567, + "ronpo/residual": 0.07269351929426193, + "ronpo/residual_abs": 0.11123079061508179, + "ronpo/target": -0.00278373877517879, + "ronpo/target_abs": 0.05910308286547661, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 190 + }, + { + "drift/chosen_abs": 0.21298687160015106, + "drift/rejected_abs": 0.1180461049079895, + "epoch": 0.06197854588796186, + "grad_norm": 6.0625, + "learning_rate": 4.995336996054667e-08, + "logits/chosen": -2.0543062686920166, + "logits/rejected": -2.029996156692505, + "logps/chosen": -0.2776458263397217, + "logps/rejected": -0.2725464403629303, + "loss": 0.06012827157974243, + "loss/core": 0.04569826275110245, + "loss/preference_sft": 0.2735288739204407, + "loss/reference_anchor": 0.11694715172052383, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.128607988357544, + "rewards/margins": 0.9508472681045532, + "rewards/rejected": 1.1777608394622803, + "ronpo/logit": 0.09508472681045532, + "ronpo/residual": 0.09097838401794434, + "ronpo/residual_abs": 0.11423306167125702, + "ronpo/target": 0.004106341395527124, + "ronpo/target_abs": 0.04503884166479111, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 195 + }, + { + "drift/chosen_abs": 0.2828276753425598, + "drift/rejected_abs": 0.08978013694286346, + "epoch": 0.06356773937226858, + "grad_norm": 9.625, + "learning_rate": 4.991413756244404e-08, + "logits/chosen": -2.0380451679229736, + "logits/rejected": -2.1581947803497314, + "logps/chosen": -0.2897427976131439, + "logps/rejected": -0.2790738642215729, + "loss": 0.5045924186706543, + "loss/core": 0.4699951708316803, + "loss/preference_sft": 0.28447893261909485, + "loss/reference_anchor": 0.31752467155456543, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.827852964401245, + "rewards/margins": 1.9351660013198853, + "rewards/rejected": 0.8926870226860046, + "ronpo/logit": 0.19351661205291748, + "ronpo/residual": 0.18979412317276, + "ronpo/residual_abs": 0.22088727355003357, + "ronpo/target": 0.003722475143149495, + "ronpo/target_abs": 0.04679802060127258, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 200 + }, + { + "drift/chosen_abs": 0.2945198118686676, + "drift/rejected_abs": 0.10771737992763519, + "epoch": 0.06515693285657528, + "grad_norm": 98.0, + "learning_rate": 4.986304738420683e-08, + "logits/chosen": -2.0024924278259277, + "logits/rejected": -2.0828309059143066, + "logps/chosen": -0.28146928548812866, + "logps/rejected": -0.2874804735183716, + "loss": 0.7829475998878479, + "loss/core": 0.7258787155151367, + "loss/preference_sft": 0.28588175773620605, + "loss/reference_anchor": 0.5421004295349121, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.942234992980957, + "rewards/margins": 1.8656885623931885, + "rewards/rejected": 1.076546549797058, + "ronpo/logit": 0.18656882643699646, + "ronpo/residual": 0.19427460432052612, + "ronpo/residual_abs": 0.22354765236377716, + "ronpo/target": -0.007705748081207275, + "ronpo/target_abs": 0.053630173206329346, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 205 + }, + { + "drift/chosen_abs": 0.21879103779792786, + "drift/rejected_abs": 0.057897716760635376, + "epoch": 0.066746126340882, + "grad_norm": 4.0625, + "learning_rate": 4.980012374199287e-08, + "logits/chosen": -2.2520713806152344, + "logits/rejected": -2.1708178520202637, + "logps/chosen": -0.308249294757843, + "logps/rejected": -0.315049946308136, + "loss": 1.0403220653533936, + "loss/core": 0.9826607704162598, + "loss/preference_sft": 0.30766767263412476, + "loss/reference_anchor": 0.5458493828773499, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.1878044605255127, + "rewards/margins": 1.6101818084716797, + "rewards/rejected": 0.5776225924491882, + "ronpo/logit": 0.16101817786693573, + "ronpo/residual": 0.17357835173606873, + "ronpo/residual_abs": 0.19822609424591064, + "ronpo/target": -0.012560190632939339, + "ronpo/target_abs": 0.052357811480760574, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 210 + }, + { + "drift/chosen_abs": 0.3271009922027588, + "drift/rejected_abs": 0.16482210159301758, + "epoch": 0.06833531982518871, + "grad_norm": 1.59375, + "learning_rate": 4.9725396584047915e-08, + "logits/chosen": -2.1691105365753174, + "logits/rejected": -2.1548848152160645, + "logps/chosen": -0.2653549015522003, + "logps/rejected": -0.2719860374927521, + "loss": 0.32540175318717957, + "loss/core": 0.26584428548812866, + "loss/preference_sft": 0.270608127117157, + "loss/reference_anchor": 0.5685140490531921, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 3.271010160446167, + "rewards/margins": 1.6233627796173096, + "rewards/rejected": 1.6476472616195679, + "ronpo/logit": 0.16233626008033752, + "ronpo/residual": 0.1596986949443817, + "ronpo/residual_abs": 0.18813616037368774, + "ronpo/target": 0.0026375583838671446, + "ronpo/target_abs": 0.04490605369210243, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 215 + }, + { + "drift/chosen_abs": 0.11970488727092743, + "drift/rejected_abs": 0.08118265867233276, + "epoch": 0.06992451330949544, + "grad_norm": 0.36328125, + "learning_rate": 4.963890147645195e-08, + "logits/chosen": -2.2142767906188965, + "logits/rejected": -2.081634759902954, + "logps/chosen": -0.3046541213989258, + "logps/rejected": -0.2997341454029083, + "loss": 0.05313153192400932, + "loss/core": 0.045478448271751404, + "loss/preference_sft": 0.3017224669456482, + "loss/reference_anchor": 0.04635859280824661, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.1934808492660522, + "rewards/margins": 0.384704053401947, + "rewards/rejected": 0.8087766766548157, + "ronpo/logit": 0.03847040981054306, + "ronpo/residual": 0.03697671741247177, + "ronpo/residual_abs": 0.10223197937011719, + "ronpo/target": 0.0014936840161681175, + "ronpo/target_abs": 0.04688495397567749, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 220 + }, + { + "drift/chosen_abs": 0.2311873733997345, + "drift/rejected_abs": 0.10326783359050751, + "epoch": 0.07151370679380215, + "grad_norm": 5.09375, + "learning_rate": 4.95406795861916e-08, + "logits/chosen": -2.177318572998047, + "logits/rejected": -2.1946113109588623, + "logps/chosen": -0.2876010835170746, + "logps/rejected": -0.2852502465248108, + "loss": 0.20730121433734894, + "loss/core": 0.18707375228405, + "loss/preference_sft": 0.2840942442417145, + "loss/reference_anchor": 0.17386510968208313, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.3118736743927, + "rewards/margins": 1.279848575592041, + "rewards/rejected": 1.0320250988006592, + "ronpo/logit": 0.12798485159873962, + "ronpo/residual": 0.12988588213920593, + "ronpo/residual_abs": 0.16303423047065735, + "ronpo/target": -0.0019010124960914254, + "ronpo/target_abs": 0.056721486151218414, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 225 + }, + { + "drift/chosen_abs": 0.2658592760562897, + "drift/rejected_abs": 0.09674189984798431, + "epoch": 0.07310290027810885, + "grad_norm": 2.359375, + "learning_rate": 4.943077766156697e-08, + "logits/chosen": -2.2422900199890137, + "logits/rejected": -2.2595908641815186, + "logps/chosen": -0.2615046203136444, + "logps/rejected": -0.26030460000038147, + "loss": 0.4114556908607483, + "loss/core": 0.38013729453086853, + "loss/preference_sft": 0.2594269812107086, + "loss/reference_anchor": 0.28724151849746704, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.6584160327911377, + "rewards/margins": 1.691518783569336, + "rewards/rejected": 0.9668973684310913, + "ronpo/logit": 0.16915185749530792, + "ronpo/residual": 0.17316488921642303, + "ronpo/residual_abs": 0.19615182280540466, + "ronpo/target": -0.004013021010905504, + "ronpo/target_abs": 0.05004221946001053, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 230 + }, + { + "drift/chosen_abs": 0.4598860740661621, + "drift/rejected_abs": 0.2359577715396881, + "epoch": 0.07469209376241558, + "grad_norm": 64.5, + "learning_rate": 4.930924800994191e-08, + "logits/chosen": -2.114593982696533, + "logits/rejected": -2.116748809814453, + "logps/chosen": -0.2821466028690338, + "logps/rejected": -0.2878660261631012, + "loss": 1.2597297430038452, + "loss/core": 1.1379563808441162, + "loss/preference_sft": 0.27529674768447876, + "loss/reference_anchor": 1.1902048587799072, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.597031593322754, + "rewards/margins": 2.3334996700286865, + "rewards/rejected": 2.2635321617126465, + "ronpo/logit": 0.2333499640226364, + "ronpo/residual": 0.23511190712451935, + "ronpo/residual_abs": 0.37244728207588196, + "ronpo/target": -0.0017619434511289, + "ronpo/target_abs": 0.04279688373208046, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 235 + }, + { + "drift/chosen_abs": 0.29015716910362244, + "drift/rejected_abs": 0.16504958271980286, + "epoch": 0.07628128724672228, + "grad_norm": 149.0, + "learning_rate": 4.9176148472848576e-08, + "logits/chosen": -2.139037609100342, + "logits/rejected": -2.214994192123413, + "logps/chosen": -0.26754242181777954, + "logps/rejected": -0.287071168422699, + "loss": 0.24881801009178162, + "loss/core": 0.19462670385837555, + "loss/preference_sft": 0.28026410937309265, + "loss/reference_anchor": 0.5138863325119019, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.9009900093078613, + "rewards/margins": 1.2524168491363525, + "rewards/rejected": 1.6485732793807983, + "ronpo/logit": 0.12524166703224182, + "ronpo/residual": 0.1251114010810852, + "ronpo/residual_abs": 0.15745453536510468, + "ronpo/target": 0.00013028904504608363, + "ronpo/target_abs": 0.05093258619308472, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 240 + }, + { + "drift/chosen_abs": 0.1876174658536911, + "drift/rejected_abs": 0.07340504974126816, + "epoch": 0.07787048073102901, + "grad_norm": 1.609375, + "learning_rate": 4.903154239845797e-08, + "logits/chosen": -2.1785225868225098, + "logits/rejected": -2.1970629692077637, + "logps/chosen": -0.29958945512771606, + "logps/rejected": -0.3053562045097351, + "loss": 0.28825491666793823, + "loss/core": 0.26845070719718933, + "loss/preference_sft": 0.31076282262802124, + "loss/reference_anchor": 0.16696596145629883, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8747392892837524, + "rewards/margins": 1.171970009803772, + "rewards/rejected": 0.7027692198753357, + "ronpo/logit": 0.11719699203968048, + "ronpo/residual": 0.11347774416208267, + "ronpo/residual_abs": 0.15537722408771515, + "ronpo/target": 0.0037192567251622677, + "ronpo/target_abs": 0.05189714580774307, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 245 + }, + { + "drift/chosen_abs": 0.36311304569244385, + "drift/rejected_abs": 0.08675487339496613, + "epoch": 0.07945967421533572, + "grad_norm": 7.0625, + "learning_rate": 4.887549861142967e-08, + "logits/chosen": -2.1118674278259277, + "logits/rejected": -2.088712215423584, + "logps/chosen": -0.3086296021938324, + "logps/rejected": -0.3087017834186554, + "loss": 1.2505347728729248, + "loss/core": 1.1789346933364868, + "loss/preference_sft": 0.30625787377357483, + "loss/reference_anchor": 0.6853760480880737, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 3.6298019886016846, + "rewards/margins": 2.7633728981018066, + "rewards/rejected": 0.8664296269416809, + "ronpo/logit": 0.27633726596832275, + "ronpo/residual": 0.2702479362487793, + "ronpo/residual_abs": 0.30587834119796753, + "ronpo/target": 0.006089328322559595, + "ronpo/target_abs": 0.04417353868484497, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 250 + }, + { + "drift/chosen_abs": 0.44501981139183044, + "drift/rejected_abs": 0.19686473906040192, + "epoch": 0.08104886769964244, + "grad_norm": 14.0, + "learning_rate": 4.870809138015498e-08, + "logits/chosen": -2.0998764038085938, + "logits/rejected": -2.103111505508423, + "logps/chosen": -0.26382118463516235, + "logps/rejected": -0.27121174335479736, + "loss": 0.7604058384895325, + "loss/core": 0.6632653474807739, + "loss/preference_sft": 0.2692394554615021, + "loss/reference_anchor": 0.9444811940193176, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 4.449104309082031, + "rewards/margins": 2.5365662574768066, + "rewards/rejected": 1.9125378131866455, + "ronpo/logit": 0.25365665555000305, + "ronpo/residual": 0.25510507822036743, + "ronpo/residual_abs": 0.28715819120407104, + "ronpo/target": -0.0014484294224530458, + "ronpo/target_abs": 0.056101828813552856, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 255 + }, + { + "drift/chosen_abs": 0.32862967252731323, + "drift/rejected_abs": 0.08690224587917328, + "epoch": 0.08263806118394915, + "grad_norm": 976.0, + "learning_rate": 4.852940038140926e-08, + "logits/chosen": -2.1928184032440186, + "logits/rejected": -2.1674201488494873, + "logps/chosen": -0.2829035818576813, + "logps/rejected": -0.2785223722457886, + "loss": 0.8419889211654663, + "loss/core": 0.789025068283081, + "loss/preference_sft": 0.27425065636634827, + "loss/reference_anchor": 0.5022132992744446, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.285032272338867, + "rewards/margins": 2.417757034301758, + "rewards/rejected": 0.867275059223175, + "ronpo/logit": 0.24177570641040802, + "ronpo/residual": 0.23542478680610657, + "ronpo/residual_abs": 0.2721962332725525, + "ronpo/target": 0.006350924260914326, + "ronpo/target_abs": 0.055059123784303665, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 260 + }, + { + "drift/chosen_abs": 0.18202148377895355, + "drift/rejected_abs": 0.08143140375614166, + "epoch": 0.08422725466825585, + "grad_norm": 16.25, + "learning_rate": 4.833951066243004e-08, + "logits/chosen": -2.1530301570892334, + "logits/rejected": -2.1527392864227295, + "logps/chosen": -0.28495079278945923, + "logps/rejected": -0.29102346301078796, + "loss": 0.08294971287250519, + "loss/core": 0.07107045501470566, + "loss/preference_sft": 0.2882830500602722, + "loss/reference_anchor": 0.08996426314115524, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8192421197891235, + "rewards/margins": 1.0141630172729492, + "rewards/rejected": 0.8050788640975952, + "ronpo/logit": 0.10141630470752716, + "ronpo/residual": 0.10920605808496475, + "ronpo/residual_abs": 0.13682009279727936, + "ronpo/target": -0.0077897594310343266, + "ronpo/target_abs": 0.05120920389890671, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 265 + }, + { + "drift/chosen_abs": 0.2922496199607849, + "drift/rejected_abs": 0.1604980230331421, + "epoch": 0.08581644815256258, + "grad_norm": 5.625, + "learning_rate": 4.813851260043916e-08, + "logits/chosen": -2.22133469581604, + "logits/rejected": -2.1806182861328125, + "logps/chosen": -0.2800171375274658, + "logps/rejected": -0.2896736264228821, + "loss": 0.3876208961009979, + "loss/core": 0.31458601355552673, + "loss/preference_sft": 0.2803085148334503, + "loss/reference_anchor": 0.7023177146911621, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.9203429222106934, + "rewards/margins": 1.317029595375061, + "rewards/rejected": 1.6033132076263428, + "ronpo/logit": 0.13170292973518372, + "ronpo/residual": 0.12294062227010727, + "ronpo/residual_abs": 0.16244474053382874, + "ronpo/target": 0.008762339130043983, + "ronpo/target_abs": 0.058000266551971436, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 270 + }, + { + "drift/chosen_abs": 0.17858698964118958, + "drift/rejected_abs": 0.10436340421438217, + "epoch": 0.08740564163686929, + "grad_norm": 9.125, + "learning_rate": 4.79265018596281e-08, + "logits/chosen": -2.271712064743042, + "logits/rejected": -2.3456873893737793, + "logps/chosen": -0.279397577047348, + "logps/rejected": -0.26725754141807556, + "loss": 0.04921260103583336, + "loss/core": 0.03643937036395073, + "loss/preference_sft": 0.2758616507053375, + "loss/reference_anchor": 0.10014615207910538, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.785718560218811, + "rewards/margins": 0.7422192096710205, + "rewards/rejected": 1.043499231338501, + "ronpo/logit": 0.07422193139791489, + "ronpo/residual": 0.06601117551326752, + "ronpo/residual_abs": 0.10768099874258041, + "ronpo/target": 0.008210750296711922, + "ronpo/target_abs": 0.05532875657081604, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 275 + }, + { + "drift/chosen_abs": 0.169549822807312, + "drift/rejected_abs": 0.11006991565227509, + "epoch": 0.08899483512117601, + "grad_norm": 55.75, + "learning_rate": 4.770357934562703e-08, + "logits/chosen": -2.1815786361694336, + "logits/rejected": -2.182666063308716, + "logps/chosen": -0.28446492552757263, + "logps/rejected": -0.291333943605423, + "loss": 0.11231124401092529, + "loss/core": 0.09980011731386185, + "loss/preference_sft": 0.2837428152561188, + "loss/reference_anchor": 0.09673698246479034, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.6950247287750244, + "rewards/margins": 0.5955981612205505, + "rewards/rejected": 1.099426507949829, + "ronpo/logit": 0.05955982953310013, + "ronpo/residual": 0.06428845226764679, + "ronpo/residual_abs": 0.13956977427005768, + "ronpo/target": -0.004728636238723993, + "ronpo/target_abs": 0.04768453910946846, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 280 + }, + { + "drift/chosen_abs": 0.43819236755371094, + "drift/rejected_abs": 0.13878020644187927, + "epoch": 0.09058402860548272, + "grad_norm": 18.625, + "learning_rate": 4.746985115747917e-08, + "logits/chosen": -2.2233951091766357, + "logits/rejected": -2.1950364112854004, + "logps/chosen": -0.280063658952713, + "logps/rejected": -0.2928066849708557, + "loss": 1.5658351182937622, + "loss/core": 1.4394580125808716, + "loss/preference_sft": 0.2897217869758606, + "loss/reference_anchor": 1.2347972393035889, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 4.379513740539551, + "rewards/margins": 2.9923789501190186, + "rewards/rejected": 1.3871352672576904, + "ronpo/logit": 0.2992379069328308, + "ronpo/residual": 0.29981642961502075, + "ronpo/residual_abs": 0.33501166105270386, + "ronpo/target": -0.00057854043552652, + "ronpo/target_abs": 0.04776735231280327, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 285 + }, + { + "drift/chosen_abs": 0.2989239990711212, + "drift/rejected_abs": 0.13974739611148834, + "epoch": 0.09217322208978942, + "grad_norm": 136.0, + "learning_rate": 4.722542853714341e-08, + "logits/chosen": -2.012983560562134, + "logits/rejected": -2.0874452590942383, + "logps/chosen": -0.27262991666793823, + "logps/rejected": -0.28246259689331055, + "loss": 0.27830126881599426, + "loss/core": 0.2490129917860031, + "loss/preference_sft": 0.27377456426620483, + "loss/reference_anchor": 0.26550552248954773, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.9884445667266846, + "rewards/margins": 1.591313123703003, + "rewards/rejected": 1.3971314430236816, + "ronpo/logit": 0.15913130342960358, + "ronpo/residual": 0.1600702404975891, + "ronpo/residual_abs": 0.21637621521949768, + "ronpo/target": -0.0009389538317918777, + "ronpo/target_abs": 0.048052363097667694, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 290 + }, + { + "drift/chosen_abs": 0.19468945264816284, + "drift/rejected_abs": 0.07088744640350342, + "epoch": 0.09376241557409615, + "grad_norm": 192.0, + "learning_rate": 4.697042781654913e-08, + "logits/chosen": -2.1905837059020996, + "logits/rejected": -2.1836931705474854, + "logps/chosen": -0.30156785249710083, + "logps/rejected": -0.3063206672668457, + "loss": 0.2443789690732956, + "loss/core": 0.22306425869464874, + "loss/preference_sft": 0.29774826765060425, + "loss/reference_anchor": 0.18337255716323853, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.9455058574676514, + "rewards/margins": 1.2405574321746826, + "rewards/rejected": 0.7049483060836792, + "ronpo/logit": 0.12405575811862946, + "ronpo/residual": 0.11795137822628021, + "ronpo/residual_abs": 0.15991681814193726, + "ronpo/target": 0.006104385945945978, + "ronpo/target_abs": 0.05217462033033371, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 295 + }, + { + "drift/chosen_abs": 0.13699117302894592, + "drift/rejected_abs": 0.08163593709468842, + "epoch": 0.09535160905840286, + "grad_norm": 6.125, + "learning_rate": 4.6704970362228555e-08, + "logits/chosen": -2.405043840408325, + "logits/rejected": -2.4092018604278564, + "logps/chosen": -0.29529136419296265, + "logps/rejected": -0.2865511476993561, + "loss": 0.032488640397787094, + "loss/core": 0.024509349837899208, + "loss/preference_sft": 0.29015666246414185, + "loss/reference_anchor": 0.05077719688415527, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.3689161539077759, + "rewards/margins": 0.5563350915908813, + "rewards/rejected": 0.8125810623168945, + "ronpo/logit": 0.055633507668972015, + "ronpo/residual": 0.055543310940265656, + "ronpo/residual_abs": 0.09124297648668289, + "ronpo/target": 9.020241122925654e-05, + "ronpo/target_abs": 0.051511455327272415, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 300 + }, + { + "drift/chosen_abs": 0.22262239456176758, + "drift/rejected_abs": 0.10259328037500381, + "epoch": 0.09694080254270958, + "grad_norm": 1.5546875, + "learning_rate": 4.6429182517552804e-08, + "logits/chosen": -2.077086925506592, + "logits/rejected": -2.105011463165283, + "logps/chosen": -0.3044688105583191, + "logps/rejected": -0.292930543422699, + "loss": 0.2044544517993927, + "loss/core": 0.1758301556110382, + "loss/preference_sft": 0.30388787388801575, + "loss/reference_anchor": 0.25585389137268066, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.2251529693603516, + "rewards/margins": 1.2022806406021118, + "rewards/rejected": 1.0228722095489502, + "ronpo/logit": 0.12022805213928223, + "ronpo/residual": 0.11487364768981934, + "ronpo/residual_abs": 0.14453622698783875, + "ronpo/target": 0.005354395601898432, + "ronpo/target_abs": 0.05369141697883606, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 305 + }, + { + "drift/chosen_abs": 0.23092417418956757, + "drift/rejected_abs": 0.1479761302471161, + "epoch": 0.09852999602701629, + "grad_norm": 201.0, + "learning_rate": 4.614319554259933e-08, + "logits/chosen": -2.096682071685791, + "logits/rejected": -1.9904273748397827, + "logps/chosen": -0.3039776682853699, + "logps/rejected": -0.3026920557022095, + "loss": 0.10154803842306137, + "loss/core": 0.06678663194179535, + "loss/preference_sft": 0.2993779182434082, + "loss/reference_anchor": 0.3176761865615845, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.3076138496398926, + "rewards/margins": 0.8309847712516785, + "rewards/rejected": 1.4766290187835693, + "ronpo/logit": 0.08309847861528397, + "ronpo/residual": 0.0783224031329155, + "ronpo/residual_abs": 0.13604936003684998, + "ronpo/target": 0.004776070825755596, + "ronpo/target_abs": 0.05459970235824585, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 310 + }, + { + "drift/chosen_abs": 0.21857643127441406, + "drift/rejected_abs": 0.17166323959827423, + "epoch": 0.10011918951132301, + "grad_norm": 1.859375, + "learning_rate": 4.58471455516792e-08, + "logits/chosen": -2.208028793334961, + "logits/rejected": -2.2415013313293457, + "logps/chosen": -0.28857478499412537, + "logps/rejected": -0.2832663655281067, + "loss": 0.16358520090579987, + "loss/core": 0.08099617063999176, + "loss/preference_sft": 0.2877857983112335, + "loss/reference_anchor": 0.7971116304397583, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.18546724319458, + "rewards/margins": 0.4695800840854645, + "rewards/rejected": 1.7158870697021484, + "ronpo/logit": 0.04695800319314003, + "ronpo/residual": 0.04399949312210083, + "ronpo/residual_abs": 0.11905704438686371, + "ronpo/target": 0.0029585096053779125, + "ronpo/target_abs": 0.05115630105137825, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 315 + }, + { + "drift/chosen_abs": 0.30618441104888916, + "drift/rejected_abs": 0.15159662067890167, + "epoch": 0.10170838299562972, + "grad_norm": 15.6875, + "learning_rate": 4.5541173448554095e-08, + "logits/chosen": -2.0477356910705566, + "logits/rejected": -2.046473979949951, + "logps/chosen": -0.26754650473594666, + "logps/rejected": -0.26750463247299194, + "loss": 0.45313024520874023, + "loss/core": 0.3808836340904236, + "loss/preference_sft": 0.26480069756507874, + "loss/reference_anchor": 0.6959866285324097, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.0603110790252686, + "rewards/margins": 1.54562246799469, + "rewards/rejected": 1.5146887302398682, + "ronpo/logit": 0.15456224977970123, + "ronpo/residual": 0.1472359597682953, + "ronpo/residual_abs": 0.19273725152015686, + "ronpo/target": 0.007326282560825348, + "ronpo/target_abs": 0.047880660742521286, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 320 + }, + { + "drift/chosen_abs": 0.27261799573898315, + "drift/rejected_abs": 0.06441514194011688, + "epoch": 0.10329757647993643, + "grad_norm": 24.25, + "learning_rate": 4.522542485937369e-08, + "logits/chosen": -2.213456153869629, + "logits/rejected": -2.237545967102051, + "logps/chosen": -0.3028256595134735, + "logps/rejected": -0.29002341628074646, + "loss": 1.2407621145248413, + "loss/core": 1.174018144607544, + "loss/preference_sft": 0.2956632375717163, + "loss/reference_anchor": 0.6378735899925232, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.7259883880615234, + "rewards/margins": 2.0907344818115234, + "rewards/rejected": 0.6352539658546448, + "ronpo/logit": 0.20907345414161682, + "ronpo/residual": 0.21372437477111816, + "ronpo/residual_abs": 0.24546225368976593, + "ronpo/target": -0.004650958813726902, + "ronpo/target_abs": 0.04953046515583992, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 325 + }, + { + "drift/chosen_abs": 0.2498444765806198, + "drift/rejected_abs": 0.127003476023674, + "epoch": 0.10488676996424315, + "grad_norm": 11.0625, + "learning_rate": 4.490005006336554e-08, + "logits/chosen": -2.322620391845703, + "logits/rejected": -2.3220651149749756, + "logps/chosen": -0.26767295598983765, + "logps/rejected": -0.26008275151252747, + "loss": 0.614111065864563, + "loss/core": 0.5768492221832275, + "loss/preference_sft": 0.2576170861721039, + "loss/reference_anchor": 0.34685665369033813, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.4984447956085205, + "rewards/margins": 1.2349770069122314, + "rewards/rejected": 1.263467788696289, + "ronpo/logit": 0.12349770218133926, + "ronpo/residual": 0.13091148436069489, + "ronpo/residual_abs": 0.23883113265037537, + "ronpo/target": -0.007413757033646107, + "ronpo/target_abs": 0.054652709513902664, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 330 + }, + { + "drift/chosen_abs": 0.4506363868713379, + "drift/rejected_abs": 0.2148597538471222, + "epoch": 0.10647596344854986, + "grad_norm": 13.6875, + "learning_rate": 4.456520392131034e-08, + "logits/chosen": -1.9939444065093994, + "logits/rejected": -1.9420192241668701, + "logps/chosen": -0.2896828055381775, + "logps/rejected": -0.2902643084526062, + "loss": 0.8482235670089722, + "loss/core": 0.6990407705307007, + "loss/preference_sft": 0.29258525371551514, + "loss/reference_anchor": 1.4625699520111084, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.504998207092285, + "rewards/margins": 2.361280918121338, + "rewards/rejected": 2.1437175273895264, + "ronpo/logit": 0.2361280918121338, + "ronpo/residual": 0.23645956814289093, + "ronpo/residual_abs": 0.2861987054347992, + "ronpo/target": -0.00033148974762298167, + "ronpo/target_abs": 0.04662436991930008, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 335 + }, + { + "drift/chosen_abs": 0.4584285616874695, + "drift/rejected_abs": 0.16387048363685608, + "epoch": 0.10806515693285658, + "grad_norm": 85.5, + "learning_rate": 4.422104580183649e-08, + "logits/chosen": -1.8824104070663452, + "logits/rejected": -1.9405462741851807, + "logps/chosen": -0.28784456849098206, + "logps/rejected": -0.27431824803352356, + "loss": 1.0368645191192627, + "loss/core": 0.9412893056869507, + "loss/preference_sft": 0.2768869400024414, + "loss/reference_anchor": 0.9280635714530945, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.582031726837158, + "rewards/margins": 2.948826313018799, + "rewards/rejected": 1.633205771446228, + "ronpo/logit": 0.294882595539093, + "ronpo/residual": 0.2988366186618805, + "ronpo/residual_abs": 0.3257350027561188, + "ronpo/target": -0.003954009618610144, + "ronpo/target_abs": 0.047440093010663986, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 340 + }, + { + "drift/chosen_abs": 0.24579569697380066, + "drift/rejected_abs": 0.12019170820713043, + "epoch": 0.10965435041716329, + "grad_norm": 64.5, + "learning_rate": 4.38677395055693e-08, + "logits/chosen": -2.1810965538024902, + "logits/rejected": -2.252776622772217, + "logps/chosen": -0.27312830090522766, + "logps/rejected": -0.27882593870162964, + "loss": 0.1356753408908844, + "loss/core": 0.10093406587839127, + "loss/preference_sft": 0.2720806300640106, + "loss/reference_anchor": 0.3202047348022461, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.456185817718506, + "rewards/margins": 1.2865262031555176, + "rewards/rejected": 1.1696596145629883, + "ronpo/logit": 0.1286526322364807, + "ronpo/residual": 0.1406368762254715, + "ronpo/residual_abs": 0.15837253630161285, + "ronpo/target": -0.011984249576926231, + "ronpo/target_abs": 0.04884602874517441, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 345 + }, + { + "drift/chosen_abs": 0.4141915440559387, + "drift/rejected_abs": 0.16395843029022217, + "epoch": 0.11124354390147001, + "grad_norm": 1.5703125, + "learning_rate": 4.35054531871708e-08, + "logits/chosen": -2.269747257232666, + "logits/rejected": -2.2979190349578857, + "logps/chosen": -0.23346249759197235, + "logps/rejected": -0.2476530522108078, + "loss": 1.0521429777145386, + "loss/core": 0.9406593441963196, + "loss/preference_sft": 0.24042610824108124, + "loss/reference_anchor": 1.090794324874878, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.141915321350098, + "rewards/margins": 2.502330780029297, + "rewards/rejected": 1.6395843029022217, + "ronpo/logit": 0.2502330541610718, + "ronpo/residual": 0.25256186723709106, + "ronpo/residual_abs": 0.2842928171157837, + "ronpo/target": -0.002328807720914483, + "ronpo/target_abs": 0.04533926397562027, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 350 + }, + { + "drift/chosen_abs": 0.2807723879814148, + "drift/rejected_abs": 0.10302431881427765, + "epoch": 0.11283273738577672, + "grad_norm": 15.125, + "learning_rate": 4.313435927530719e-08, + "logits/chosen": -2.1268205642700195, + "logits/rejected": -2.025103807449341, + "logps/chosen": -0.2836920917034149, + "logps/rejected": -0.2762809693813324, + "loss": 0.4303739070892334, + "loss/core": 0.3964649438858032, + "loss/preference_sft": 0.2680888772010803, + "loss/reference_anchor": 0.31228065490722656, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.8077235221862793, + "rewards/margins": 1.7782020568847656, + "rewards/rejected": 1.0295217037200928, + "ronpo/logit": 0.17782019078731537, + "ronpo/residual": 0.17538082599639893, + "ronpo/residual_abs": 0.21557548642158508, + "ronpo/target": 0.0024393785279244184, + "ronpo/target_abs": 0.04851159825921059, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 355 + }, + { + "drift/chosen_abs": 0.2171783745288849, + "drift/rejected_abs": 0.08919801563024521, + "epoch": 0.11442193087008343, + "grad_norm": 11.9375, + "learning_rate": 4.2754634390582136e-08, + "logits/chosen": -2.2076804637908936, + "logits/rejected": -2.2731592655181885, + "logps/chosen": -0.26778849959373474, + "logps/rejected": -0.2578774094581604, + "loss": 0.09732436388731003, + "loss/core": 0.08528517186641693, + "loss/preference_sft": 0.2644139230251312, + "loss/reference_anchor": 0.0939505323767662, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.1707053184509277, + "rewards/margins": 1.27962327003479, + "rewards/rejected": 0.8910821676254272, + "ronpo/logit": 0.12796232104301453, + "ronpo/residual": 0.1286647766828537, + "ronpo/residual_abs": 0.1663980484008789, + "ronpo/target": -0.0007024522055871785, + "ronpo/target_abs": 0.04947920888662338, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 360 + }, + { + "drift/chosen_abs": 0.2075362503528595, + "drift/rejected_abs": 0.13140513002872467, + "epoch": 0.11601112435439015, + "grad_norm": 2.171875, + "learning_rate": 4.236645926147493e-08, + "logits/chosen": -2.0933029651641846, + "logits/rejected": -2.131420373916626, + "logps/chosen": -0.28616708517074585, + "logps/rejected": -0.3243045210838318, + "loss": 0.07761023193597794, + "loss/core": 0.036653272807598114, + "loss/preference_sft": 0.321381539106369, + "loss/reference_anchor": 0.3774314522743225, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.0736491680145264, + "rewards/margins": 0.7846614718437195, + "rewards/rejected": 1.2889881134033203, + "ronpo/logit": 0.07846614718437195, + "ronpo/residual": 0.08032883703708649, + "ronpo/residual_abs": 0.11741244792938232, + "ronpo/target": -0.0018626858945935965, + "ronpo/target_abs": 0.05729759484529495, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 365 + }, + { + "drift/chosen_abs": 0.31861263513565063, + "drift/rejected_abs": 0.2158060520887375, + "epoch": 0.11760031783869686, + "grad_norm": 36.25, + "learning_rate": 4.197001863832354e-08, + "logits/chosen": -2.1318399906158447, + "logits/rejected": -2.1656715869903564, + "logps/chosen": -0.2784048318862915, + "logps/rejected": -0.28124070167541504, + "loss": 0.22356906533241272, + "loss/core": 0.10556735098361969, + "loss/preference_sft": 0.28225523233413696, + "loss/reference_anchor": 1.1517915725708008, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.1856679916381836, + "rewards/margins": 1.0293043851852417, + "rewards/rejected": 2.1563634872436523, + "ronpo/logit": 0.10293042659759521, + "ronpo/residual": 0.09626283496618271, + "ronpo/residual_abs": 0.14562055468559265, + "ronpo/target": 0.006667587906122208, + "ronpo/target_abs": 0.05160321667790413, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 370 + }, + { + "drift/chosen_abs": 0.21357055008411407, + "drift/rejected_abs": 0.0986676812171936, + "epoch": 0.11918951132300358, + "grad_norm": 18.875, + "learning_rate": 4.156550120539344e-08, + "logits/chosen": -2.2781190872192383, + "logits/rejected": -2.3018202781677246, + "logps/chosen": -0.2936388850212097, + "logps/rejected": -0.2881937623023987, + "loss": 0.13922074437141418, + "loss/core": 0.12170878797769547, + "loss/preference_sft": 0.28684449195861816, + "loss/reference_anchor": 0.14643503725528717, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.1355700492858887, + "rewards/margins": 1.1490267515182495, + "rewards/rejected": 0.9865430593490601, + "ronpo/logit": 0.11490269005298615, + "ronpo/residual": 0.10873659700155258, + "ronpo/residual_abs": 0.14544065296649933, + "ronpo/target": 0.006166078150272369, + "ronpo/target_abs": 0.05552104860544205, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 375 + }, + { + "drift/chosen_abs": 0.2752930521965027, + "drift/rejected_abs": 0.10053251683712006, + "epoch": 0.12077870480731029, + "grad_norm": 1.046875, + "learning_rate": 4.1153099491074094e-08, + "logits/chosen": -2.1597490310668945, + "logits/rejected": -2.1769750118255615, + "logps/chosen": -0.28902846574783325, + "logps/rejected": -0.282781720161438, + "loss": 0.529904842376709, + "loss/core": 0.4918580651283264, + "loss/preference_sft": 0.28759074211120605, + "loss/reference_anchor": 0.3517104983329773, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.750469207763672, + "rewards/margins": 1.7569026947021484, + "rewards/rejected": 0.993566632270813, + "ronpo/logit": 0.17569026350975037, + "ronpo/residual": 0.17634570598602295, + "ronpo/residual_abs": 0.20597271621227264, + "ronpo/target": -0.000655452604405582, + "ronpo/target_abs": 0.04808509349822998, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 380 + }, + { + "drift/chosen_abs": 0.25277113914489746, + "drift/rejected_abs": 0.1480860561132431, + "epoch": 0.12236789829161701, + "grad_norm": 61.25, + "learning_rate": 4.073300977624594e-08, + "logits/chosen": -2.277191162109375, + "logits/rejected": -2.2477023601531982, + "logps/chosen": -0.27077725529670715, + "logps/rejected": -0.26460614800453186, + "loss": 0.1768111288547516, + "loss/core": 0.151821106672287, + "loss/preference_sft": 0.2671385705471039, + "loss/reference_anchor": 0.22318613529205322, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.526538133621216, + "rewards/margins": 1.048628330230713, + "rewards/rejected": 1.477910041809082, + "ronpo/logit": 0.10486283153295517, + "ronpo/residual": 0.09956089407205582, + "ronpo/residual_abs": 0.1376153975725174, + "ronpo/target": 0.005301934666931629, + "ronpo/target_abs": 0.05118630453944206, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 385 + }, + { + "drift/chosen_abs": 0.21099695563316345, + "drift/rejected_abs": 0.12248991429805756, + "epoch": 0.12395709177592372, + "grad_norm": 3.078125, + "learning_rate": 4.030543200086123e-08, + "logits/chosen": -2.0786709785461426, + "logits/rejected": -2.1199498176574707, + "logps/chosen": -0.3045576214790344, + "logps/rejected": -0.29476088285446167, + "loss": 0.0992443710565567, + "loss/core": 0.07167808711528778, + "loss/preference_sft": 0.29941990971565247, + "loss/reference_anchor": 0.24572083353996277, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.1079843044281006, + "rewards/margins": 0.8853281140327454, + "rewards/rejected": 1.222656011581421, + "ronpo/logit": 0.08853280544281006, + "ronpo/residual": 0.0776071846485138, + "ronpo/residual_abs": 0.12177129089832306, + "ronpo/target": 0.01092563010752201, + "ronpo/target_abs": 0.04902078956365585, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 390 + }, + { + "drift/chosen_abs": 0.21322067081928253, + "drift/rejected_abs": 0.06863544136285782, + "epoch": 0.12554628526023043, + "grad_norm": 3.21875, + "learning_rate": 3.987056966878353e-08, + "logits/chosen": -2.158668041229248, + "logits/rejected": -2.221909999847412, + "logps/chosen": -0.31034913659095764, + "logps/rejected": -0.29723021388053894, + "loss": 0.3569168448448181, + "loss/core": 0.33192580938339233, + "loss/preference_sft": 0.28455764055252075, + "loss/reference_anchor": 0.22145459055900574, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.131547689437866, + "rewards/margins": 1.4457036256790161, + "rewards/rejected": 0.6858440637588501, + "ronpo/logit": 0.14457036554813385, + "ronpo/residual": 0.1422502100467682, + "ronpo/residual_abs": 0.1726013422012329, + "ronpo/target": 0.002320161322131753, + "ronpo/target_abs": 0.0495462641119957, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 395 + }, + { + "drift/chosen_abs": 0.1918472945690155, + "drift/rejected_abs": 0.08430652320384979, + "epoch": 0.12713547874453715, + "grad_norm": 10.5625, + "learning_rate": 3.9428629750930845e-08, + "logits/chosen": -2.2298977375030518, + "logits/rejected": -2.319603443145752, + "logps/chosen": -0.2567993700504303, + "logps/rejected": -0.26063793897628784, + "loss": 0.07930155098438263, + "loss/core": 0.06460737437009811, + "loss/preference_sft": 0.254304975271225, + "loss/reference_anchor": 0.12151118367910385, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 1.9175297021865845, + "rewards/margins": 1.078814148902893, + "rewards/rejected": 0.8387155532836914, + "ronpo/logit": 0.10788138955831528, + "ronpo/residual": 0.11212491989135742, + "ronpo/residual_abs": 0.1347557008266449, + "ronpo/target": -0.004243509843945503, + "ronpo/target_abs": 0.048654936254024506, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 400 + }, + { + "drift/chosen_abs": 0.17124497890472412, + "drift/rejected_abs": 0.1462007462978363, + "epoch": 0.12872467222884387, + "grad_norm": 2.515625, + "learning_rate": 3.8979822586768666e-08, + "logits/chosen": -2.290794610977173, + "logits/rejected": -2.2985095977783203, + "logps/chosen": -0.2945011258125305, + "logps/rejected": -0.2886204123497009, + "loss": 0.30920687317848206, + "loss/core": 0.2832431197166443, + "loss/preference_sft": 0.2873281240463257, + "loss/reference_anchor": 0.23090443015098572, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.7103097438812256, + "rewards/margins": 0.24959950149059296, + "rewards/rejected": 1.4607102870941162, + "ronpo/logit": 0.024959946051239967, + "ronpo/residual": 0.02641598880290985, + "ronpo/residual_abs": 0.16461867094039917, + "ronpo/target": -0.0014560356503352523, + "ronpo/target_abs": 0.055648498237133026, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 405 + }, + { + "drift/chosen_abs": 0.1872716248035431, + "drift/rejected_abs": 0.057905036956071854, + "epoch": 0.13031386571315057, + "grad_norm": 0.67578125, + "learning_rate": 3.852436178419985e-08, + "logits/chosen": -2.2279515266418457, + "logits/rejected": -2.141326904296875, + "logps/chosen": -0.28925570845603943, + "logps/rejected": -0.28760525584220886, + "loss": 0.3526923656463623, + "loss/core": 0.33196255564689636, + "loss/preference_sft": 0.28182536363601685, + "loss/reference_anchor": 0.17911583185195923, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.870459794998169, + "rewards/margins": 1.3097550868988037, + "rewards/rejected": 0.5607047080993652, + "ronpo/logit": 0.13097552955150604, + "ronpo/residual": 0.13209015130996704, + "ronpo/residual_abs": 0.16732732951641083, + "ronpo/target": -0.001114630838856101, + "ronpo/target_abs": 0.05251222103834152, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 410 + }, + { + "drift/chosen_abs": 0.12823328375816345, + "drift/rejected_abs": 0.05924879387021065, + "epoch": 0.1319030591974573, + "grad_norm": 18.375, + "learning_rate": 3.806246411789872e-08, + "logits/chosen": -2.2676451206207275, + "logits/rejected": -2.2973361015319824, + "logps/chosen": -0.2848006784915924, + "logps/rejected": -0.27459055185317993, + "loss": 0.040306154638528824, + "loss/core": 0.033028461039066315, + "loss/preference_sft": 0.2755371332168579, + "loss/reference_anchor": 0.04522323980927467, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.2822843790054321, + "rewards/margins": 0.6960135698318481, + "rewards/rejected": 0.5862708687782288, + "ronpo/logit": 0.06960136443376541, + "ronpo/residual": 0.0626053735613823, + "ronpo/residual_abs": 0.09992660582065582, + "ronpo/target": 0.006995986215770245, + "ronpo/target_abs": 0.05004571005702019, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 415 + }, + { + "drift/chosen_abs": 0.387541264295578, + "drift/rejected_abs": 0.1291671097278595, + "epoch": 0.133492252681764, + "grad_norm": 3.625, + "learning_rate": 3.7594349426138154e-08, + "logits/chosen": -2.113781452178955, + "logits/rejected": -2.233093738555908, + "logps/chosen": -0.2736510634422302, + "logps/rejected": -0.27200594544410706, + "loss": 0.9080482721328735, + "loss/core": 0.8385866284370422, + "loss/preference_sft": 0.2741411626338959, + "loss/reference_anchor": 0.667202353477478, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.875412702560425, + "rewards/margins": 2.708035945892334, + "rewards/rejected": 1.1673767566680908, + "ronpo/logit": 0.27080363035202026, + "ronpo/residual": 0.2666502892971039, + "ronpo/residual_abs": 0.3021991550922394, + "ronpo/target": 0.004153319634497166, + "ronpo/target_abs": 0.04889250919222832, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 420 + }, + { + "drift/chosen_abs": 0.42560648918151855, + "drift/rejected_abs": 0.13950805366039276, + "epoch": 0.1350814461660707, + "grad_norm": 304.0, + "learning_rate": 3.712024050615843e-08, + "logits/chosen": -2.164858341217041, + "logits/rejected": -2.1596226692199707, + "logps/chosen": -0.2971101403236389, + "logps/rejected": -0.29576635360717773, + "loss": 1.2745723724365234, + "loss/core": 1.1691445112228394, + "loss/preference_sft": 0.28978782892227173, + "loss/reference_anchor": 1.0252996683120728, + "rewards/accuracies": 0.9375, + "rewards/chosen": 4.2549614906311035, + "rewards/margins": 2.861567735671997, + "rewards/rejected": 1.393393874168396, + "ronpo/logit": 0.2861567735671997, + "ronpo/residual": 0.288315087556839, + "ronpo/residual_abs": 0.3225054144859314, + "ronpo/target": -0.0021583049092441797, + "ronpo/target_abs": 0.05234162136912346, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 425 + }, + { + "drift/chosen_abs": 0.1872471123933792, + "drift/rejected_abs": 0.09143074601888657, + "epoch": 0.13667063965037743, + "grad_norm": 11.625, + "learning_rate": 3.6640363008127786e-08, + "logits/chosen": -2.432072162628174, + "logits/rejected": -2.4184775352478027, + "logps/chosen": -0.2796880304813385, + "logps/rejected": -0.27485981583595276, + "loss": 0.1038626879453659, + "loss/core": 0.08703657239675522, + "loss/preference_sft": 0.2725329101085663, + "loss/reference_anchor": 0.1410079151391983, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.8717552423477173, + "rewards/margins": 0.9586809277534485, + "rewards/rejected": 0.9130741357803345, + "ronpo/logit": 0.09586808830499649, + "ronpo/residual": 0.08066330850124359, + "ronpo/residual_abs": 0.12503182888031006, + "ronpo/target": 0.01520479004830122, + "ronpo/target_abs": 0.049617547541856766, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 430 + }, + { + "drift/chosen_abs": 0.17280510067939758, + "drift/rejected_abs": 0.09314249455928802, + "epoch": 0.13825983313468415, + "grad_norm": 12.1875, + "learning_rate": 3.615494532774522e-08, + "logits/chosen": -2.1827242374420166, + "logits/rejected": -2.1762850284576416, + "logps/chosen": -0.280172199010849, + "logps/rejected": -0.28061169385910034, + "loss": 0.043207619339227676, + "loss/core": 0.03257911652326584, + "loss/preference_sft": 0.27778077125549316, + "loss/reference_anchor": 0.07850693166255951, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.7262004613876343, + "rewards/margins": 0.7954661846160889, + "rewards/rejected": 0.9307343363761902, + "ronpo/logit": 0.07954661548137665, + "ronpo/residual": 0.06194382905960083, + "ronpo/residual_abs": 0.10705231130123138, + "ronpo/target": 0.01760278083384037, + "ronpo/target_abs": 0.04770347476005554, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 435 + }, + { + "drift/chosen_abs": 0.3187355399131775, + "drift/rejected_abs": 0.13290651142597198, + "epoch": 0.13984902661899087, + "grad_norm": 4.125, + "learning_rate": 3.566421849753646e-08, + "logits/chosen": -2.2726149559020996, + "logits/rejected": -2.297689437866211, + "logps/chosen": -0.2947022020816803, + "logps/rejected": -0.2913546562194824, + "loss": 0.7720416784286499, + "loss/core": 0.6805315017700195, + "loss/preference_sft": 0.300406277179718, + "loss/reference_anchor": 0.8850604295730591, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.1873040199279785, + "rewards/margins": 1.865846872329712, + "rewards/rejected": 1.3214576244354248, + "ronpo/logit": 0.18658466637134552, + "ronpo/residual": 0.18800434470176697, + "ronpo/residual_abs": 0.21804949641227722, + "ronpo/target": -0.0014196861302480102, + "ronpo/target_abs": 0.051555730402469635, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 440 + }, + { + "drift/chosen_abs": 0.190468430519104, + "drift/rejected_abs": 0.1678762435913086, + "epoch": 0.14143822010329757, + "grad_norm": 300.0, + "learning_rate": 3.516841607689501e-08, + "logits/chosen": -2.2291038036346436, + "logits/rejected": -2.1525654792785645, + "logps/chosen": -0.2803241014480591, + "logps/rejected": -0.27273237705230713, + "loss": 0.2623751163482666, + "loss/core": 0.23482990264892578, + "loss/preference_sft": 0.27626365423202515, + "loss/reference_anchor": 0.24782554805278778, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.9043534994125366, + "rewards/margins": 0.22563567757606506, + "rewards/rejected": 1.678717851638794, + "ronpo/logit": 0.022563569247722626, + "ronpo/residual": 0.02049192786216736, + "ronpo/residual_abs": 0.18030428886413574, + "ronpo/target": 0.002071630908176303, + "ronpo/target_abs": 0.052827440202236176, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 445 + }, + { + "drift/chosen_abs": 0.3158915638923645, + "drift/rejected_abs": 0.10503937304019928, + "epoch": 0.1430274135876043, + "grad_norm": 30.25, + "learning_rate": 3.4667774040920516e-08, + "logits/chosen": -2.262582778930664, + "logits/rejected": -2.3268892765045166, + "logps/chosen": -0.27218812704086304, + "logps/rejected": -0.2808297276496887, + "loss": 0.7278597950935364, + "loss/core": 0.6705979704856873, + "loss/preference_sft": 0.26000097393989563, + "loss/reference_anchor": 0.5466181039810181, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.158046007156372, + "rewards/margins": 2.112226963043213, + "rewards/rejected": 1.0458192825317383, + "ronpo/logit": 0.21122269332408905, + "ronpo/residual": 0.22126352787017822, + "ronpo/residual_abs": 0.25051742792129517, + "ronpo/target": -0.010040839202702045, + "ronpo/target_abs": 0.054663896560668945, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 450 + }, + { + "drift/chosen_abs": 0.20814290642738342, + "drift/rejected_abs": 0.09922685474157333, + "epoch": 0.144616607071911, + "grad_norm": 11.1875, + "learning_rate": 3.416253066810743e-08, + "logits/chosen": -2.3161652088165283, + "logits/rejected": -2.3485805988311768, + "logps/chosen": -0.28402179479599, + "logps/rejected": -0.2825237512588501, + "loss": 0.11367638409137726, + "loss/core": 0.09682723134756088, + "loss/preference_sft": 0.28498321771621704, + "loss/reference_anchor": 0.13999313116073608, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.0814290046691895, + "rewards/margins": 1.0895073413848877, + "rewards/rejected": 0.9919217228889465, + "ronpo/logit": 0.10895073413848877, + "ronpo/residual": 0.11198439449071884, + "ronpo/residual_abs": 0.1464233696460724, + "ronpo/target": -0.00303367106243968, + "ronpo/target_abs": 0.04594602435827255, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 455 + }, + { + "drift/chosen_abs": 0.3775826096534729, + "drift/rejected_abs": 0.16409118473529816, + "epoch": 0.1462058005562177, + "grad_norm": 119.0, + "learning_rate": 3.365292642693732e-08, + "logits/chosen": -1.8064380884170532, + "logits/rejected": -1.66707444190979, + "logps/chosen": -0.28099215030670166, + "logps/rejected": -0.3002210259437561, + "loss": 0.5818950533866882, + "loss/core": 0.5199607610702515, + "loss/preference_sft": 0.2856101095676422, + "loss/reference_anchor": 0.5907815098762512, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.770456314086914, + "rewards/margins": 2.13781476020813, + "rewards/rejected": 1.6326417922973633, + "ronpo/logit": 0.2137814462184906, + "ronpo/residual": 0.21489810943603516, + "ronpo/residual_abs": 0.2495894730091095, + "ronpo/target": -0.0011166559997946024, + "ronpo/target_abs": 0.05269808694720268, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 460 + }, + { + "drift/chosen_abs": 0.3443106412887573, + "drift/rejected_abs": 0.23541593551635742, + "epoch": 0.14779499404052443, + "grad_norm": 366.0, + "learning_rate": 3.313920386142892e-08, + "logits/chosen": -2.094979763031006, + "logits/rejected": -2.115394115447998, + "logps/chosen": -0.26985684037208557, + "logps/rejected": -0.2721291780471802, + "loss": 0.859258770942688, + "loss/core": 0.7965814471244812, + "loss/preference_sft": 0.26244303584098816, + "loss/reference_anchor": 0.6005281209945679, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.441904067993164, + "rewards/margins": 1.089691400527954, + "rewards/rejected": 2.35221266746521, + "ronpo/logit": 0.10896915197372437, + "ronpo/residual": 0.10788600146770477, + "ronpo/residual_abs": 0.293454110622406, + "ronpo/target": 0.0010831309482455254, + "ronpo/target_abs": 0.052142005413770676, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 465 + }, + { + "drift/chosen_abs": 0.3005021810531616, + "drift/rejected_abs": 0.11847560107707977, + "epoch": 0.14938418752483115, + "grad_norm": 2.921875, + "learning_rate": 3.262160747570027e-08, + "logits/chosen": -2.0722744464874268, + "logits/rejected": -2.0413122177124023, + "logps/chosen": -0.253449410200119, + "logps/rejected": -0.2583273947238922, + "loss": 0.804643988609314, + "loss/core": 0.7490277886390686, + "loss/preference_sft": 0.2524202764034271, + "loss/reference_anchor": 0.5309203267097473, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.0032966136932373, + "rewards/margins": 1.820664644241333, + "rewards/rejected": 1.1826322078704834, + "ronpo/logit": 0.18206645548343658, + "ronpo/residual": 0.18194696307182312, + "ronpo/residual_abs": 0.2314409464597702, + "ronpo/target": 0.00011948794417548925, + "ronpo/target_abs": 0.04965994879603386, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 470 + }, + { + "drift/chosen_abs": 0.25074881315231323, + "drift/rejected_abs": 0.0969192162156105, + "epoch": 0.15097338100913787, + "grad_norm": 7.875, + "learning_rate": 3.210038361759807e-08, + "logits/chosen": -2.196094512939453, + "logits/rejected": -2.1309614181518555, + "logps/chosen": -0.3051135540008545, + "logps/rejected": -0.2989034950733185, + "loss": 0.2300875186920166, + "loss/core": 0.21017341315746307, + "loss/preference_sft": 0.30538254976272583, + "loss/reference_anchor": 0.16860318183898926, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.5062077045440674, + "rewards/margins": 1.53762686252594, + "rewards/rejected": 0.9685808420181274, + "ronpo/logit": 0.15376266837120056, + "ronpo/residual": 0.15053051710128784, + "ronpo/residual_abs": 0.19873568415641785, + "ronpo/target": 0.0032321817707270384, + "ronpo/target_abs": 0.05262786149978638, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 475 + }, + { + "drift/chosen_abs": 0.22192128002643585, + "drift/rejected_abs": 0.109128437936306, + "epoch": 0.15256257449344457, + "grad_norm": 1.515625, + "learning_rate": 3.157578036144937e-08, + "logits/chosen": -2.1180901527404785, + "logits/rejected": -2.140153169631958, + "logps/chosen": -0.27319201827049255, + "logps/rejected": -0.2684481739997864, + "loss": 0.08363901823759079, + "loss/core": 0.06819086521863937, + "loss/preference_sft": 0.2708436846733093, + "loss/reference_anchor": 0.12739713490009308, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.2160263061523438, + "rewards/margins": 1.133367896080017, + "rewards/rejected": 1.0826584100723267, + "ronpo/logit": 0.11333680152893066, + "ronpo/residual": 0.1177079826593399, + "ronpo/residual_abs": 0.14758066833019257, + "ronpo/target": -0.004371200688183308, + "ronpo/target_abs": 0.050663966685533524, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 480 + }, + { + "drift/chosen_abs": 0.29827818274497986, + "drift/rejected_abs": 0.16067150235176086, + "epoch": 0.1541517679777513, + "grad_norm": 6.59375, + "learning_rate": 3.104804738999169e-08, + "logits/chosen": -1.9035288095474243, + "logits/rejected": -2.026747226715088, + "logps/chosen": -0.28207504749298096, + "logps/rejected": -0.27979451417922974, + "loss": 0.14665299654006958, + "loss/core": 0.11166276037693024, + "loss/preference_sft": 0.2819157540798187, + "loss/reference_anchor": 0.32171088457107544, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.982020139694214, + "rewards/margins": 1.3762387037277222, + "rewards/rejected": 1.6057815551757812, + "ronpo/logit": 0.1376238763332367, + "ronpo/residual": 0.13066348433494568, + "ronpo/residual_abs": 0.17018786072731018, + "ronpo/target": 0.006960384547710419, + "ronpo/target_abs": 0.05105891078710556, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 485 + }, + { + "drift/chosen_abs": 0.368802547454834, + "drift/rejected_abs": 0.15182280540466309, + "epoch": 0.15574096146205801, + "grad_norm": 288.0, + "learning_rate": 3.0517435875537536e-08, + "logits/chosen": -2.1045501232147217, + "logits/rejected": -2.08597993850708, + "logps/chosen": -0.2816130220890045, + "logps/rejected": -0.2822210490703583, + "loss": 0.7077787518501282, + "loss/core": 0.6370621919631958, + "loss/preference_sft": 0.28190574049949646, + "loss/reference_anchor": 0.6789735555648804, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.685183048248291, + "rewards/margins": 2.1684653759002686, + "rewards/rejected": 1.5167176723480225, + "ronpo/logit": 0.2168465107679367, + "ronpo/residual": 0.20779669284820557, + "ronpo/residual_abs": 0.25034013390541077, + "ronpo/target": 0.009049828164279461, + "ronpo/target_abs": 0.05838625505566597, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 490 + }, + { + "drift/chosen_abs": 0.21987831592559814, + "drift/rejected_abs": 0.18404293060302734, + "epoch": 0.1573301549463647, + "grad_norm": 496.0, + "learning_rate": 2.998419836042993e-08, + "logits/chosen": -2.125863552093506, + "logits/rejected": -2.0966343879699707, + "logps/chosen": -0.28703004121780396, + "logps/rejected": -0.2746201157569885, + "loss": 1.2086600065231323, + "loss/core": 1.1395339965820312, + "loss/preference_sft": 0.2791707217693329, + "loss/reference_anchor": 0.6633409261703491, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.197369337081909, + "rewards/margins": 0.36156225204467773, + "rewards/rejected": 1.8358070850372314, + "ronpo/logit": 0.03615620732307434, + "ronpo/residual": 0.03486531972885132, + "ronpo/residual_abs": 0.2834121584892273, + "ronpo/target": 0.0012908976059406996, + "ronpo/target_abs": 0.05236129090189934, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 495 + }, + { + "drift/chosen_abs": 0.24376916885375977, + "drift/rejected_abs": 0.1475829780101776, + "epoch": 0.15891934843067143, + "grad_norm": 4.6875, + "learning_rate": 2.9448588636846045e-08, + "logits/chosen": -2.2326369285583496, + "logits/rejected": -2.090822696685791, + "logps/chosen": -0.28119930624961853, + "logps/rejected": -0.2759275734424591, + "loss": 0.24097900092601776, + "loss/core": 0.20966272056102753, + "loss/preference_sft": 0.2712128758430481, + "loss/reference_anchor": 0.28604134917259216, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.4376919269561768, + "rewards/margins": 0.9774544835090637, + "rewards/rejected": 1.4602375030517578, + "ronpo/logit": 0.09774543344974518, + "ronpo/residual": 0.09335900098085403, + "ronpo/residual_abs": 0.19132252037525177, + "ronpo/target": 0.004386442247778177, + "ronpo/target_abs": 0.05662950128316879, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 500 + }, + { + "drift/chosen_abs": 0.16036273539066315, + "drift/rejected_abs": 0.106539785861969, + "epoch": 0.16050854191497815, + "grad_norm": 4.46875, + "learning_rate": 2.8910861626005773e-08, + "logits/chosen": -2.283858299255371, + "logits/rejected": -2.20324444770813, + "logps/chosen": -0.2959023714065552, + "logps/rejected": -0.28080320358276367, + "loss": 0.032736729830503464, + "loss/core": 0.022858766838908195, + "loss/preference_sft": 0.28779202699661255, + "loss/reference_anchor": 0.07000039517879486, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.6006122827529907, + "rewards/margins": 0.5359739661216736, + "rewards/rejected": 1.064638376235962, + "ronpo/logit": 0.053597401827573776, + "ronpo/residual": 0.04705274850130081, + "ronpo/residual_abs": 0.09807590395212173, + "ronpo/target": 0.006544650532305241, + "ronpo/target_abs": 0.05331002548336983, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 505 + }, + { + "drift/chosen_abs": 0.31569427251815796, + "drift/rejected_abs": 0.16049371659755707, + "epoch": 0.16209773539928488, + "grad_norm": 2.625, + "learning_rate": 2.8371273256843075e-08, + "logits/chosen": -1.9404888153076172, + "logits/rejected": -1.963454008102417, + "logps/chosen": -0.2612414062023163, + "logps/rejected": -0.2635475993156433, + "loss": 0.46916890144348145, + "loss/core": 0.4046415686607361, + "loss/preference_sft": 0.2662178874015808, + "loss/reference_anchor": 0.6186517477035522, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.1566901206970215, + "rewards/margins": 1.5550490617752075, + "rewards/rejected": 1.601641058921814, + "ronpo/logit": 0.15550491213798523, + "ronpo/residual": 0.15648245811462402, + "ronpo/residual_abs": 0.18898537755012512, + "ronpo/target": -0.0009775443468242884, + "ronpo/target_abs": 0.04822375625371933, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 510 + }, + { + "drift/chosen_abs": 0.2568056285381317, + "drift/rejected_abs": 0.15386393666267395, + "epoch": 0.16368692888359157, + "grad_norm": 5.0, + "learning_rate": 2.7830080344197672e-08, + "logits/chosen": -2.1424307823181152, + "logits/rejected": -2.210108995437622, + "logps/chosen": -0.290412038564682, + "logps/rejected": -0.2754800617694855, + "loss": 0.3476158082485199, + "loss/core": 0.317764550447464, + "loss/preference_sft": 0.2852453589439392, + "loss/reference_anchor": 0.26998820900917053, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.568056344985962, + "rewards/margins": 1.0325605869293213, + "rewards/rejected": 1.5354955196380615, + "ronpo/logit": 0.10325608402490616, + "ronpo/residual": 0.1111401692032814, + "ronpo/residual_abs": 0.21046268939971924, + "ronpo/target": -0.007884078659117222, + "ronpo/target_abs": 0.05043824389576912, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 515 + }, + { + "drift/chosen_abs": 0.21701356768608093, + "drift/rejected_abs": 0.10474522411823273, + "epoch": 0.1652761223678983, + "grad_norm": 3.859375, + "learning_rate": 2.7287540466585063e-08, + "logits/chosen": -2.1483113765716553, + "logits/rejected": -2.1584086418151855, + "logps/chosen": -0.29263603687286377, + "logps/rejected": -0.30166834592819214, + "loss": 0.28205159306526184, + "loss/core": 0.22824864089488983, + "loss/preference_sft": 0.2935117483139038, + "loss/reference_anchor": 0.508678138256073, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.169997215270996, + "rewards/margins": 1.130325436592102, + "rewards/rejected": 1.0396720170974731, + "ronpo/logit": 0.11303253471851349, + "ronpo/residual": 0.10396580398082733, + "ronpo/residual_abs": 0.13924115896224976, + "ronpo/target": 0.009066728875041008, + "ronpo/target_abs": 0.052173011004924774, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 520 + }, + { + "drift/chosen_abs": 0.22040951251983643, + "drift/rejected_abs": 0.0650288462638855, + "epoch": 0.16686531585220502, + "grad_norm": 7.59375, + "learning_rate": 2.6743911843603128e-08, + "logits/chosen": -1.9065526723861694, + "logits/rejected": -2.050851583480835, + "logps/chosen": -0.31132742762565613, + "logps/rejected": -0.3201202154159546, + "loss": 0.2915444076061249, + "loss/core": 0.2698300778865814, + "loss/preference_sft": 0.3184563219547272, + "loss/reference_anchor": 0.1852976679801941, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.2039241790771484, + "rewards/margins": 1.5547595024108887, + "rewards/rejected": 0.6491645574569702, + "ronpo/logit": 0.15547595918178558, + "ronpo/residual": 0.1664518564939499, + "ronpo/residual_abs": 0.18785083293914795, + "ronpo/target": -0.010975899174809456, + "ronpo/target_abs": 0.049125999212265015, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 525 + }, + { + "drift/chosen_abs": 0.1532839685678482, + "drift/rejected_abs": 0.06518113613128662, + "epoch": 0.1684545093365117, + "grad_norm": 1.0390625, + "learning_rate": 2.6199453213033594e-08, + "logits/chosen": -2.3217177391052246, + "logits/rejected": -2.3166098594665527, + "logps/chosen": -0.2668350636959076, + "logps/rejected": -0.26081985235214233, + "loss": 0.05902089923620224, + "loss/core": 0.05048280954360962, + "loss/preference_sft": 0.2698516249656677, + "loss/reference_anchor": 0.05839569494128227, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 1.5320897102355957, + "rewards/margins": 0.8829070329666138, + "rewards/rejected": 0.6491826772689819, + "ronpo/logit": 0.08829069137573242, + "ronpo/residual": 0.08748365938663483, + "ronpo/residual_abs": 0.11516772210597992, + "ronpo/target": 0.0008070375770330429, + "ronpo/target_abs": 0.05328748747706413, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 530 + }, + { + "drift/chosen_abs": 0.3506208658218384, + "drift/rejected_abs": 0.1504441797733307, + "epoch": 0.17004370282081843, + "grad_norm": 3.34375, + "learning_rate": 2.5654423707696833e-08, + "logits/chosen": -2.242438793182373, + "logits/rejected": -2.2580552101135254, + "logps/chosen": -0.3120534420013428, + "logps/rejected": -0.3226138949394226, + "loss": 1.6565309762954712, + "loss/core": 1.5435130596160889, + "loss/preference_sft": 0.32026439905166626, + "loss/reference_anchor": 1.0981547832489014, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.5062084197998047, + "rewards/margins": 2.2400765419006348, + "rewards/rejected": 1.2661319971084595, + "ronpo/logit": 0.22400769591331482, + "ronpo/residual": 0.24428728222846985, + "ronpo/residual_abs": 0.32966747879981995, + "ronpo/target": -0.02027963474392891, + "ronpo/target_abs": 0.054383207112550735, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 535 + }, + { + "drift/chosen_abs": 0.2452022284269333, + "drift/rejected_abs": 0.10782649368047714, + "epoch": 0.17163289630512515, + "grad_norm": 13.0, + "learning_rate": 2.5109082732118664e-08, + "logits/chosen": -2.201077938079834, + "logits/rejected": -2.2382140159606934, + "logps/chosen": -0.2791464030742645, + "logps/rejected": -0.2900802791118622, + "loss": 0.23707738518714905, + "loss/core": 0.197381854057312, + "loss/preference_sft": 0.290230393409729, + "loss/reference_anchor": 0.36793252825737, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.4503917694091797, + "rewards/margins": 1.3725776672363281, + "rewards/rejected": 1.0778138637542725, + "ronpo/logit": 0.13725778460502625, + "ronpo/residual": 0.14476758241653442, + "ronpo/residual_abs": 0.17167086899280548, + "ronpo/target": -0.007509827613830566, + "ronpo/target_abs": 0.04967178776860237, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 540 + }, + { + "drift/chosen_abs": 0.398145467042923, + "drift/rejected_abs": 0.24695797264575958, + "epoch": 0.17322208978943188, + "grad_norm": 5.3125, + "learning_rate": 2.456368983906791e-08, + "logits/chosen": -1.9901946783065796, + "logits/rejected": -2.0143864154815674, + "logps/chosen": -0.261131227016449, + "logps/rejected": -0.27835986018180847, + "loss": 0.4815913736820221, + "loss/core": 0.3962658643722534, + "loss/preference_sft": 0.27543601393699646, + "loss/reference_anchor": 0.8257115483283997, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.9795594215393066, + "rewards/margins": 1.5161901712417603, + "rewards/rejected": 2.463369369506836, + "ronpo/logit": 0.15161903202533722, + "ronpo/residual": 0.14838778972625732, + "ronpo/residual_abs": 0.2382703274488449, + "ronpo/target": 0.003231218084692955, + "ronpo/target_abs": 0.047506727278232574, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 545 + }, + { + "drift/chosen_abs": 0.275449275970459, + "drift/rejected_abs": 0.13889315724372864, + "epoch": 0.17481128327373857, + "grad_norm": 1.390625, + "learning_rate": 2.401850460602329e-08, + "logits/chosen": -2.1420063972473145, + "logits/rejected": -2.098811149597168, + "logps/chosen": -0.25522199273109436, + "logps/rejected": -0.2641642689704895, + "loss": 0.3213452696800232, + "loss/core": 0.28511881828308105, + "loss/preference_sft": 0.26377683877944946, + "loss/reference_anchor": 0.33588656783103943, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.7539238929748535, + "rewards/margins": 1.3655030727386475, + "rewards/rejected": 1.388420820236206, + "ronpo/logit": 0.13655030727386475, + "ronpo/residual": 0.1357690542936325, + "ronpo/residual_abs": 0.1875356137752533, + "ronpo/target": 0.0007812596159055829, + "ronpo/target_abs": 0.05187007039785385, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 550 + }, + { + "drift/chosen_abs": 0.12571373581886292, + "drift/rejected_abs": 0.055512648075819016, + "epoch": 0.1764004767580453, + "grad_norm": 0.447265625, + "learning_rate": 2.3473786511628573e-08, + "logits/chosen": -2.2189087867736816, + "logits/rejected": -2.261380434036255, + "logps/chosen": -0.31099390983581543, + "logps/rejected": -0.3084818720817566, + "loss": 0.0710396021604538, + "loss/core": 0.06051533669233322, + "loss/preference_sft": 0.30863305926322937, + "loss/reference_anchor": 0.0743793398141861, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.2571372985839844, + "rewards/margins": 0.7021911144256592, + "rewards/rejected": 0.5549461245536804, + "ronpo/logit": 0.07021911442279816, + "ronpo/residual": 0.07059456408023834, + "ronpo/residual_abs": 0.11249272525310516, + "ronpo/target": -0.0003754418285097927, + "ronpo/target_abs": 0.05336303636431694, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 555 + }, + { + "drift/chosen_abs": 0.2654932141304016, + "drift/rejected_abs": 0.12235883623361588, + "epoch": 0.17798967024235202, + "grad_norm": 16.875, + "learning_rate": 2.2929794812194894e-08, + "logits/chosen": -2.0882561206817627, + "logits/rejected": -2.0949742794036865, + "logps/chosen": -0.2720016837120056, + "logps/rejected": -0.2790417969226837, + "loss": 0.26181310415267944, + "loss/core": 0.23417861759662628, + "loss/preference_sft": 0.2686235010623932, + "loss/reference_anchor": 0.24948279559612274, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.6532604694366455, + "rewards/margins": 1.4311665296554565, + "rewards/rejected": 1.222094178199768, + "ronpo/logit": 0.14311663806438446, + "ronpo/residual": 0.13579586148262024, + "ronpo/residual_abs": 0.1613430231809616, + "ronpo/target": 0.007320776581764221, + "ronpo/target_abs": 0.045001320540905, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 560 + }, + { + "drift/chosen_abs": 0.25932857394218445, + "drift/rejected_abs": 0.11433260142803192, + "epoch": 0.1795788637266587, + "grad_norm": 29.625, + "learning_rate": 2.2386788418308666e-08, + "logits/chosen": -2.0426180362701416, + "logits/rejected": -2.067936658859253, + "logps/chosen": -0.3414672017097473, + "logps/rejected": -0.3361096978187561, + "loss": 0.38202768564224243, + "loss/core": 0.33821603655815125, + "loss/preference_sft": 0.3408833146095276, + "loss/reference_anchor": 0.40402793884277344, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.5908656120300293, + "rewards/margins": 1.4496145248413086, + "rewards/rejected": 1.1412513256072998, + "ronpo/logit": 0.14496144652366638, + "ronpo/residual": 0.14660702645778656, + "ronpo/residual_abs": 0.17200031876564026, + "ronpo/target": -0.001645585522055626, + "ronpo/target_abs": 0.04828788712620735, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 565 + }, + { + "drift/chosen_abs": 0.4130350947380066, + "drift/rejected_abs": 0.10012461245059967, + "epoch": 0.18116805721096543, + "grad_norm": 18.875, + "learning_rate": 2.184502577160426e-08, + "logits/chosen": -2.149862289428711, + "logits/rejected": -2.119326114654541, + "logps/chosen": -0.27440696954727173, + "logps/rejected": -0.26447421312332153, + "loss": 1.456210970878601, + "loss/core": 1.372689962387085, + "loss/preference_sft": 0.25947293639183044, + "loss/reference_anchor": 0.8092624545097351, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 4.13002872467041, + "rewards/margins": 3.131789445877075, + "rewards/rejected": 0.9982389211654663, + "ronpo/logit": 0.3131789565086365, + "ronpo/residual": 0.3171713352203369, + "ronpo/residual_abs": 0.35084468126296997, + "ronpo/target": -0.00399237172678113, + "ronpo/target_abs": 0.05224527791142464, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 570 + }, + { + "drift/chosen_abs": 0.19252817332744598, + "drift/rejected_abs": 0.0961676687002182, + "epoch": 0.18275725069527216, + "grad_norm": 3.796875, + "learning_rate": 2.130476472175973e-08, + "logits/chosen": -2.009521722793579, + "logits/rejected": -2.0971667766571045, + "logps/chosen": -0.282725065946579, + "logps/rejected": -0.27961233258247375, + "loss": 0.06870145350694656, + "loss/core": 0.05739961937069893, + "loss/preference_sft": 0.2835802137851715, + "loss/reference_anchor": 0.08466026186943054, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 1.9235862493515015, + "rewards/margins": 0.9644415974617004, + "rewards/rejected": 0.9591447114944458, + "ronpo/logit": 0.09644416719675064, + "ronpo/residual": 0.1159183606505394, + "ronpo/residual_abs": 0.14433664083480835, + "ronpo/target": -0.01947418972849846, + "ronpo/target_abs": 0.0581415593624115, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 575 + }, + { + "drift/chosen_abs": 0.3638065457344055, + "drift/rejected_abs": 0.16699595749378204, + "epoch": 0.18434644417957885, + "grad_norm": 3.46875, + "learning_rate": 2.0766262403774385e-08, + "logits/chosen": -2.002498149871826, + "logits/rejected": -2.032501697540283, + "logps/chosen": -0.28927743434906006, + "logps/rejected": -0.26778870820999146, + "loss": 0.6307193040847778, + "loss/core": 0.5643881559371948, + "loss/preference_sft": 0.2721135914325714, + "loss/reference_anchor": 0.6360998153686523, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.637876033782959, + "rewards/margins": 1.9686304330825806, + "rewards/rejected": 1.669245719909668, + "ronpo/logit": 0.19686304032802582, + "ronpo/residual": 0.1981230527162552, + "ronpo/residual_abs": 0.24869951605796814, + "ronpo/target": -0.0012599985348060727, + "ronpo/target_abs": 0.04539477080106735, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 580 + }, + { + "drift/chosen_abs": 0.23143327236175537, + "drift/rejected_abs": 0.09587498754262924, + "epoch": 0.18593563766388557, + "grad_norm": 1.796875, + "learning_rate": 2.0229775115586378e-08, + "logits/chosen": -1.9558576345443726, + "logits/rejected": -1.9806501865386963, + "logps/chosen": -0.28681617975234985, + "logps/rejected": -0.28583577275276184, + "loss": 0.16384239494800568, + "loss/core": 0.14417728781700134, + "loss/preference_sft": 0.2843136489391327, + "loss/reference_anchor": 0.1682194620370865, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 2.3142428398132324, + "rewards/margins": 1.3574857711791992, + "rewards/rejected": 0.9567572474479675, + "ronpo/logit": 0.13574858009815216, + "ronpo/residual": 0.13210666179656982, + "ronpo/residual_abs": 0.16300050914287567, + "ronpo/target": 0.00364193064160645, + "ronpo/target_abs": 0.05011861398816109, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 585 + }, + { + "drift/chosen_abs": 0.25644367933273315, + "drift/rejected_abs": 0.123075470328331, + "epoch": 0.1875248311481923, + "grad_norm": 1.671875, + "learning_rate": 1.9695558196088843e-08, + "logits/chosen": -2.1181397438049316, + "logits/rejected": -2.086034059524536, + "logps/chosen": -0.31038814783096313, + "logps/rejected": -0.31416597962379456, + "loss": 0.5733916163444519, + "loss/core": 0.526887059211731, + "loss/preference_sft": 0.30243176221847534, + "loss/reference_anchor": 0.4348035454750061, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.560229778289795, + "rewards/margins": 1.3305037021636963, + "rewards/rejected": 1.2297260761260986, + "ronpo/logit": 0.1330503672361374, + "ronpo/residual": 0.1380425989627838, + "ronpo/residual_abs": 0.1819685846567154, + "ronpo/target": -0.0049922168254852295, + "ronpo/target_abs": 0.04599359631538391, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 590 + }, + { + "drift/chosen_abs": 0.232018381357193, + "drift/rejected_abs": 0.11203984171152115, + "epoch": 0.18911402463249902, + "grad_norm": 5.28125, + "learning_rate": 1.9163865903602372e-08, + "logits/chosen": -1.9771127700805664, + "logits/rejected": -1.9363014698028564, + "logps/chosen": -0.28953754901885986, + "logps/rejected": -0.2934724986553192, + "loss": 0.2176361083984375, + "loss/core": 0.19568440318107605, + "loss/preference_sft": 0.29219508171081543, + "loss/reference_anchor": 0.19029754400253296, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.3178868293762207, + "rewards/margins": 1.200582504272461, + "rewards/rejected": 1.1173043251037598, + "ronpo/logit": 0.12005823850631714, + "ronpo/residual": 0.12189905345439911, + "ronpo/residual_abs": 0.17195966839790344, + "ronpo/target": -0.0018408190226182342, + "ronpo/target_abs": 0.048326022922992706, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 595 + }, + { + "drift/chosen_abs": 0.20352081954479218, + "drift/rejected_abs": 0.14214810729026794, + "epoch": 0.1907032181168057, + "grad_norm": 154.0, + "learning_rate": 1.8634951294861806e-08, + "logits/chosen": -2.1343958377838135, + "logits/rejected": -2.1073379516601562, + "logps/chosen": -0.2995579242706299, + "logps/rejected": -0.3202636241912842, + "loss": 0.4734867513179779, + "loss/core": 0.4431106150150299, + "loss/preference_sft": 0.3036237061023712, + "loss/reference_anchor": 0.27340036630630493, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.0335845947265625, + "rewards/margins": 0.6139718890190125, + "rewards/rejected": 1.4196126461029053, + "ronpo/logit": 0.06139718368649483, + "ronpo/residual": 0.06256367266178131, + "ronpo/residual_abs": 0.2166094332933426, + "ronpo/target": -0.0011664947960525751, + "ronpo/target_abs": 0.049785248935222626, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 600 + }, + { + "drift/chosen_abs": 0.34506934881210327, + "drift/rejected_abs": 0.1576377898454666, + "epoch": 0.19229241160111243, + "grad_norm": 1.796875, + "learning_rate": 1.810906610457502e-08, + "logits/chosen": -2.029658794403076, + "logits/rejected": -1.9551337957382202, + "logps/chosen": -0.28458595275878906, + "logps/rejected": -0.2794283330440521, + "loss": 0.507605254650116, + "loss/core": 0.44724521040916443, + "loss/preference_sft": 0.2836672067642212, + "loss/reference_anchor": 0.5752339363098145, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.4416699409484863, + "rewards/margins": 1.8752292394638062, + "rewards/rejected": 1.5664407014846802, + "ronpo/logit": 0.18752291798591614, + "ronpo/residual": 0.1868840903043747, + "ronpo/residual_abs": 0.22322769463062286, + "ronpo/target": 0.0006388464826159179, + "ronpo/target_abs": 0.05182533711194992, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 605 + }, + { + "drift/chosen_abs": 0.3366909623146057, + "drift/rejected_abs": 0.1535976231098175, + "epoch": 0.19388160508541916, + "grad_norm": 92.0, + "learning_rate": 1.7586460625610727e-08, + "logits/chosen": -2.0207197666168213, + "logits/rejected": -1.9711004495620728, + "logps/chosen": -0.30812549591064453, + "logps/rejected": -0.29248955845832825, + "loss": 0.3734953701496124, + "loss/core": 0.3218352198600769, + "loss/preference_sft": 0.30512887239456177, + "loss/reference_anchor": 0.48608869314193726, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.365083694458008, + "rewards/margins": 1.8310083150863647, + "rewards/rejected": 1.534075140953064, + "ronpo/logit": 0.1831008344888687, + "ronpo/residual": 0.18432757258415222, + "ronpo/residual_abs": 0.2148648202419281, + "ronpo/target": -0.00122673565056175, + "ronpo/target_abs": 0.05218973010778427, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 610 + }, + { + "drift/chosen_abs": 0.25414878129959106, + "drift/rejected_abs": 0.1142294853925705, + "epoch": 0.19547079856972585, + "grad_norm": 129.0, + "learning_rate": 1.70673835898727e-08, + "logits/chosen": -2.2016429901123047, + "logits/rejected": -2.16279673576355, + "logps/chosen": -0.2886617183685303, + "logps/rejected": -0.28729248046875, + "loss": 0.32489103078842163, + "loss/core": 0.2884674370288849, + "loss/preference_sft": 0.2909470200538635, + "loss/reference_anchor": 0.3351413607597351, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.5414881706237793, + "rewards/margins": 1.4145499467849731, + "rewards/rejected": 1.1269382238388062, + "ronpo/logit": 0.14145499467849731, + "ronpo/residual": 0.140377014875412, + "ronpo/residual_abs": 0.17693236470222473, + "ronpo/target": 0.0010779699077829719, + "ronpo/target_abs": 0.05794703960418701, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 615 + }, + { + "drift/chosen_abs": 0.22227022051811218, + "drift/rejected_abs": 0.10932586342096329, + "epoch": 0.19705999205403257, + "grad_norm": 13.6875, + "learning_rate": 1.6552082049916824e-08, + "logits/chosen": -2.0862820148468018, + "logits/rejected": -2.021430253982544, + "logps/chosen": -0.291514128446579, + "logps/rejected": -0.28724825382232666, + "loss": 0.11000826209783554, + "loss/core": 0.09267652034759521, + "loss/preference_sft": 0.2868395745754242, + "loss/reference_anchor": 0.1446334570646286, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.2227022647857666, + "rewards/margins": 1.1303623914718628, + "rewards/rejected": 1.0923398733139038, + "ronpo/logit": 0.11303625255823135, + "ronpo/residual": 0.1041765958070755, + "ronpo/residual_abs": 0.14824050664901733, + "ronpo/target": 0.00885964184999466, + "ronpo/target_abs": 0.05069901794195175, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 620 + }, + { + "drift/chosen_abs": 0.24614998698234558, + "drift/rejected_abs": 0.0928288996219635, + "epoch": 0.1986491855383393, + "grad_norm": 43.75, + "learning_rate": 1.604080126136749e-08, + "logits/chosen": -2.038576126098633, + "logits/rejected": -2.042659044265747, + "logps/chosen": -0.2963336408138275, + "logps/rejected": -0.29823175072669983, + "loss": 0.34140458703041077, + "loss/core": 0.3157057762145996, + "loss/preference_sft": 0.2944321036338806, + "loss/reference_anchor": 0.22754497826099396, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.4598302841186523, + "rewards/margins": 1.5353715419769287, + "rewards/rejected": 0.9244586825370789, + "ronpo/logit": 0.15353716909885406, + "ronpo/residual": 0.15676531195640564, + "ronpo/residual_abs": 0.19076670706272125, + "ronpo/target": -0.0032281391322612762, + "ronpo/target_abs": 0.052575696259737015, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 625 + }, + { + "drift/chosen_abs": 0.23410654067993164, + "drift/rejected_abs": 0.08934091031551361, + "epoch": 0.20023837902264602, + "grad_norm": 6.875, + "learning_rate": 1.5533784566189177e-08, + "logits/chosen": -2.328397035598755, + "logits/rejected": -2.4190592765808105, + "logps/chosen": -0.2671896815299988, + "logps/rejected": -0.2637612819671631, + "loss": 0.32761818170547485, + "loss/core": 0.29400014877319336, + "loss/preference_sft": 0.26751917600631714, + "loss/reference_anchor": 0.309428334236145, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 2.3410582542419434, + "rewards/margins": 1.4506725072860718, + "rewards/rejected": 0.8903859853744507, + "ronpo/logit": 0.1450672447681427, + "ronpo/residual": 0.14449718594551086, + "ronpo/residual_abs": 0.17810878157615662, + "ronpo/target": 0.0005700558540411294, + "ronpo/target_abs": 0.04920119792222977, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 630 + }, + { + "drift/chosen_abs": 0.2907083034515381, + "drift/rejected_abs": 0.1089189276099205, + "epoch": 0.2018275725069527, + "grad_norm": 15.75, + "learning_rate": 1.5031273276868844e-08, + "logits/chosen": -2.112438440322876, + "logits/rejected": -2.1512842178344727, + "logps/chosen": -0.26441654562950134, + "logps/rejected": -0.2715560495853424, + "loss": 0.3996767997741699, + "loss/core": 0.3656352162361145, + "loss/preference_sft": 0.265216588973999, + "loss/reference_anchor": 0.31389421224594116, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.905461549758911, + "rewards/margins": 1.824873685836792, + "rewards/rejected": 1.0805878639221191, + "ronpo/logit": 0.1824873685836792, + "ronpo/residual": 0.18542273342609406, + "ronpo/residual_abs": 0.21292610466480255, + "ronpo/target": -0.002935352735221386, + "ronpo/target_abs": 0.04666103422641754, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 635 + }, + { + "drift/chosen_abs": 0.324721097946167, + "drift/rejected_abs": 0.09727884829044342, + "epoch": 0.20341676599125944, + "grad_norm": 1.25, + "learning_rate": 1.4533506561564305e-08, + "logits/chosen": -2.0161423683166504, + "logits/rejected": -2.0071678161621094, + "logps/chosen": -0.31316065788269043, + "logps/rejected": -0.3185476064682007, + "loss": 0.8060309290885925, + "loss/core": 0.7467989921569824, + "loss/preference_sft": 0.30903545022010803, + "loss/reference_anchor": 0.5614163875579834, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.2465310096740723, + "rewards/margins": 2.313912868499756, + "rewards/rejected": 0.9326180219650269, + "ronpo/logit": 0.23139123618602753, + "ronpo/residual": 0.22939686477184296, + "ronpo/residual_abs": 0.26494738459587097, + "ronpo/target": 0.0019944149535149336, + "ronpo/target_abs": 0.05449249595403671, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 640 + }, + { + "drift/chosen_abs": 0.2655789256095886, + "drift/rejected_abs": 0.11475946009159088, + "epoch": 0.20500595947556616, + "grad_norm": 9.1875, + "learning_rate": 1.4040721330273062e-08, + "logits/chosen": -2.1235086917877197, + "logits/rejected": -2.1292612552642822, + "logps/chosen": -0.3084896206855774, + "logps/rejected": -0.2943161129951477, + "loss": 0.36062735319137573, + "loss/core": 0.3258729875087738, + "loss/preference_sft": 0.29873496294021606, + "loss/reference_anchor": 0.3176696300506592, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.6545615196228027, + "rewards/margins": 1.5085389614105225, + "rewards/rejected": 1.1460225582122803, + "ronpo/logit": 0.15085388720035553, + "ronpo/residual": 0.14222462475299835, + "ronpo/residual_abs": 0.18793636560440063, + "ronpo/target": 0.008629274554550648, + "ronpo/target_abs": 0.05275329202413559, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 645 + }, + { + "drift/chosen_abs": 0.15096431970596313, + "drift/rejected_abs": 0.09060250222682953, + "epoch": 0.20659515295987285, + "grad_norm": 1.53125, + "learning_rate": 1.3553152122076079e-08, + "logits/chosen": -2.2055015563964844, + "logits/rejected": -2.133873462677002, + "logps/chosen": -0.30167868733406067, + "logps/rejected": -0.29324105381965637, + "loss": 0.056435346603393555, + "loss/core": 0.046567730605602264, + "loss/preference_sft": 0.30248433351516724, + "loss/reference_anchor": 0.06842776387929916, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.5087251663208008, + "rewards/margins": 0.652277410030365, + "rewards/rejected": 0.856447696685791, + "ronpo/logit": 0.06522773951292038, + "ronpo/residual": 0.06795114278793335, + "ronpo/residual_abs": 0.11372838169336319, + "ronpo/target": -0.0027233988512307405, + "ronpo/target_abs": 0.05443141609430313, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 650 + }, + { + "drift/chosen_abs": 0.20415695011615753, + "drift/rejected_abs": 0.06474515795707703, + "epoch": 0.20818434644417957, + "grad_norm": 169.0, + "learning_rate": 1.3071030993509787e-08, + "logits/chosen": -2.2794108390808105, + "logits/rejected": -2.285342216491699, + "logps/chosen": -0.2840138077735901, + "logps/rejected": -0.29859381914138794, + "loss": 0.38816702365875244, + "loss/core": 0.3634727895259857, + "loss/preference_sft": 0.2865355908870697, + "loss/reference_anchor": 0.21828866004943848, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.04097843170166, + "rewards/margins": 1.3973244428634644, + "rewards/rejected": 0.6436540484428406, + "ronpo/logit": 0.13973243534564972, + "ronpo/residual": 0.15421327948570251, + "ronpo/residual_abs": 0.18282660841941833, + "ronpo/target": -0.014480856247246265, + "ronpo/target_abs": 0.05253823474049568, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 655 + }, + { + "drift/chosen_abs": 0.1760435551404953, + "drift/rejected_abs": 0.07354947924613953, + "epoch": 0.2097735399284863, + "grad_norm": 6.5, + "learning_rate": 1.2594587408119802e-08, + "logits/chosen": -2.2539172172546387, + "logits/rejected": -2.2968709468841553, + "logps/chosen": -0.2949877381324768, + "logps/rejected": -0.2898675203323364, + "loss": 0.14153003692626953, + "loss/core": 0.1278771013021469, + "loss/preference_sft": 0.2885059118270874, + "loss/reference_anchor": 0.10767867416143417, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 1.758472204208374, + "rewards/margins": 1.0266904830932617, + "rewards/rejected": 0.7317818999290466, + "ronpo/logit": 0.10266906023025513, + "ronpo/residual": 0.09563291817903519, + "ronpo/residual_abs": 0.12419185787439346, + "ronpo/target": 0.007036135531961918, + "ronpo/target_abs": 0.051446299999952316, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 660 + }, + { + "drift/chosen_abs": 0.23514601588249207, + "drift/rejected_abs": 0.13599976897239685, + "epoch": 0.21136273341279302, + "grad_norm": 0.4765625, + "learning_rate": 1.2124048127248643e-08, + "logits/chosen": -2.0788931846618652, + "logits/rejected": -2.023658514022827, + "logps/chosen": -0.2821238040924072, + "logps/rejected": -0.2862771153450012, + "loss": 0.2474207878112793, + "loss/core": 0.2166004627943039, + "loss/preference_sft": 0.28421351313591003, + "loss/reference_anchor": 0.2797810733318329, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.348618268966675, + "rewards/margins": 0.9887655377388, + "rewards/rejected": 1.35985267162323, + "ronpo/logit": 0.09887654334306717, + "ronpo/residual": 0.09633339941501617, + "ronpo/residual_abs": 0.14032281935214996, + "ronpo/target": 0.002543145790696144, + "ronpo/target_abs": 0.044390588998794556, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 665 + }, + { + "drift/chosen_abs": 0.21025514602661133, + "drift/rejected_abs": 0.07857976108789444, + "epoch": 0.2129519268970997, + "grad_norm": 5.65625, + "learning_rate": 1.1659637102109713e-08, + "logits/chosen": -2.3492414951324463, + "logits/rejected": -2.356902599334717, + "logps/chosen": -0.2673395574092865, + "logps/rejected": -0.2719830572605133, + "loss": 0.20967233180999756, + "loss/core": 0.19076594710350037, + "loss/preference_sft": 0.2634381651878357, + "loss/reference_anchor": 0.1627199351787567, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.1009294986724854, + "rewards/margins": 1.3163961172103882, + "rewards/rejected": 0.7845333814620972, + "ronpo/logit": 0.13163962960243225, + "ronpo/residual": 0.11512235552072525, + "ronpo/residual_abs": 0.15650084614753723, + "ronpo/target": 0.016517281532287598, + "ronpo/target_abs": 0.05511252209544182, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 670 + }, + { + "drift/chosen_abs": 0.40095004439353943, + "drift/rejected_abs": 0.08112253993749619, + "epoch": 0.21454112038140644, + "grad_norm": 2.3125, + "learning_rate": 1.1201575367198545e-08, + "logits/chosen": -1.9876753091812134, + "logits/rejected": -2.1013565063476562, + "logps/chosen": -0.27906253933906555, + "logps/rejected": -0.26979774236679077, + "loss": 1.4776394367218018, + "loss/core": 1.3953857421875, + "loss/preference_sft": 0.26779764890670776, + "loss/reference_anchor": 0.7957568764686584, + "rewards/accuracies": 0.9375, + "rewards/chosen": 4.009209156036377, + "rewards/margins": 3.201277256011963, + "rewards/rejected": 0.8079322576522827, + "ronpo/logit": 0.3201276957988739, + "ronpo/residual": 0.3296274244785309, + "ronpo/residual_abs": 0.34753426909446716, + "ronpo/target": -0.009499681182205677, + "ronpo/target_abs": 0.053924452513456345, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 675 + }, + { + "drift/chosen_abs": 0.19604100286960602, + "drift/rejected_abs": 0.1168225035071373, + "epoch": 0.21613031386571316, + "grad_norm": 2.265625, + "learning_rate": 1.0750080935092422e-08, + "logits/chosen": -1.9559427499771118, + "logits/rejected": -1.9504743814468384, + "logps/chosen": -0.28676366806030273, + "logps/rejected": -0.2847217917442322, + "loss": 0.07230311632156372, + "loss/core": 0.05109725520014763, + "loss/preference_sft": 0.28723257780075073, + "loss/reference_anchor": 0.18333539366722107, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9591118097305298, + "rewards/margins": 0.791954755783081, + "rewards/rejected": 1.1671569347381592, + "ronpo/logit": 0.07919548451900482, + "ronpo/residual": 0.0781867653131485, + "ronpo/residual_abs": 0.1194804310798645, + "ronpo/target": 0.0010087231639772654, + "ronpo/target_abs": 0.05514944717288017, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 680 + }, + { + "drift/chosen_abs": 0.18207547068595886, + "drift/rejected_abs": 0.09372608363628387, + "epoch": 0.21771950735001985, + "grad_norm": 3.109375, + "learning_rate": 1.0305368692688174e-08, + "logits/chosen": -1.9711679220199585, + "logits/rejected": -2.0610291957855225, + "logps/chosen": -0.29287078976631165, + "logps/rejected": -0.30058807134628296, + "loss": 0.05506179481744766, + "loss/core": 0.04439196363091469, + "loss/preference_sft": 0.2933097183704376, + "loss/reference_anchor": 0.07736729830503464, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.8186616897583008, + "rewards/margins": 0.8839622735977173, + "rewards/rejected": 0.9346992373466492, + "ronpo/logit": 0.08839622884988785, + "ronpo/residual": 0.08107195794582367, + "ronpo/residual_abs": 0.11638529598712921, + "ronpo/target": 0.007324265781790018, + "ronpo/target_abs": 0.04978422075510025, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 685 + }, + { + "drift/chosen_abs": 0.4522300362586975, + "drift/rejected_abs": 0.1868419349193573, + "epoch": 0.21930870083432658, + "grad_norm": 440.0, + "learning_rate": 9.867650298927644e-09, + "logits/chosen": -1.9316879510879517, + "logits/rejected": -1.9761654138565063, + "logps/chosen": -0.27664968371391296, + "logps/rejected": -0.27031582593917847, + "loss": 0.9360002279281616, + "loss/core": 0.8222877383232117, + "loss/preference_sft": 0.2849580645561218, + "loss/reference_anchor": 1.108628511428833, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 4.52152156829834, + "rewards/margins": 2.65468692779541, + "rewards/rejected": 1.866834044456482, + "ronpo/logit": 0.2654687762260437, + "ronpo/residual": 0.2594831585884094, + "ronpo/residual_abs": 0.2916156053543091, + "ronpo/target": 0.0059855724684894085, + "ronpo/target_abs": 0.048761725425720215, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 690 + }, + { + "drift/chosen_abs": 0.18377462029457092, + "drift/rejected_abs": 0.10978679358959198, + "epoch": 0.2208978943186333, + "grad_norm": 12.5, + "learning_rate": 9.437134084059515e-09, + "logits/chosen": -2.163043260574341, + "logits/rejected": -2.150045156478882, + "logps/chosen": -0.2978960871696472, + "logps/rejected": -0.30089619755744934, + "loss": 0.11494803428649902, + "loss/core": 0.09975998103618622, + "loss/preference_sft": 0.29669278860092163, + "loss/reference_anchor": 0.12221113592386246, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.8371896743774414, + "rewards/margins": 0.7406196594238281, + "rewards/rejected": 1.0965700149536133, + "ronpo/logit": 0.07406197488307953, + "ronpo/residual": 0.07061334699392319, + "ronpo/residual_abs": 0.1484992951154709, + "ronpo/target": 0.003448624163866043, + "ronpo/target_abs": 0.05069219321012497, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 695 + }, + { + "drift/chosen_abs": 0.22288422286510468, + "drift/rejected_abs": 0.13863565027713776, + "epoch": 0.22248708780294002, + "grad_norm": 5.9375, + "learning_rate": 9.014024950485382e-09, + "logits/chosen": -2.21722674369812, + "logits/rejected": -2.178441047668457, + "logps/chosen": -0.32178056240081787, + "logps/rejected": -0.31281206011772156, + "loss": 0.460977703332901, + "loss/core": 0.4184110760688782, + "loss/preference_sft": 0.31066298484802246, + "loss/reference_anchor": 0.3945997357368469, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.2285068035125732, + "rewards/margins": 0.8443525433540344, + "rewards/rejected": 1.3841540813446045, + "ronpo/logit": 0.08443526178598404, + "ronpo/residual": 0.08723791688680649, + "ronpo/residual_abs": 0.1965109258890152, + "ronpo/target": -0.002802655566483736, + "ronpo/target_abs": 0.05208230018615723, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 700 + }, + { + "drift/chosen_abs": 0.18381109833717346, + "drift/rejected_abs": 0.08396384865045547, + "epoch": 0.22407628128724671, + "grad_norm": 2.359375, + "learning_rate": 8.59852427523732e-09, + "logits/chosen": -2.262627601623535, + "logits/rejected": -2.367927312850952, + "logps/chosen": -0.2676858901977539, + "logps/rejected": -0.27092617750167847, + "loss": 0.08023377507925034, + "loss/core": 0.06743164360523224, + "loss/preference_sft": 0.2751052975654602, + "loss/reference_anchor": 0.10051079839468002, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.8374383449554443, + "rewards/margins": 1.0038096904754639, + "rewards/rejected": 0.8336284756660461, + "ronpo/logit": 0.10038097947835922, + "ronpo/residual": 0.10365533828735352, + "ronpo/residual_abs": 0.1367378532886505, + "ronpo/target": -0.0032743613701313734, + "ronpo/target_abs": 0.05378307029604912, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 705 + }, + { + "drift/chosen_abs": 0.28812941908836365, + "drift/rejected_abs": 0.10422752052545547, + "epoch": 0.22566547477155344, + "grad_norm": 103.5, + "learning_rate": 8.190829814133293e-09, + "logits/chosen": -2.1671175956726074, + "logits/rejected": -2.1793408393859863, + "logps/chosen": -0.30027687549591064, + "logps/rejected": -0.29821285605430603, + "loss": 0.7324048280715942, + "loss/core": 0.6899200677871704, + "loss/preference_sft": 0.29592448472976685, + "loss/reference_anchor": 0.3952544927597046, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.881293773651123, + "rewards/margins": 1.9280929565429688, + "rewards/rejected": 0.9532011151313782, + "ronpo/logit": 0.19280926883220673, + "ronpo/residual": 0.19424277544021606, + "ronpo/residual_abs": 0.24397198855876923, + "ronpo/target": -0.0014334864681586623, + "ronpo/target_abs": 0.055146049708127975, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 710 + }, + { + "drift/chosen_abs": 0.1660590022802353, + "drift/rejected_abs": 0.09195362776517868, + "epoch": 0.22725466825586016, + "grad_norm": 1.328125, + "learning_rate": 7.791135607656146e-09, + "logits/chosen": -2.0482685565948486, + "logits/rejected": -1.9684371948242188, + "logps/chosen": -0.3031107187271118, + "logps/rejected": -0.3003300726413727, + "loss": 0.07312048971652985, + "loss/core": 0.06187158077955246, + "loss/preference_sft": 0.3050219416618347, + "loss/reference_anchor": 0.08198684453964233, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.6589847803115845, + "rewards/margins": 0.7399789690971375, + "rewards/rejected": 0.9190058708190918, + "ronpo/logit": 0.07399789243936539, + "ronpo/residual": 0.06657494604587555, + "ronpo/residual_abs": 0.1202554851770401, + "ronpo/target": 0.007422941271215677, + "ronpo/target_abs": 0.04994440823793411, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 715 + }, + { + "drift/chosen_abs": 0.3468184471130371, + "drift/rejected_abs": 0.14277954399585724, + "epoch": 0.22884386174016685, + "grad_norm": 452.0, + "learning_rate": 7.399631888600796e-09, + "logits/chosen": -1.9520069360733032, + "logits/rejected": -2.0314078330993652, + "logps/chosen": -0.29010164737701416, + "logps/rejected": -0.28294092416763306, + "loss": 0.6354092359542847, + "loss/core": 0.5749737024307251, + "loss/preference_sft": 0.2856560945510864, + "loss/reference_anchor": 0.5757890939712524, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.46818470954895, + "rewards/margins": 2.045009136199951, + "rewards/rejected": 1.4231754541397095, + "ronpo/logit": 0.20450091361999512, + "ronpo/residual": 0.214862659573555, + "ronpo/residual_abs": 0.2341841459274292, + "ronpo/target": -0.010361749678850174, + "ronpo/target_abs": 0.049705568701028824, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 720 + }, + { + "drift/chosen_abs": 0.2965972125530243, + "drift/rejected_abs": 0.06823800504207611, + "epoch": 0.23043305522447358, + "grad_norm": 4.65625, + "learning_rate": 7.016504991533726e-09, + "logits/chosen": -2.0876588821411133, + "logits/rejected": -2.0283491611480713, + "logps/chosen": -0.28961247205734253, + "logps/rejected": -0.29187989234924316, + "loss": 1.162423849105835, + "loss/core": 1.1012930870056152, + "loss/preference_sft": 0.29563257098197937, + "loss/reference_anchor": 0.5817441344261169, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.964808464050293, + "rewards/margins": 2.2854580879211426, + "rewards/rejected": 0.6793509721755981, + "ronpo/logit": 0.22854574024677277, + "ronpo/residual": 0.23741579055786133, + "ronpo/residual_abs": 0.267715722322464, + "ronpo/target": -0.008870027028024197, + "ronpo/target_abs": 0.0548769049346447, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 725 + }, + { + "drift/chosen_abs": 0.3146856725215912, + "drift/rejected_abs": 0.093979112803936, + "epoch": 0.2320222487087803, + "grad_norm": 2032.0, + "learning_rate": 6.641937264107866e-09, + "logits/chosen": -2.183267593383789, + "logits/rejected": -2.218003511428833, + "logps/chosen": -0.29960209131240845, + "logps/rejected": -0.29752200841903687, + "loss": 0.9546008110046387, + "loss/core": 0.8912256360054016, + "loss/preference_sft": 0.29938754439353943, + "loss/reference_anchor": 0.6038151383399963, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.1452383995056152, + "rewards/margins": 2.208786725997925, + "rewards/rejected": 0.9364517331123352, + "ronpo/logit": 0.22087864577770233, + "ronpo/residual": 0.21167048811912537, + "ronpo/residual_abs": 0.24570854008197784, + "ronpo/target": 0.009208188392221928, + "ronpo/target_abs": 0.050097811967134476, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 730 + }, + { + "drift/chosen_abs": 0.4118995666503906, + "drift/rejected_abs": 0.15235108137130737, + "epoch": 0.23361144219308702, + "grad_norm": 11.6875, + "learning_rate": 6.276106980274945e-09, + "logits/chosen": -1.8743293285369873, + "logits/rejected": -1.925328016281128, + "logps/chosen": -0.31055787205696106, + "logps/rejected": -0.2830975651741028, + "loss": 1.4325848817825317, + "loss/core": 1.343813180923462, + "loss/preference_sft": 0.2843840718269348, + "loss/reference_anchor": 0.8592769503593445, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 4.117542266845703, + "rewards/margins": 2.6022226810455322, + "rewards/rejected": 1.5153193473815918, + "ronpo/logit": 0.2602222263813019, + "ronpo/residual": 0.2596417963504791, + "ronpo/residual_abs": 0.3390171229839325, + "ronpo/target": 0.0005804725224152207, + "ronpo/target_abs": 0.052908122539520264, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 735 + }, + { + "drift/chosen_abs": 0.40725356340408325, + "drift/rejected_abs": 0.17515242099761963, + "epoch": 0.23520063567739372, + "grad_norm": 9.1875, + "learning_rate": 5.919188255436777e-09, + "logits/chosen": -2.062140941619873, + "logits/rejected": -2.053471088409424, + "logps/chosen": -0.2751580774784088, + "logps/rejected": -0.2721211016178131, + "loss": 0.9029391407966614, + "loss/core": 0.7974163889884949, + "loss/preference_sft": 0.274557888507843, + "loss/reference_anchor": 1.0277706384658813, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 4.072251796722412, + "rewards/margins": 2.321990966796875, + "rewards/rejected": 1.7502609491348267, + "ronpo/logit": 0.23219911754131317, + "ronpo/residual": 0.22244659066200256, + "ronpo/residual_abs": 0.27284926176071167, + "ronpo/target": 0.009752524085342884, + "ronpo/target_abs": 0.04651670157909393, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 740 + }, + { + "drift/chosen_abs": 0.2788258194923401, + "drift/rejected_abs": 0.13999952375888824, + "epoch": 0.23678982916170044, + "grad_norm": 1.1640625, + "learning_rate": 5.571350963575727e-09, + "logits/chosen": -1.965963363647461, + "logits/rejected": -1.9410635232925415, + "logps/chosen": -0.3108576834201813, + "logps/rejected": -0.29554474353790283, + "loss": 0.42476820945739746, + "loss/core": 0.38765430450439453, + "loss/preference_sft": 0.3039025664329529, + "loss/reference_anchor": 0.34074899554252625, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.785062313079834, + "rewards/margins": 1.4028537273406982, + "rewards/rejected": 1.382208228111267, + "ronpo/logit": 0.14028538763523102, + "ronpo/residual": 0.1309804618358612, + "ronpo/residual_abs": 0.1938273310661316, + "ronpo/target": 0.009304934181272984, + "ronpo/target_abs": 0.05095517635345459, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 745 + }, + { + "drift/chosen_abs": 0.49405422806739807, + "drift/rejected_abs": 0.08988231420516968, + "epoch": 0.23837902264600716, + "grad_norm": 28.875, + "learning_rate": 5.2327606564039225e-09, + "logits/chosen": -1.8724329471588135, + "logits/rejected": -2.024134397506714, + "logps/chosen": -0.3255074918270111, + "logps/rejected": -0.3025345206260681, + "loss": 1.7781035900115967, + "loss/core": 1.674705147743225, + "loss/preference_sft": 0.2975428104400635, + "loss/reference_anchor": 1.0042304992675781, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.939142227172852, + "rewards/margins": 4.043478012084961, + "rewards/rejected": 0.8956640958786011, + "ronpo/logit": 0.40434780716896057, + "ronpo/residual": 0.41805681586265564, + "ronpo/residual_abs": 0.43893271684646606, + "ronpo/target": -0.013708988204598427, + "ronpo/target_abs": 0.04939410090446472, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 750 + }, + { + "drift/chosen_abs": 0.550828754901886, + "drift/rejected_abs": 0.25195178389549255, + "epoch": 0.23996821613031386, + "grad_norm": 183.0, + "learning_rate": 4.903578484569568e-09, + "logits/chosen": -1.7783501148223877, + "logits/rejected": -1.706907033920288, + "logps/chosen": -0.2732313275337219, + "logps/rejected": -0.2803798317909241, + "loss": 1.803733468055725, + "loss/core": 1.6817384958267212, + "loss/preference_sft": 0.25985172390937805, + "loss/reference_anchor": 1.1939656734466553, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 5.507561683654785, + "rewards/margins": 3.0016417503356934, + "rewards/rejected": 2.505919933319092, + "ronpo/logit": 0.3001641631126404, + "ronpo/residual": 0.2940429151058197, + "ronpo/residual_abs": 0.4681456685066223, + "ronpo/target": 0.006121277809143066, + "ronpo/target_abs": 0.05067913979291916, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 755 + }, + { + "drift/chosen_abs": 0.25603729486465454, + "drift/rejected_abs": 0.10197993367910385, + "epoch": 0.24155740961462058, + "grad_norm": 1.84375, + "learning_rate": 4.583961120958027e-09, + "logits/chosen": -2.2031126022338867, + "logits/rejected": -2.253415107727051, + "logps/chosen": -0.26150673627853394, + "logps/rejected": -0.26170969009399414, + "loss": 0.41260021924972534, + "loss/core": 0.379812628030777, + "loss/preference_sft": 0.2550763189792633, + "loss/reference_anchor": 0.3023683428764343, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.5603137016296387, + "rewards/margins": 1.5461097955703735, + "rewards/rejected": 1.0142039060592651, + "ronpo/logit": 0.15461096167564392, + "ronpo/residual": 0.1472678929567337, + "ronpo/residual_abs": 0.19386346638202667, + "ronpo/target": 0.007343077566474676, + "ronpo/target_abs": 0.05549933761358261, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 760 + }, + { + "drift/chosen_abs": 0.16723796725273132, + "drift/rejected_abs": 0.09124104678630829, + "epoch": 0.2431466030989273, + "grad_norm": 1.125, + "learning_rate": 4.2740606861239594e-09, + "logits/chosen": -2.399120330810547, + "logits/rejected": -2.3372297286987305, + "logps/chosen": -0.2863905429840088, + "logps/rejected": -0.29001763463020325, + "loss": 0.04658041149377823, + "loss/core": 0.033091410994529724, + "loss/preference_sft": 0.28594884276390076, + "loss/reference_anchor": 0.1062951311469078, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6717811822891235, + "rewards/margins": 0.7596646547317505, + "rewards/rejected": 0.9121164083480835, + "ronpo/logit": 0.07596646994352341, + "ronpo/residual": 0.0702570378780365, + "ronpo/residual_abs": 0.10007613897323608, + "ronpo/target": 0.005709432996809483, + "ronpo/target_abs": 0.043748289346694946, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 765 + }, + { + "drift/chosen_abs": 0.33148249983787537, + "drift/rejected_abs": 0.10127073526382446, + "epoch": 0.24473579658323402, + "grad_norm": 65.0, + "learning_rate": 3.974024675890189e-09, + "logits/chosen": -1.9635722637176514, + "logits/rejected": -2.0935862064361572, + "logps/chosen": -0.254546195268631, + "logps/rejected": -0.27278387546539307, + "loss": 0.6218898296356201, + "loss/core": 0.5817257761955261, + "loss/preference_sft": 0.25621524453163147, + "loss/reference_anchor": 0.3760190010070801, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.3138370513916016, + "rewards/margins": 2.3918099403381348, + "rewards/rejected": 0.9220269322395325, + "ronpo/logit": 0.23918099701404572, + "ronpo/residual": 0.2456188201904297, + "ronpo/residual_abs": 0.27006497979164124, + "ronpo/target": -0.006437826901674271, + "ronpo/target_abs": 0.05173778533935547, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 770 + }, + { + "drift/chosen_abs": 0.344692587852478, + "drift/rejected_abs": 0.1439075767993927, + "epoch": 0.24632499006754072, + "grad_norm": 0.94921875, + "learning_rate": 3.6839958911476955e-09, + "logits/chosen": -1.8287601470947266, + "logits/rejected": -1.816575288772583, + "logps/chosen": -0.2944333553314209, + "logps/rejected": -0.29648080468177795, + "loss": 0.5895982980728149, + "loss/core": 0.5448721051216125, + "loss/preference_sft": 0.29511556029319763, + "loss/reference_anchor": 0.41775059700012207, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.4456286430358887, + "rewards/margins": 2.052269458770752, + "rewards/rejected": 1.393358826637268, + "ronpo/logit": 0.20522697269916534, + "ronpo/residual": 0.20050707459449768, + "ronpo/residual_abs": 0.23854902386665344, + "ronpo/target": 0.004719903226941824, + "ronpo/target_abs": 0.0534793958067894, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 775 + }, + { + "drift/chosen_abs": 0.35038870573043823, + "drift/rejected_abs": 0.1556740552186966, + "epoch": 0.24791418355184744, + "grad_norm": 3.421875, + "learning_rate": 3.404112369890108e-09, + "logits/chosen": -2.1595654487609863, + "logits/rejected": -2.1828014850616455, + "logps/chosen": -0.2698805332183838, + "logps/rejected": -0.27884024381637573, + "loss": 0.37909770011901855, + "loss/core": 0.3323928713798523, + "loss/preference_sft": 0.2687809467315674, + "loss/reference_anchor": 0.4401705265045166, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.50298810005188, + "rewards/margins": 1.947210669517517, + "rewards/rejected": 1.5557773113250732, + "ronpo/logit": 0.1947210729122162, + "ronpo/residual": 0.19420704245567322, + "ronpo/residual_abs": 0.26663556694984436, + "ronpo/target": 0.0005140554858371615, + "ronpo/target_abs": 0.05390627309679985, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 780 + }, + { + "drift/chosen_abs": 0.4771912097930908, + "drift/rejected_abs": 0.10854575783014297, + "epoch": 0.24950337703615416, + "grad_norm": 4.1875, + "learning_rate": 3.1345073215151064e-09, + "logits/chosen": -2.103487491607666, + "logits/rejected": -2.214913845062256, + "logps/chosen": -0.2716423571109772, + "logps/rejected": -0.2626985013484955, + "loss": 1.8278989791870117, + "loss/core": 1.7319368124008179, + "loss/preference_sft": 0.2729416489601135, + "loss/reference_anchor": 0.9323285818099976, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.765838146209717, + "rewards/margins": 3.703094482421875, + "rewards/rejected": 1.0627435445785522, + "ronpo/logit": 0.370309442281723, + "ronpo/residual": 0.3793594241142273, + "ronpo/residual_abs": 0.40880364179611206, + "ronpo/target": -0.009049968793988228, + "ronpo/target_abs": 0.05032235383987427, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 785 + }, + { + "drift/chosen_abs": 0.3605960011482239, + "drift/rejected_abs": 0.12088502943515778, + "epoch": 0.25109257052046086, + "grad_norm": 196.0, + "learning_rate": 2.8753090634239558e-09, + "logits/chosen": -2.0059657096862793, + "logits/rejected": -2.025968074798584, + "logps/chosen": -0.3111969530582428, + "logps/rejected": -0.29000169038772583, + "loss": 0.5503872036933899, + "loss/core": 0.5009673237800598, + "loss/preference_sft": 0.30275315046310425, + "loss/reference_anchor": 0.463923841714859, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.6052303314208984, + "rewards/margins": 2.423473596572876, + "rewards/rejected": 1.1817567348480225, + "ronpo/logit": 0.2423473298549652, + "ronpo/residual": 0.24348480999469757, + "ronpo/residual_abs": 0.29033395648002625, + "ronpo/target": -0.001137460581958294, + "ronpo/target_abs": 0.05171912908554077, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 790 + }, + { + "drift/chosen_abs": 0.2400600016117096, + "drift/rejected_abs": 0.11403791606426239, + "epoch": 0.2526817640047676, + "grad_norm": 1.8125, + "learning_rate": 2.626640959949375e-09, + "logits/chosen": -2.132143497467041, + "logits/rejected": -2.166182041168213, + "logps/chosen": -0.2598033547401428, + "logps/rejected": -0.2518232464790344, + "loss": 0.12491835653781891, + "loss/core": 0.10928516089916229, + "loss/preference_sft": 0.2489791363477707, + "loss/reference_anchor": 0.13143403828144073, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.399862766265869, + "rewards/margins": 1.2609279155731201, + "rewards/rejected": 1.138934850692749, + "ronpo/logit": 0.1260928064584732, + "ronpo/residual": 0.13136708736419678, + "ronpo/residual_abs": 0.1680411547422409, + "ronpo/target": -0.00527427950873971, + "ronpo/target_abs": 0.05185922235250473, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 795 + }, + { + "drift/chosen_abs": 0.283835768699646, + "drift/rejected_abs": 0.1590053141117096, + "epoch": 0.2542709574890743, + "grad_norm": 18.875, + "learning_rate": 2.388621363640797e-09, + "logits/chosen": -2.1680550575256348, + "logits/rejected": -2.2392494678497314, + "logps/chosen": -0.2695561349391937, + "logps/rejected": -0.2701888978481293, + "loss": 0.2641626000404358, + "loss/core": 0.18199072778224945, + "loss/preference_sft": 0.27051591873168945, + "loss/reference_anchor": 0.7946673631668091, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.837977886199951, + "rewards/margins": 1.2485079765319824, + "rewards/rejected": 1.5894701480865479, + "ronpo/logit": 0.124850794672966, + "ronpo/residual": 0.12345902621746063, + "ronpo/residual_abs": 0.157121941447258, + "ronpo/target": 0.0013917500618845224, + "ronpo/target_abs": 0.04755866900086403, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 800 + }, + { + "drift/chosen_abs": 0.3202213644981384, + "drift/rejected_abs": 0.13916157186031342, + "epoch": 0.255860150973381, + "grad_norm": 204.0, + "learning_rate": 2.1613635589349754e-09, + "logits/chosen": -2.27490234375, + "logits/rejected": -2.2372684478759766, + "logps/chosen": -0.30233731865882874, + "logps/rejected": -0.3073555827140808, + "loss": 0.9722781181335449, + "loss/core": 0.9004875421524048, + "loss/preference_sft": 0.28705883026123047, + "loss/reference_anchor": 0.6892002820968628, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.202214002609253, + "rewards/margins": 1.8725093603134155, + "rewards/rejected": 1.3297045230865479, + "ronpo/logit": 0.18725092709064484, + "ronpo/residual": 0.18470832705497742, + "ronpo/residual_abs": 0.2942502200603485, + "ronpo/target": 0.0025425744242966175, + "ronpo/target_abs": 0.050916336476802826, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 805 + }, + { + "drift/chosen_abs": 0.21148745715618134, + "drift/rejected_abs": 0.1105772852897644, + "epoch": 0.25744934445768775, + "grad_norm": 8.4375, + "learning_rate": 1.944975708238708e-09, + "logits/chosen": -2.0623979568481445, + "logits/rejected": -2.1137001514434814, + "logps/chosen": -0.2832852602005005, + "logps/rejected": -0.2870853841304779, + "loss": 0.05209481716156006, + "loss/core": 0.03912948817014694, + "loss/preference_sft": 0.2837418019771576, + "loss/reference_anchor": 0.10127910226583481, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.1126019954681396, + "rewards/margins": 1.0071337223052979, + "rewards/rejected": 1.1054683923721313, + "ronpo/logit": 0.10071338713169098, + "ronpo/residual": 0.09261061996221542, + "ronpo/residual_abs": 0.12470117956399918, + "ronpo/target": 0.008102752268314362, + "ronpo/target_abs": 0.048462167382240295, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 810 + }, + { + "drift/chosen_abs": 0.25939127802848816, + "drift/rejected_abs": 0.16253964602947235, + "epoch": 0.25903853794199444, + "grad_norm": 12.875, + "learning_rate": 1.7395608004493884e-09, + "logits/chosen": -2.110250949859619, + "logits/rejected": -2.0329298973083496, + "logps/chosen": -0.2672348916530609, + "logps/rejected": -0.2788865268230438, + "loss": 0.23105821013450623, + "loss/core": 0.20882940292358398, + "loss/preference_sft": 0.263536661863327, + "loss/reference_anchor": 0.19593465328216553, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.5937724113464355, + "rewards/margins": 0.9874376058578491, + "rewards/rejected": 1.606334924697876, + "ronpo/logit": 0.09874377399682999, + "ronpo/residual": 0.0899277850985527, + "ronpo/residual_abs": 0.20764711499214172, + "ronpo/target": 0.008815980516374111, + "ronpo/target_abs": 0.05238635092973709, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 815 + }, + { + "drift/chosen_abs": 0.19982877373695374, + "drift/rejected_abs": 0.1136329397559166, + "epoch": 0.26062773142630113, + "grad_norm": 3.046875, + "learning_rate": 1.5452166019378988e-09, + "logits/chosen": -2.021080493927002, + "logits/rejected": -1.9773486852645874, + "logps/chosen": -0.2758404612541199, + "logps/rejected": -0.2889498174190521, + "loss": 0.07101830095052719, + "loss/core": 0.05234837532043457, + "loss/preference_sft": 0.27237778902053833, + "loss/reference_anchor": 0.15946142375469208, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.9943138360977173, + "rewards/margins": 0.8593200445175171, + "rewards/rejected": 1.1349937915802002, + "ronpo/logit": 0.08593200892210007, + "ronpo/residual": 0.08751102536916733, + "ronpo/residual_abs": 0.1283148229122162, + "ronpo/target": -0.0015790194738656282, + "ronpo/target_abs": 0.04799572750926018, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 820 + }, + { + "drift/chosen_abs": 0.2875332832336426, + "drift/rejected_abs": 0.1883879154920578, + "epoch": 0.2622169249106079, + "grad_norm": 88.0, + "learning_rate": 1.3620356100170789e-09, + "logits/chosen": -2.214282512664795, + "logits/rejected": -2.162520170211792, + "logps/chosen": -0.2991814613342285, + "logps/rejected": -0.30333730578422546, + "loss": 0.39491066336631775, + "loss/core": 0.30867427587509155, + "loss/preference_sft": 0.2967020869255066, + "loss/reference_anchor": 0.8326936960220337, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.87361478805542, + "rewards/margins": 1.0094401836395264, + "rewards/rejected": 1.864174485206604, + "ronpo/logit": 0.10094402730464935, + "ronpo/residual": 0.09369300305843353, + "ronpo/residual_abs": 0.20826761424541473, + "ronpo/target": 0.007251021917909384, + "ronpo/target_abs": 0.058990202844142914, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 825 + }, + { + "drift/chosen_abs": 0.18447622656822205, + "drift/rejected_abs": 0.11325931549072266, + "epoch": 0.2638061183949146, + "grad_norm": 31.875, + "learning_rate": 1.190105008918041e-09, + "logits/chosen": -2.2358505725860596, + "logits/rejected": -2.23689603805542, + "logps/chosen": -0.2778850197792053, + "logps/rejected": -0.2730768620967865, + "loss": 0.13070687651634216, + "loss/core": 0.11438991874456406, + "loss/preference_sft": 0.2781921625137329, + "loss/reference_anchor": 0.13535039126873016, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8447624444961548, + "rewards/margins": 0.7130740880966187, + "rewards/rejected": 1.1316883563995361, + "ronpo/logit": 0.07130740582942963, + "ronpo/residual": 0.06643342226743698, + "ronpo/residual_abs": 0.14979875087738037, + "ronpo/target": 0.004873988218605518, + "ronpo/target_abs": 0.05250249058008194, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 830 + }, + { + "drift/chosen_abs": 0.40701180696487427, + "drift/rejected_abs": 0.162872314453125, + "epoch": 0.2653953118792213, + "grad_norm": 33.25, + "learning_rate": 1.0295066282951737e-09, + "logits/chosen": -2.190506935119629, + "logits/rejected": -2.155287504196167, + "logps/chosen": -0.24197907745838165, + "logps/rejected": -0.24426302313804626, + "loss": 1.1218777894973755, + "loss/core": 1.0501762628555298, + "loss/preference_sft": 0.24557368457317352, + "loss/reference_anchor": 0.6924582719802856, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 4.069449424743652, + "rewards/margins": 2.4409031867980957, + "rewards/rejected": 1.6285464763641357, + "ronpo/logit": 0.24409031867980957, + "ronpo/residual": 0.24195992946624756, + "ronpo/residual_abs": 0.3348272442817688, + "ronpo/target": 0.0021303787361830473, + "ronpo/target_abs": 0.051504336297512054, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 835 + }, + { + "drift/chosen_abs": 0.2866782248020172, + "drift/rejected_abs": 0.13607479631900787, + "epoch": 0.266984505363528, + "grad_norm": 1.0546875, + "learning_rate": 8.803169042796765e-10, + "logits/chosen": -2.36369252204895, + "logits/rejected": -2.4254086017608643, + "logps/chosen": -0.2668394148349762, + "logps/rejected": -0.2654018998146057, + "loss": 0.3814314007759094, + "loss/core": 0.3395002484321594, + "loss/preference_sft": 0.26231107115745544, + "loss/reference_anchor": 0.39308053255081177, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.8666539192199707, + "rewards/margins": 1.5065144300460815, + "rewards/rejected": 1.3601396083831787, + "ronpo/logit": 0.15065142512321472, + "ronpo/residual": 0.15467844903469086, + "ronpo/residual_abs": 0.1781061887741089, + "ronpo/target": -0.004027015529572964, + "ronpo/target_abs": 0.05107799917459488, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 840 + }, + { + "drift/chosen_abs": 0.29828858375549316, + "drift/rejected_abs": 0.1848953515291214, + "epoch": 0.2685736988478347, + "grad_norm": 5.15625, + "learning_rate": 7.426068431000881e-10, + "logits/chosen": -2.143845558166504, + "logits/rejected": -2.135876178741455, + "logps/chosen": -0.24773220717906952, + "logps/rejected": -0.2345115840435028, + "loss": 0.10221978276968002, + "loss/core": 0.0692344605922699, + "loss/preference_sft": 0.23999452590942383, + "loss/reference_anchor": 0.3058537244796753, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.9807627201080322, + "rewards/margins": 1.1323689222335815, + "rewards/rejected": 1.8483936786651611, + "ronpo/logit": 0.11323690414428711, + "ronpo/residual": 0.10838621854782104, + "ronpo/residual_abs": 0.13771787285804749, + "ronpo/target": 0.004850681871175766, + "ronpo/target_abs": 0.0496140718460083, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 845 + }, + { + "drift/chosen_abs": 0.41046634316444397, + "drift/rejected_abs": 0.1459590494632721, + "epoch": 0.2701628923321414, + "grad_norm": 80.5, + "learning_rate": 6.164419872871834e-10, + "logits/chosen": -1.9537031650543213, + "logits/rejected": -1.8872268199920654, + "logps/chosen": -0.2728812098503113, + "logps/rejected": -0.28422796726226807, + "loss": 0.9486082196235657, + "loss/core": 0.8727816343307495, + "loss/preference_sft": 0.26899510622024536, + "loss/reference_anchor": 0.7313664555549622, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 4.103553771972656, + "rewards/margins": 2.6454689502716064, + "rewards/rejected": 1.4580847024917603, + "ronpo/logit": 0.2645469009876251, + "ronpo/residual": 0.2555916905403137, + "ronpo/residual_abs": 0.3104216456413269, + "ronpo/target": 0.008955182507634163, + "ronpo/target_abs": 0.05471789091825485, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 850 + }, + { + "drift/chosen_abs": 0.20613236725330353, + "drift/rejected_abs": 0.0725497379899025, + "epoch": 0.27175208581644816, + "grad_norm": 2.890625, + "learning_rate": 5.018823844792602e-10, + "logits/chosen": -2.3794169425964355, + "logits/rejected": -2.366502046585083, + "logps/chosen": -0.2781080901622772, + "logps/rejected": -0.2757300138473511, + "loss": 0.4294048845767975, + "loss/core": 0.4002324938774109, + "loss/preference_sft": 0.27767062187194824, + "loss/reference_anchor": 0.26395726203918457, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.061227798461914, + "rewards/margins": 1.3357303142547607, + "rewards/rejected": 0.7254973649978638, + "ronpo/logit": 0.1335730254650116, + "ronpo/residual": 0.1437883824110031, + "ronpo/residual_abs": 0.17174139618873596, + "ronpo/target": -0.010215363465249538, + "ronpo/target_abs": 0.05027645081281662, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 855 + }, + { + "drift/chosen_abs": 0.1879555881023407, + "drift/rejected_abs": 0.10524116456508636, + "epoch": 0.27334127930075486, + "grad_norm": 46.5, + "learning_rate": 3.9898255884272814e-10, + "logits/chosen": -2.2028138637542725, + "logits/rejected": -2.1289145946502686, + "logps/chosen": -0.2808927893638611, + "logps/rejected": -0.2781389653682709, + "loss": 0.13080891966819763, + "loss/core": 0.11396379768848419, + "loss/preference_sft": 0.2718280255794525, + "loss/reference_anchor": 0.1412682831287384, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.8792381286621094, + "rewards/margins": 0.8328661918640137, + "rewards/rejected": 1.0463716983795166, + "ronpo/logit": 0.08328662812709808, + "ronpo/residual": 0.07004024088382721, + "ronpo/residual_abs": 0.1364951878786087, + "ronpo/target": 0.013246382586658001, + "ronpo/target_abs": 0.05126240849494934, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 860 + }, + { + "drift/chosen_abs": 0.24820244312286377, + "drift/rejected_abs": 0.12920786440372467, + "epoch": 0.2749304727850616, + "grad_norm": 0.474609375, + "learning_rate": 3.077914851215585e-10, + "logits/chosen": -2.0905251502990723, + "logits/rejected": -2.0355658531188965, + "logps/chosen": -0.2801133990287781, + "logps/rejected": -0.2894144058227539, + "loss": 0.11979007720947266, + "loss/core": 0.09946328401565552, + "loss/preference_sft": 0.2898562550544739, + "loss/reference_anchor": 0.1742822378873825, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.481431484222412, + "rewards/margins": 1.1907166242599487, + "rewards/rejected": 1.2907148599624634, + "ronpo/logit": 0.11907164752483368, + "ronpo/residual": 0.11746648699045181, + "ronpo/residual_abs": 0.15590353310108185, + "ronpo/target": 0.0016051754355430603, + "ronpo/target_abs": 0.046399109065532684, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 865 + }, + { + "drift/chosen_abs": 0.46365779638290405, + "drift/rejected_abs": 0.21867212653160095, + "epoch": 0.2765196662693683, + "grad_norm": 536.0, + "learning_rate": 2.283525653279439e-10, + "logits/chosen": -2.0990447998046875, + "logits/rejected": -2.087538003921509, + "logps/chosen": -0.2689060866832733, + "logps/rejected": -0.27739399671554565, + "loss": 0.7828746438026428, + "loss/core": 0.6796854734420776, + "loss/preference_sft": 0.2810099124908447, + "loss/reference_anchor": 1.0037904977798462, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.635507583618164, + "rewards/margins": 2.44916033744812, + "rewards/rejected": 2.186347484588623, + "ronpo/logit": 0.24491600692272186, + "ronpo/residual": 0.24868957698345184, + "ronpo/residual_abs": 0.2878829836845398, + "ronpo/target": -0.003773549571633339, + "ronpo/target_abs": 0.04606860131025314, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 870 + }, + { + "drift/chosen_abs": 0.3057054877281189, + "drift/rejected_abs": 0.153896301984787, + "epoch": 0.278108859753675, + "grad_norm": 6.3125, + "learning_rate": 1.6070360808531359e-10, + "logits/chosen": -1.9459526538848877, + "logits/rejected": -1.9143660068511963, + "logps/chosen": -0.2711811065673828, + "logps/rejected": -0.27196747064590454, + "loss": 0.2286219298839569, + "loss/core": 0.1964854896068573, + "loss/preference_sft": 0.2671479284763336, + "loss/reference_anchor": 0.2946498394012451, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.0563113689422607, + "rewards/margins": 1.5175580978393555, + "rewards/rejected": 1.5387529134750366, + "ronpo/logit": 0.15175580978393555, + "ronpo/residual": 0.14992663264274597, + "ronpo/residual_abs": 0.18774335086345673, + "ronpo/target": 0.0018291939049959183, + "ronpo/target_abs": 0.049851469695568085, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 875 + }, + { + "drift/chosen_abs": 0.24419137835502625, + "drift/rejected_abs": 0.09081504493951797, + "epoch": 0.27969805323798175, + "grad_norm": 5.125, + "learning_rate": 1.0487681063345854e-10, + "logits/chosen": -2.1151018142700195, + "logits/rejected": -2.156067371368408, + "logps/chosen": -0.2783779501914978, + "logps/rejected": -0.2911151945590973, + "loss": 0.4191954731941223, + "loss/core": 0.391798198223114, + "loss/preference_sft": 0.2866339087486267, + "loss/reference_anchor": 0.2453095018863678, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.4419138431549072, + "rewards/margins": 1.5502523183822632, + "rewards/rejected": 0.8916614651679993, + "ronpo/logit": 0.15502521395683289, + "ronpo/residual": 0.1629984974861145, + "ronpo/residual_abs": 0.1879701018333435, + "ronpo/target": -0.00797327235341072, + "ronpo/target_abs": 0.055687934160232544, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 880 + }, + { + "drift/chosen_abs": 0.2644875645637512, + "drift/rejected_abs": 0.10445030778646469, + "epoch": 0.28128724672228844, + "grad_norm": 9.0625, + "learning_rate": 6.089874350439506e-11, + "logits/chosen": -2.162407636642456, + "logits/rejected": -2.2234578132629395, + "logps/chosen": -0.2983078062534332, + "logps/rejected": -0.2950352132320404, + "loss": 0.42109808325767517, + "loss/core": 0.3663472533226013, + "loss/preference_sft": 0.2981397211551666, + "loss/reference_anchor": 0.5176942348480225, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.643937110900879, + "rewards/margins": 1.6017318964004517, + "rewards/rejected": 1.0422050952911377, + "ronpo/logit": 0.1601731926202774, + "ronpo/residual": 0.1630907952785492, + "ronpo/residual_abs": 0.18677613139152527, + "ronpo/target": -0.0029176094103604555, + "ronpo/target_abs": 0.04585547372698784, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 885 + }, + { + "drift/chosen_abs": 0.29927030205726624, + "drift/rejected_abs": 0.12154443562030792, + "epoch": 0.28287644020659514, + "grad_norm": 3.453125, + "learning_rate": 2.8790337876233307e-11, + "logits/chosen": -2.306915760040283, + "logits/rejected": -2.2651777267456055, + "logps/chosen": -0.28179579973220825, + "logps/rejected": -0.28089454770088196, + "loss": 0.43596991896629333, + "loss/core": 0.38446953892707825, + "loss/preference_sft": 0.28348347544670105, + "loss/reference_anchor": 0.48665523529052734, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.9908390045166016, + "rewards/margins": 1.7791423797607422, + "rewards/rejected": 1.2116966247558594, + "ronpo/logit": 0.17791421711444855, + "ronpo/residual": 0.17613449692726135, + "ronpo/residual_abs": 0.2077271044254303, + "ronpo/target": 0.0017797171603888273, + "ronpo/target_abs": 0.053874753415584564, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 890 + }, + { + "drift/chosen_abs": 0.3066047728061676, + "drift/rejected_abs": 0.12381645292043686, + "epoch": 0.2844656336909019, + "grad_norm": 10.3125, + "learning_rate": 8.566875611068502e-12, + "logits/chosen": -2.080899477005005, + "logits/rejected": -2.0696678161621094, + "logps/chosen": -0.28545522689819336, + "logps/rejected": -0.2945188283920288, + "loss": 0.39896756410598755, + "loss/core": 0.35809260606765747, + "loss/preference_sft": 0.28190794587135315, + "loss/reference_anchor": 0.38055914640426636, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.0644736289978027, + "rewards/margins": 1.9215772151947021, + "rewards/rejected": 1.142896294593811, + "ronpo/logit": 0.19215771555900574, + "ronpo/residual": 0.1853954941034317, + "ronpo/residual_abs": 0.21138229966163635, + "ronpo/target": 0.0067622363567352295, + "ronpo/target_abs": 0.051556866616010666, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 895 + }, + { + "drift/chosen_abs": 0.1592852622270584, + "drift/rejected_abs": 0.084005206823349, + "epoch": 0.2860548271752086, + "grad_norm": 17.75, + "learning_rate": 2.379819816378248e-13, + "logits/chosen": -2.1036276817321777, + "logits/rejected": -2.0870792865753174, + "logps/chosen": -0.31869596242904663, + "logps/rejected": -0.31833693385124207, + "loss": 0.14820127189159393, + "loss/core": 0.13302530348300934, + "loss/preference_sft": 0.31609418988227844, + "loss/reference_anchor": 0.12015028297901154, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.5925084352493286, + "rewards/margins": 0.8815566897392273, + "rewards/rejected": 0.7109516859054565, + "ronpo/logit": 0.08815567195415497, + "ronpo/residual": 0.09216871112585068, + "ronpo/residual_abs": 0.1262686550617218, + "ronpo/target": -0.00401304429396987, + "ronpo/target_abs": 0.050638485699892044, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 900 + }, + { + "epoch": 0.2860548271752086, + "step": 900, + "total_flos": 0.0, + "train_loss": 0.4462373371918996, + "train_runtime": 7238.9345, + "train_samples_per_second": 1.989, + "train_steps_per_second": 0.124 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..7668f9e --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51539858b1e537f64c01ce29eab4f3d24d92761aace55148017c1e0c1dcf2a97 +size 6993 diff --git a/training_status.json b/training_status.json new file mode 100644 index 0000000..d82bb3d --- /dev/null +++ b/training_status.json @@ -0,0 +1,20 @@ +{ + "status": "completed", + "candidate_id": "ronpo_top_a", + "method": "ronpo_k_only", + "seed": 42, + "gpu": 2, + "pid": 90276, + "config": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/fair_demo_20260715/sweep/candidates/ronpo_top_a/config.yaml", + "dataset": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/fair_demo_20260715/precomputed/ronpo_k_only_k1", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "b8ffed8fc967", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/b8ffed8fc967", + "started_at": "2026-07-15T09:31:12+09:00", + "returncode": 0, + "model_complete": true, + "finite_results": true, + "measured_step": null, + "completed_at": "2026-07-15T11:35:44+09:00" +}