commit 502a45182e6f425d0ed8adb6982436321a043665 Author: ModelHub XC Date: Sat Jul 18 11:51:09 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: promotion/ronpo-qwen3-8b-fair-ht-mnpo-safety-s42 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..92ab974 --- /dev/null +++ b/README.md @@ -0,0 +1,12 @@ +--- +base_model: Qwen/Qwen3-8B +library_name: transformers +tags: +- preference-optimization +- ronpo +- qwen3 +--- + +# Qwen3-8B fair-demo checkpoint: ht_mnpo_safety + +Validation-selected candidate: `ht_safety_a`. Exact evaluator, selection, sweep, and training metadata are stored under `experiment/`. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..afdacef --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8589835361488904, + "total_flos": 0.0, + "train_loss": 0.36845981503112446, + "train_runtime": 7265.877, + "train_samples": 16764, + "train_samples_per_second": 1.982, + "train_steps_per_second": 0.124 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..d207dbf --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +model_name_or_path: /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218 +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/ht_mnpo_safety: 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: ht_mnpo +eta: 0.005 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.02 +preference_sft_weight: 0.002 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 2.5e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 42 +gradient_accumulation_steps: 16 +gradient_checkpointing: true +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 1 +per_device_eval_batch_size: 1 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/fair_demo_20260715/sweep/candidates/ht_safety_a +run_name: qwen3-8b-fair-demo-ht_safety_a diff --git a/experiment/evaluator_lock.json b/experiment/evaluator_lock.json new file mode 100644 index 0000000..b406c9a --- /dev/null +++ b/experiment/evaluator_lock.json @@ -0,0 +1,85 @@ +{ + "status": "LOCKED_BEFORE_ANY_NEW_METHOD_RANKING", + "locked_at": "2026-07-15T20:20:29+09:00", + "objective_signals": [ + "skywork", + "armo_safety", + "length_conciseness" + ], + "objective_semantics": [ + "helpfulness", + "safety", + "conciseness" + ], + "primary": { + "name": "open_weight_panel_mean_prompt_worst_vs_base", + "definition": "For each prompt and objective, average win=1/tie=0.5/loss=0 over two A/B positions and two judges; take the minimum objective, then mean over prompts.", + "judges": [ + { + "model": "openai/gpt-oss-120b", + "revision": "b5c939de8f754692c1647ca79fbf85e8c1e70f8a" + }, + { + "model": "Qwen/Qwen3-32B", + "revision": "9216db5781bf21249d130ec9da846c4624c16137" + } + ], + "decode": { + "temperature": 0.0, + "top_p": 1.0, + "max_new_tokens": 512, + "seed": 42 + }, + "position_swap": true, + "validation_selection_rule": "Highest eligible mean prompt-level worst panel score within each method; an exact numeric tie is broken by candidate_id lexical order." + }, + "secondary": { + "normalization": "For each locked signal, divide each evaluation prompt's candidate-minus-base raw delta by the population SD of the diagnostic base and matched-control scores; take the prompt-level minimum across objectives, then the mean. No per-prompt min-max.", + "diagnostic_control_scale": { + "skywork": 22.807301785782975, + "armo_safety": 0.3098083353203591, + "length_conciseness": 1.5891969646897472 + }, + "reward_signal_provenance": { + "skywork": { + "model": "Skywork/Skywork-Reward-V2-Llama-3.1-8B", + "revision": "cba2f842f3f1af2f1b2f0d35e794d789976390c5", + "semantics": "helpfulness" + }, + "armo_safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "revision": "eb2676d20da2f2d41082289d23c59b9f7427f955", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "length_conciseness": { + "deterministic": "-log1p(response_word_count)", + "tokenization": "Python str.split whitespace words" + } + }, + "report_raw_paired_deltas": true + }, + "bootstrap": { + "paired_prompt_resamples": 2000, + "seed": 42, + "interval": "percentile_95" + }, + "power": { + "target_absolute_effect": 0.05, + "paired_sd": 0.13831629563357775, + "required_prompts_80pct_power": 61, + "planned_fresh_test_prompts": 1024, + "alpha_two_sided": 0.05, + "power": 0.8 + }, + "fresh_test_source": { + "dataset": "HuggingFaceH4/ultrachat_200k", + "revision": "8049631c405ae6576f93f445c6b8166f76f5505a", + "split": "test_sft" + }, + "diagnostic_summary_sha256": "e365ed814926ac5c17777e0fc65d0299a45b81bd390542889bbd55b5453d4ffe", + "judge_diagnostic_lock_sha256": "fce2d97806053cabdf3d358806f4baa80df7f89d71d8c1c298883327d4e02635", + "prereg_sha256": "71cbd1ac17c3e807f2ece1cb0a82374bbe74edb654c2845b97c3222ed680f0d0", + "method_ranking_computed": false, + "spent_sealed_split_touched": false +} diff --git a/experiment/selection_lock.json b/experiment/selection_lock.json new file mode 100644 index 0000000..bae8b9c --- /dev/null +++ b/experiment/selection_lock.json @@ -0,0 +1,99 @@ +{ + "status": "VALIDATION_SELECTION_LOCKED_BEFORE_FRESH_TEST", + "locked_at": "2026-07-15T20:37:11+09:00", + "selection_metric": "open_weight_panel_mean_prompt_worst_vs_base", + "tie_break": "candidate_id lexical order", + "selected_by_method": { + "dpo": { + "candidate_id": "dpo_b", + "validation_primary": 0.40625, + "validation_primary_ci95": [ + 0.3837890625, + 0.427734375 + ], + "eligible_candidates": [ + "dpo_a", + "dpo_b" + ] + }, + "ht_mnpo_helpfulness": { + "candidate_id": "ht_help_b", + "validation_primary": 0.4072265625, + "validation_primary_ci95": [ + 0.3828125, + 0.4296875 + ], + "eligible_candidates": [ + "ht_help_b" + ] + }, + "ht_mnpo_safety": { + "candidate_id": "ht_safety_a", + "validation_primary": 0.4150390625, + "validation_primary_ci95": [ + 0.3925537109375, + 0.4384765625 + ], + "eligible_candidates": [ + "ht_safety_a", + "ht_safety_b" + ] + }, + "inpo_avg": { + "candidate_id": "inpo_b", + "validation_primary": 0.40625, + "validation_primary_ci95": [ + 0.3818359375, + 0.427734375 + ], + "eligible_candidates": [ + "inpo_b" + ] + }, + "ronpo_full_expect": { + "candidate_id": "ronpo_full_a", + "validation_primary": 0.4033203125, + "validation_primary_ci95": [ + 0.37890625, + 0.42580566406249987 + ], + "eligible_candidates": [ + "ronpo_full_a", + "ronpo_full_b" + ] + }, + "ronpo_k_only": { + "candidate_id": "ronpo_top_a", + "validation_primary": 0.412109375, + "validation_primary_ci95": [ + 0.3876953125, + 0.43557128906249987 + ], + "eligible_candidates": [ + "ronpo_top_a" + ] + }, + "sppo_avg": { + "candidate_id": "sppo_b", + "validation_primary": 0.4111328125, + "validation_primary_ci95": [ + 0.3866943359375, + 0.4345703125 + ], + "eligible_candidates": [ + "sppo_b" + ] + } + }, + "selected_ronpo_overall": "ronpo_top_a", + "failed_methods": [ + "ht_mnpo_conciseness", + "ipo", + "simpo" + ], + "panel_summary_sha256": "8850f23e16cadf56a99e3d562d224ce5337ab36fa588d8a843382853e6ad17bd", + "evaluator_lock_sha256": "ab7b12c84932ed57acfec7edbcbc1ff1df5e9acd203869b1458343adf6e5b2c8", + "grid_sha256": "abfd982ef10c3bc71364752241150286ebf06be5e97f087596adf80e55d459e0", + "fresh_test_opened": false, + "spent_sealed_split_touched": false +} diff --git a/experiment/sweep_grid.json b/experiment/sweep_grid.json new file mode 100644 index 0000000..1ee759f --- /dev/null +++ b/experiment/sweep_grid.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "status": "frozen_before_training_and_validation_ranking", + "frozen_at_kst": "2026-07-15T09:01:45+09:00", + "seed": 42, + "common": { + "optimizer_steps": 900, + "effective_batch_size": 16, + "per_device_train_batch_size": 1, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "bf16": true, + "attn_implementation": "sdpa", + "cudnn_sdpa": false, + "lr_scheduler_type": "cosine", + "max_length": 2048, + "max_prompt_length": 1800, + "save_total_limit": 1, + "wandb_entity": "promotion-kim", + "wandb_project": "mnpo", + "wandb_group": "qwen3-8b-fair-demo-20260715" + }, + "budget_rule": "Exactly two configs per reported method; no extension after validation rankings are visible.", + "candidates": [ + {"id":"ronpo_full_a","method":"ronpo_full_expect","dataset":"ronpo_full_expect_kall","learning_rate":1e-7,"warmup_ratio":0.15,"ronpo_alpha":0.25,"ronpo_tau":0.10,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075,"expected_support_k":0}, + {"id":"ronpo_full_b","method":"ronpo_full_expect","dataset":"ronpo_full_expect_k6","learning_rate":2.5e-7,"warmup_ratio":0.20,"ronpo_alpha":0.35,"ronpo_tau":0.05,"reference_anchor_weight":0.10,"preference_sft_weight":0.01,"expected_support_k":6}, + {"id":"ronpo_top_a","method":"ronpo_k_only","dataset":"ronpo_k_only_k1","learning_rate":5e-8,"warmup_ratio":0.20,"ronpo_alpha":0.15,"ronpo_tau":0.10,"reference_anchor_weight":0.10,"preference_sft_weight":0.01,"expected_support_k":1}, + {"id":"ronpo_top_b","method":"ronpo_k_only","dataset":"ronpo_k_only_k2","learning_rate":1e-7,"warmup_ratio":0.15,"ronpo_alpha":0.25,"ronpo_tau":0.10,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075,"expected_support_k":2}, + {"id":"dpo_a","method":"dpo","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.10,"dpo_beta":0.05,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"dpo_b","method":"dpo","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.15,"dpo_beta":0.10,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ipo_a","method":"ipo","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.15,"dpo_beta":0.20,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ipo_b","method":"ipo","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.20,"dpo_beta":0.50,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075}, + {"id":"simpo_a","method":"simpo","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.10,"simpo_beta":1.0,"simpo_gamma":0.3,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"simpo_b","method":"simpo","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.15,"simpo_beta":2.0,"simpo_gamma":0.6,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"sppo_a","method":"sppo_avg","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.15,"eta":0.005,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"sppo_b","method":"sppo_avg","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.20,"eta":0.010,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075}, + {"id":"inpo_a","method":"inpo_avg","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.15,"eta":0.050,"ratio":0.3333,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"inpo_b","method":"inpo_avg","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.20,"eta":0.100,"ratio":0.3333,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075}, + {"id":"ht_help_a","method":"ht_mnpo_helpfulness","dataset":"ht_mnpo_helpfulness","learning_rate":2.5e-7,"warmup_ratio":0.10,"eta":0.005,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"ht_help_b","method":"ht_mnpo_helpfulness","dataset":"ht_mnpo_helpfulness","learning_rate":5e-7,"warmup_ratio":0.15,"eta":0.010,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ht_safety_a","method":"ht_mnpo_safety","dataset":"ht_mnpo_safety","learning_rate":2.5e-7,"warmup_ratio":0.10,"eta":0.005,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"ht_safety_b","method":"ht_mnpo_safety","dataset":"ht_mnpo_safety","learning_rate":5e-7,"warmup_ratio":0.15,"eta":0.010,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ht_concise_a","method":"ht_mnpo_conciseness","dataset":"ht_mnpo_conciseness","learning_rate":2.5e-7,"warmup_ratio":0.10,"eta":0.005,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"ht_concise_b","method":"ht_mnpo_conciseness","dataset":"ht_mnpo_conciseness","learning_rate":5e-7,"warmup_ratio":0.15,"eta":0.010,"reference_anchor_weight":0.05,"preference_sft_weight":0.005} + ] +} diff --git a/experiment/training_status.json b/experiment/training_status.json new file mode 100644 index 0000000..1d86ad9 --- /dev/null +++ b/experiment/training_status.json @@ -0,0 +1,20 @@ +{ + "status": "completed", + "candidate_id": "ht_safety_a", + "method": "ht_mnpo_safety", + "seed": 42, + "gpu": 1, + "pid": 1096653, + "config": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/fair_demo_20260715/sweep/candidates/ht_safety_a/config.yaml", + "dataset": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/ht_mnpo_safety", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "eec5f721eea6", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/eec5f721eea6", + "started_at": "2026-07-15T15:42:04+09:00", + "returncode": 0, + "model_complete": true, + "finite_results": true, + "measured_step": null, + "completed_at": "2026-07-15T17:44:49+09:00" +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..c3c9cbd --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab4b101558e7cf151ff1f55a974844122cee0d444a2cf90aa87f4c9ca1935b12 +size 16381517208 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..e2cb12d --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": true, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..afdacef --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8589835361488904, + "total_flos": 0.0, + "train_loss": 0.36845981503112446, + "train_runtime": 7265.877, + "train_samples": 16764, + "train_samples_per_second": 1.982, + "train_steps_per_second": 0.124 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..77f5d44 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,4543 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8589835361488904, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "drift/chosen_abs": 0.22921419143676758, + "drift/rejected_abs": 0.10946719348430634, + "epoch": 0.004772130756382725, + "grad_norm": 8.6875, + "ht_mnpo/logit": 0.12133613973855972, + "ht_mnpo/residual": 0.1163361445069313, + "ht_mnpo/residual_abs": 0.15141648054122925, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.111111111111111e-08, + "logits/chosen": -2.3204128742218018, + "logits/rejected": -2.2550835609436035, + "logps/chosen": -0.29364365339279175, + "logps/rejected": -0.2911434769630432, + "loss": 0.16518738865852356, + "loss/core": 0.1614948809146881, + "loss/preference_sft": 0.29364365339279175, + "loss/reference_anchor": 0.15526053309440613, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.292141914367676, + "rewards/margins": 1.2133615016937256, + "rewards/rejected": 1.0787802934646606, + "step": 5 + }, + { + "drift/chosen_abs": 0.3288218379020691, + "drift/rejected_abs": 0.1325603723526001, + "epoch": 0.00954426151276545, + "grad_norm": 892.0, + "ht_mnpo/logit": 0.19652661681175232, + "ht_mnpo/residual": 0.1915266215801239, + "ht_mnpo/residual_abs": 0.20962563157081604, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.5e-08, + "logits/chosen": -2.04286527633667, + "logits/rejected": -1.97250235080719, + "logps/chosen": -0.27547430992126465, + "logps/rejected": -0.2841722369194031, + "loss": 0.6829333901405334, + "loss/core": 0.6688618659973145, + "loss/preference_sft": 0.27547430992126465, + "loss/reference_anchor": 0.6760290861129761, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.287240982055664, + "rewards/margins": 1.9652659893035889, + "rewards/rejected": 1.321974754333496, + "step": 10 + }, + { + "drift/chosen_abs": 0.3020998537540436, + "drift/rejected_abs": 0.08261941373348236, + "epoch": 0.014316392269148175, + "grad_norm": 2.203125, + "ht_mnpo/logit": 0.2266816794872284, + "ht_mnpo/residual": 0.22168168425559998, + "ht_mnpo/residual_abs": 0.23883593082427979, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 3.888888888888889e-08, + "logits/chosen": -2.175687551498413, + "logits/rejected": -2.1948294639587402, + "logps/chosen": -0.3126169443130493, + "logps/rejected": -0.3139331340789795, + "loss": 0.7264648079872131, + "loss/core": 0.7176737189292908, + "loss/preference_sft": 0.3126169443130493, + "loss/reference_anchor": 0.4082910120487213, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.0202560424804688, + "rewards/margins": 2.2668166160583496, + "rewards/rejected": 0.7534393072128296, + "step": 15 + }, + { + "drift/chosen_abs": 0.28461581468582153, + "drift/rejected_abs": 0.1129002794623375, + "epoch": 0.0190885230255309, + "grad_norm": 174.0, + "ht_mnpo/logit": 0.1719159036874771, + "ht_mnpo/residual": 0.1669158935546875, + "ht_mnpo/residual_abs": 0.18884992599487305, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 5.2777777777777776e-08, + "logits/chosen": -2.1989941596984863, + "logits/rejected": -2.175231456756592, + "logps/chosen": -0.31303125619888306, + "logps/rejected": -0.3244328498840332, + "loss": 0.6520941853523254, + "loss/core": 0.6402677297592163, + "loss/preference_sft": 0.31303125619888306, + "loss/reference_anchor": 0.5600210428237915, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.845700740814209, + "rewards/margins": 1.7191591262817383, + "rewards/rejected": 1.1265417337417603, + "step": 20 + }, + { + "drift/chosen_abs": 0.2910028398036957, + "drift/rejected_abs": 0.1626802384853363, + "epoch": 0.023860653781913623, + "grad_norm": 1.5625, + "ht_mnpo/logit": 0.12845146656036377, + "ht_mnpo/residual": 0.12345147132873535, + "ht_mnpo/residual_abs": 0.14755485951900482, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 6.666666666666667e-08, + "logits/chosen": -2.1143784523010254, + "logits/rejected": -2.0981764793395996, + "logps/chosen": -0.3009456694126129, + "logps/rejected": -0.2978978455066681, + "loss": 0.28243863582611084, + "loss/core": 0.2706168591976166, + "loss/preference_sft": 0.3009456694126129, + "loss/reference_anchor": 0.5609945058822632, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.9088892936706543, + "rewards/margins": 1.2845146656036377, + "rewards/rejected": 1.6243746280670166, + "step": 25 + }, + { + "drift/chosen_abs": 0.47175097465515137, + "drift/rejected_abs": 0.252032995223999, + "epoch": 0.02863278453829635, + "grad_norm": 13.1875, + "ht_mnpo/logit": 0.2227689027786255, + "ht_mnpo/residual": 0.21776887774467468, + "ht_mnpo/residual_abs": 0.35318318009376526, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 8.055555555555555e-08, + "logits/chosen": -1.7267528772354126, + "logits/rejected": -1.8169645071029663, + "logps/chosen": -0.2687901258468628, + "logps/rejected": -0.24951688945293427, + "loss": 1.0632941722869873, + "loss/core": 1.041975498199463, + "loss/preference_sft": 0.2687901258468628, + "loss/reference_anchor": 1.039062738418579, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 4.71492862701416, + "rewards/margins": 2.227689266204834, + "rewards/rejected": 2.4872403144836426, + "step": 30 + }, + { + "drift/chosen_abs": 0.17663131654262543, + "drift/rejected_abs": 0.24685576558113098, + "epoch": 0.03340491529467907, + "grad_norm": 147.0, + "ht_mnpo/logit": -0.069961316883564, + "ht_mnpo/residual": -0.07496131211519241, + "ht_mnpo/residual_abs": 0.25241637229919434, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 9.444444444444444e-08, + "logits/chosen": -2.023242712020874, + "logits/rejected": -1.947217345237732, + "logps/chosen": -0.30046090483665466, + "logps/rejected": -0.2926713526248932, + "loss": 0.7559306025505066, + "loss/core": 0.745723307132721, + "loss/preference_sft": 0.30046090483665466, + "loss/reference_anchor": 0.4803217053413391, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.7663133144378662, + "rewards/margins": -0.6996129751205444, + "rewards/rejected": 2.4659264087677, + "step": 35 + }, + { + "drift/chosen_abs": 0.2406160533428192, + "drift/rejected_abs": 0.17416922748088837, + "epoch": 0.0381770460510618, + "grad_norm": 22.625, + "ht_mnpo/logit": 0.06698594987392426, + "ht_mnpo/residual": 0.06198593229055405, + "ht_mnpo/residual_abs": 0.2546267509460449, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.0833333333333334e-07, + "logits/chosen": -2.0492732524871826, + "logits/rejected": -2.0772995948791504, + "logps/chosen": -0.28532397747039795, + "logps/rejected": -0.3091915547847748, + "loss": 0.7844445109367371, + "loss/core": 0.773983895778656, + "loss/preference_sft": 0.28532397747039795, + "loss/reference_anchor": 0.49450117349624634, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.406160831451416, + "rewards/margins": 0.6698596477508545, + "rewards/rejected": 1.736301064491272, + "step": 40 + }, + { + "drift/chosen_abs": 0.23510071635246277, + "drift/rejected_abs": 0.14974211156368256, + "epoch": 0.04294917680744453, + "grad_norm": 14.125, + "ht_mnpo/logit": 0.08539210259914398, + "ht_mnpo/residual": 0.08039210736751556, + "ht_mnpo/residual_abs": 0.14728103578090668, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.2222222222222222e-07, + "logits/chosen": -1.996901512145996, + "logits/rejected": -1.968118667602539, + "logps/chosen": -0.2885960042476654, + "logps/rejected": -0.28996142745018005, + "loss": 0.1255718469619751, + "loss/core": 0.12194611132144928, + "loss/preference_sft": 0.2885960042476654, + "loss/reference_anchor": 0.15242737531661987, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.3506722450256348, + "rewards/margins": 0.853921115398407, + "rewards/rejected": 1.496751308441162, + "step": 45 + }, + { + "drift/chosen_abs": 0.28709280490875244, + "drift/rejected_abs": 0.0940680131316185, + "epoch": 0.047721307563827246, + "grad_norm": 43.0, + "ht_mnpo/logit": 0.19577643275260925, + "ht_mnpo/residual": 0.19077643752098083, + "ht_mnpo/residual_abs": 0.19558581709861755, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.3611111111111108e-07, + "logits/chosen": -2.2561166286468506, + "logits/rejected": -2.199009895324707, + "logps/chosen": -0.271379679441452, + "logps/rejected": -0.2756428122520447, + "loss": 0.4556834101676941, + "loss/core": 0.44865885376930237, + "loss/preference_sft": 0.271379679441452, + "loss/reference_anchor": 0.3240897059440613, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.870856761932373, + "rewards/margins": 1.9577643871307373, + "rewards/rejected": 0.9130924940109253, + "step": 50 + }, + { + "drift/chosen_abs": 0.22801081836223602, + "drift/rejected_abs": 0.13098636269569397, + "epoch": 0.05249343832020997, + "grad_norm": 14.6875, + "ht_mnpo/logit": 0.09714122861623764, + "ht_mnpo/residual": 0.09214121848344803, + "ht_mnpo/residual_abs": 0.10519029945135117, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.5e-07, + "logits/chosen": -2.0955872535705566, + "logits/rejected": -2.0516104698181152, + "logps/chosen": -0.28714004158973694, + "logps/rejected": -0.28851160407066345, + "loss": 0.11324092000722885, + "loss/core": 0.10730179399251938, + "loss/preference_sft": 0.28714004158973694, + "loss/reference_anchor": 0.2682426869869232, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.280061721801758, + "rewards/margins": 0.9714123010635376, + "rewards/rejected": 1.3086493015289307, + "step": 55 + }, + { + "drift/chosen_abs": 0.2036731243133545, + "drift/rejected_abs": 0.25291842222213745, + "epoch": 0.0572655690765927, + "grad_norm": 608.0, + "ht_mnpo/logit": -0.04795291647315025, + "ht_mnpo/residual": -0.05295292288064957, + "ht_mnpo/residual_abs": 0.25708532333374023, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.6388888888888888e-07, + "logits/chosen": -2.219554901123047, + "logits/rejected": -2.147869348526001, + "logps/chosen": -0.30570459365844727, + "logps/rejected": -0.30410706996917725, + "loss": 0.8895317912101746, + "loss/core": 0.877190887928009, + "loss/preference_sft": 0.30570459365844727, + "loss/reference_anchor": 0.5864753127098083, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.036237955093384, + "rewards/margins": -0.47952914237976074, + "rewards/rejected": 2.5157668590545654, + "step": 60 + }, + { + "drift/chosen_abs": 0.22737295925617218, + "drift/rejected_abs": 0.12715181708335876, + "epoch": 0.062037699832975426, + "grad_norm": 2.8125, + "ht_mnpo/logit": 0.10005078464746475, + "ht_mnpo/residual": 0.09505076706409454, + "ht_mnpo/residual_abs": 0.13454630970954895, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.7777777777777776e-07, + "logits/chosen": -2.095898389816284, + "logits/rejected": -2.160431385040283, + "logps/chosen": -0.30385154485702515, + "logps/rejected": -0.2779307961463928, + "loss": 0.1357956975698471, + "loss/core": 0.13056448101997375, + "loss/preference_sft": 0.30385154485702515, + "loss/reference_anchor": 0.23117490112781525, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.271531581878662, + "rewards/margins": 1.0005077123641968, + "rewards/rejected": 1.2710237503051758, + "step": 65 + }, + { + "drift/chosen_abs": 0.1965130865573883, + "drift/rejected_abs": 0.19766323268413544, + "epoch": 0.06680983058935815, + "grad_norm": 50.25, + "ht_mnpo/logit": 0.0027785298880189657, + "ht_mnpo/residual": -0.0022214693017303944, + "ht_mnpo/residual_abs": 0.1599881798028946, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.9166666666666668e-07, + "logits/chosen": -2.147693395614624, + "logits/rejected": -2.035130739212036, + "logps/chosen": -0.28254345059394836, + "logps/rejected": -0.28443124890327454, + "loss": 0.12861734628677368, + "loss/core": 0.1251140534877777, + "loss/preference_sft": 0.28254345059394836, + "loss/reference_anchor": 0.14691027998924255, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.9649995565414429, + "rewards/margins": 0.027785325422883034, + "rewards/rejected": 1.9372141361236572, + "step": 70 + }, + { + "drift/chosen_abs": 0.31567615270614624, + "drift/rejected_abs": 0.1452103555202484, + "epoch": 0.07158196134574088, + "grad_norm": 808.0, + "ht_mnpo/logit": 0.17037446796894073, + "ht_mnpo/residual": 0.16537444293498993, + "ht_mnpo/residual_abs": 0.20655925571918488, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.0555555555555553e-07, + "logits/chosen": -2.248660087585449, + "logits/rejected": -2.147256374359131, + "logps/chosen": -0.2782396078109741, + "logps/rejected": -0.29824963212013245, + "loss": 0.590999960899353, + "loss/core": 0.5754934549331665, + "loss/preference_sft": 0.2782396078109741, + "loss/reference_anchor": 0.7474981546401978, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.155601978302002, + "rewards/margins": 1.7037445306777954, + "rewards/rejected": 1.4518576860427856, + "step": 75 + }, + { + "drift/chosen_abs": 0.2219829261302948, + "drift/rejected_abs": 0.08889742195606232, + "epoch": 0.0763540921021236, + "grad_norm": 3.359375, + "ht_mnpo/logit": 0.13296504318714142, + "ht_mnpo/residual": 0.127965047955513, + "ht_mnpo/residual_abs": 0.14647246897220612, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.1944444444444442e-07, + "logits/chosen": -2.1656582355499268, + "logits/rejected": -2.274348735809326, + "logps/chosen": -0.2902677357196808, + "logps/rejected": -0.2846531271934509, + "loss": 0.25446221232414246, + "loss/core": 0.24999630451202393, + "loss/preference_sft": 0.2902677357196808, + "loss/reference_anchor": 0.19426903128623962, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.2158241271972656, + "rewards/margins": 1.3296505212783813, + "rewards/rejected": 0.8861739039421082, + "step": 80 + }, + { + "drift/chosen_abs": 0.19277288019657135, + "drift/rejected_abs": 0.09097670018672943, + "epoch": 0.08112622285850632, + "grad_norm": 0.376953125, + "ht_mnpo/logit": 0.10174652189016342, + "ht_mnpo/residual": 0.0967465192079544, + "ht_mnpo/residual_abs": 0.10738085210323334, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.3333333333333333e-07, + "logits/chosen": -2.0793979167938232, + "logits/rejected": -2.14670991897583, + "logps/chosen": -0.30412206053733826, + "logps/rejected": -0.2832191586494446, + "loss": 0.0978093072772026, + "loss/core": 0.09453041851520538, + "loss/preference_sft": 0.30412206053733826, + "loss/reference_anchor": 0.13353168964385986, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.9227644205093384, + "rewards/margins": 1.0174652338027954, + "rewards/rejected": 0.9052993655204773, + "step": 85 + }, + { + "drift/chosen_abs": 0.22163239121437073, + "drift/rejected_abs": 0.12397237122058868, + "epoch": 0.08589835361488905, + "grad_norm": 64.5, + "ht_mnpo/logit": 0.09769080579280853, + "ht_mnpo/residual": 0.09269081801176071, + "ht_mnpo/residual_abs": 0.1690768450498581, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.4722222222222224e-07, + "logits/chosen": -2.1062588691711426, + "logits/rejected": -2.0609793663024902, + "logps/chosen": -0.3209324777126312, + "logps/rejected": -0.2916238605976105, + "loss": 0.5165913701057434, + "loss/core": 0.5093555450439453, + "loss/preference_sft": 0.3209324777126312, + "loss/reference_anchor": 0.329694539308548, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.2147328853607178, + "rewards/margins": 0.9769083261489868, + "rewards/rejected": 1.237824559211731, + "step": 90 + }, + { + "drift/chosen_abs": 0.18284250795841217, + "drift/rejected_abs": 0.15377461910247803, + "epoch": 0.09067048437127177, + "grad_norm": 608.0, + "ht_mnpo/logit": 0.029158195480704308, + "ht_mnpo/residual": 0.02415819838643074, + "ht_mnpo/residual_abs": 0.14915595948696136, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.4998495746616845e-07, + "logits/chosen": -2.258273124694824, + "logits/rejected": -2.348480701446533, + "logps/chosen": -0.3251148760318756, + "logps/rejected": -0.3082412779331207, + "loss": 0.39904260635375977, + "loss/core": 0.38921934366226196, + "loss/preference_sft": 0.3251148760318756, + "loss/reference_anchor": 0.4586527943611145, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.8256721496582031, + "rewards/margins": 0.2915820777416229, + "rewards/rejected": 1.534090280532837, + "step": 95 + }, + { + "drift/chosen_abs": 0.39025211334228516, + "drift/rejected_abs": 0.11058616638183594, + "epoch": 0.09544261512765449, + "grad_norm": 209.0, + "ht_mnpo/logit": 0.2797187268733978, + "ht_mnpo/residual": 0.2747187614440918, + "ht_mnpo/residual_abs": 0.27753618359565735, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.4992385337738696e-07, + "logits/chosen": -1.7920671701431274, + "logits/rejected": -1.848049521446228, + "logps/chosen": -0.30774667859077454, + "logps/rejected": -0.2939962148666382, + "loss": 1.4822406768798828, + "loss/core": 1.461856722831726, + "loss/preference_sft": 0.30774667859077454, + "loss/reference_anchor": 0.9884130358695984, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.900453567504883, + "rewards/margins": 2.797187566757202, + "rewards/rejected": 1.1032661199569702, + "step": 100 + }, + { + "drift/chosen_abs": 0.2572689652442932, + "drift/rejected_abs": 0.16230344772338867, + "epoch": 0.10021474588403723, + "grad_norm": 0.498046875, + "ht_mnpo/logit": 0.09508073329925537, + "ht_mnpo/residual": 0.09008073061704636, + "ht_mnpo/residual_abs": 0.2105640172958374, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.4981577053636144e-07, + "logits/chosen": -2.1005120277404785, + "logits/rejected": -2.1056771278381348, + "logps/chosen": -0.30666694045066833, + "logps/rejected": -0.3211822807788849, + "loss": 0.569188117980957, + "loss/core": 0.5602066516876221, + "loss/preference_sft": 0.30666694045066833, + "loss/reference_anchor": 0.41840967535972595, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.5712063312530518, + "rewards/margins": 0.9508073925971985, + "rewards/rejected": 1.6203992366790771, + "step": 105 + }, + { + "drift/chosen_abs": 0.23388180136680603, + "drift/rejected_abs": 0.10054342448711395, + "epoch": 0.10498687664041995, + "grad_norm": 0.72265625, + "ht_mnpo/logit": 0.13339105248451233, + "ht_mnpo/residual": 0.12839102745056152, + "ht_mnpo/residual_abs": 0.1422457993030548, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.496607495886281e-07, + "logits/chosen": -2.2283363342285156, + "logits/rejected": -2.157383680343628, + "logps/chosen": -0.301920086145401, + "logps/rejected": -0.31350260972976685, + "loss": 0.3251662850379944, + "loss/core": 0.3147370219230652, + "loss/preference_sft": 0.301920086145401, + "loss/reference_anchor": 0.4912687838077545, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.338407039642334, + "rewards/margins": 1.3339102268218994, + "rewards/rejected": 1.0044968128204346, + "step": 110 + }, + { + "drift/chosen_abs": 0.14543868601322174, + "drift/rejected_abs": 0.08579744398593903, + "epoch": 0.10975900739680267, + "grad_norm": 4.03125, + "ht_mnpo/logit": 0.05967400223016739, + "ht_mnpo/residual": 0.05467400699853897, + "ht_mnpo/residual_abs": 0.07688696682453156, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.4945884883122553e-07, + "logits/chosen": -2.228116035461426, + "logits/rejected": -2.1459319591522217, + "logps/chosen": -0.2716502547264099, + "logps/rejected": -0.26580381393432617, + "loss": 0.05459178611636162, + "loss/core": 0.05268437787890434, + "loss/preference_sft": 0.2716502547264099, + "loss/reference_anchor": 0.06820501387119293, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.4522254467010498, + "rewards/margins": 0.5967400074005127, + "rewards/rejected": 0.8554852604866028, + "step": 115 + }, + { + "drift/chosen_abs": 0.22194862365722656, + "drift/rejected_abs": 0.1216927170753479, + "epoch": 0.1145311381531854, + "grad_norm": 23.0, + "ht_mnpo/logit": 0.10122959315776825, + "ht_mnpo/residual": 0.09622960537672043, + "ht_mnpo/residual_abs": 0.12579123675823212, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.492101441907714e-07, + "logits/chosen": -1.9909662008285522, + "logits/rejected": -2.0008864402770996, + "logps/chosen": -0.26980477571487427, + "logps/rejected": -0.26790764927864075, + "loss": 0.08855850249528885, + "loss/core": 0.08529040962457657, + "loss/preference_sft": 0.26980477571487427, + "loss/reference_anchor": 0.13642403483390808, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.2187845706939697, + "rewards/margins": 1.0122959613800049, + "rewards/rejected": 1.2064883708953857, + "step": 120 + }, + { + "drift/chosen_abs": 0.3175174295902252, + "drift/rejected_abs": 0.18678590655326843, + "epoch": 0.11930326890956812, + "grad_norm": 768.0, + "ht_mnpo/logit": 0.13146166503429413, + "ht_mnpo/residual": 0.12646165490150452, + "ht_mnpo/residual_abs": 0.18497243523597717, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.4891472919490977e-07, + "logits/chosen": -1.972855567932129, + "logits/rejected": -2.0070338249206543, + "logps/chosen": -0.3044547140598297, + "logps/rejected": -0.30631768703460693, + "loss": 0.5623260736465454, + "loss/core": 0.5474299192428589, + "loss/preference_sft": 0.3044547140598297, + "loss/reference_anchor": 0.7143507599830627, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 3.170875072479248, + "rewards/margins": 1.3146165609359741, + "rewards/rejected": 1.8562583923339844, + "step": 125 + }, + { + "drift/chosen_abs": 0.20063884556293488, + "drift/rejected_abs": 0.10184315592050552, + "epoch": 0.12407539966595085, + "grad_norm": 2.671875, + "ht_mnpo/logit": 0.11115453392267227, + "ht_mnpo/residual": 0.10615452378988266, + "ht_mnpo/residual_abs": 0.11951833963394165, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.4857271493713894e-07, + "logits/chosen": -2.1099905967712402, + "logits/rejected": -2.1065971851348877, + "logps/chosen": -0.3332940936088562, + "logps/rejected": -0.33428648114204407, + "loss": 0.3267074227333069, + "loss/core": 0.32000210881233215, + "loss/preference_sft": 0.3332940936088562, + "loss/reference_anchor": 0.30193790793418884, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.0046913623809814, + "rewards/margins": 1.1115453243255615, + "rewards/rejected": 0.8931459188461304, + "step": 130 + }, + { + "drift/chosen_abs": 0.13585230708122253, + "drift/rejected_abs": 0.12510886788368225, + "epoch": 0.12884753042233357, + "grad_norm": 105.0, + "ht_mnpo/logit": 0.010902149602770805, + "ht_mnpo/residual": 0.005902145989239216, + "ht_mnpo/residual_abs": 0.10268938541412354, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.481842300350339e-07, + "logits/chosen": -2.2717299461364746, + "logits/rejected": -2.1862072944641113, + "logps/chosen": -0.26431578397750854, + "logps/rejected": -0.2654600441455841, + "loss": 0.11229200661182404, + "loss/core": 0.10951416194438934, + "loss/preference_sft": 0.26431578397750854, + "loss/reference_anchor": 0.11246071010828018, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.3585233688354492, + "rewards/margins": 0.10902152210474014, + "rewards/rejected": 1.2495018243789673, + "step": 135 + }, + { + "drift/chosen_abs": 0.28132104873657227, + "drift/rejected_abs": 0.15722861886024475, + "epoch": 0.1336196611787163, + "grad_norm": 272.0, + "ht_mnpo/logit": 0.12429539859294891, + "ht_mnpo/residual": 0.1192954033613205, + "ht_mnpo/residual_abs": 0.16520704329013824, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.4774942058187854e-07, + "logits/chosen": -2.127556324005127, + "logits/rejected": -2.184530735015869, + "logps/chosen": -0.29260799288749695, + "logps/rejected": -0.27409014105796814, + "loss": 0.3249160051345825, + "loss/core": 0.31713393330574036, + "loss/preference_sft": 0.29260799288749695, + "loss/reference_anchor": 0.3598441779613495, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.8131933212280273, + "rewards/margins": 1.2429540157318115, + "rewards/rejected": 1.5702394247055054, + "step": 140 + }, + { + "drift/chosen_abs": 0.13580551743507385, + "drift/rejected_abs": 0.08986364305019379, + "epoch": 0.138391791935099, + "grad_norm": 0.50390625, + "ht_mnpo/logit": 0.046709753572940826, + "ht_mnpo/residual": 0.04170975089073181, + "ht_mnpo/residual_abs": 0.09605139493942261, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.472684500917257e-07, + "logits/chosen": -1.9042880535125732, + "logits/rejected": -1.8750311136245728, + "logps/chosen": -0.3126002550125122, + "logps/rejected": -0.3028576076030731, + "loss": 0.08231665939092636, + "loss/core": 0.08015841245651245, + "loss/preference_sft": 0.3126002550125122, + "loss/reference_anchor": 0.07665302604436874, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.3563940525054932, + "rewards/margins": 0.46709758043289185, + "rewards/rejected": 0.8892965316772461, + "step": 145 + }, + { + "drift/chosen_abs": 0.25080087780952454, + "drift/rejected_abs": 0.18363021314144135, + "epoch": 0.14316392269148176, + "grad_norm": 85.5, + "ht_mnpo/logit": 0.06734486669301987, + "ht_mnpo/residual": 0.06234486773610115, + "ht_mnpo/residual_abs": 0.19637265801429749, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.467414994379065e-07, + "logits/chosen": -2.0690462589263916, + "logits/rejected": -2.0414555072784424, + "logps/chosen": -0.2994147539138794, + "logps/rejected": -0.31211525201797485, + "loss": 0.49615511298179626, + "loss/core": 0.4883514940738678, + "loss/preference_sft": 0.2994147539138794, + "loss/reference_anchor": 0.36023765802383423, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.507319211959839, + "rewards/margins": 0.6734486818313599, + "rewards/rejected": 1.8338706493377686, + "step": 150 + }, + { + "drift/chosen_abs": 0.16852137446403503, + "drift/rejected_abs": 0.14129950106143951, + "epoch": 0.14793605344786448, + "grad_norm": 4.96875, + "ht_mnpo/logit": 0.028056615963578224, + "ht_mnpo/residual": 0.023056630045175552, + "ht_mnpo/residual_abs": 0.12251297384500504, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.4616876678501114e-07, + "logits/chosen": -2.131013870239258, + "logits/rejected": -2.1533591747283936, + "logps/chosen": -0.28131020069122314, + "logps/rejected": -0.2708612084388733, + "loss": 0.22561495006084442, + "loss/core": 0.2211950123310089, + "loss/preference_sft": 0.28131020069122314, + "loss/reference_anchor": 0.1928669959306717, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6846176385879517, + "rewards/margins": 0.2805662751197815, + "rewards/rejected": 1.404051423072815, + "step": 155 + }, + { + "drift/chosen_abs": 0.320243775844574, + "drift/rejected_abs": 0.16745874285697937, + "epoch": 0.1527081842042472, + "grad_norm": 222.0, + "ht_mnpo/logit": 0.15298235416412354, + "ht_mnpo/residual": 0.14798232913017273, + "ht_mnpo/residual_abs": 0.25980624556541443, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.4555046751436735e-07, + "logits/chosen": -2.047654390335083, + "logits/rejected": -2.106396198272705, + "logps/chosen": -0.28943127393722534, + "logps/rejected": -0.2772199213504791, + "loss": 0.6820236444473267, + "loss/core": 0.6723781824111938, + "loss/preference_sft": 0.28943127393722534, + "loss/reference_anchor": 0.45332932472229004, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.20243763923645, + "rewards/margins": 1.5298234224319458, + "rewards/rejected": 1.672614336013794, + "step": 160 + }, + { + "drift/chosen_abs": 0.27144092321395874, + "drift/rejected_abs": 0.1204414814710617, + "epoch": 0.15748031496062992, + "grad_norm": 2.3125, + "ht_mnpo/logit": 0.15102508664131165, + "ht_mnpo/residual": 0.14602509140968323, + "ht_mnpo/residual_abs": 0.1496870219707489, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.4488683414304425e-07, + "logits/chosen": -2.335982322692871, + "logits/rejected": -2.3015449047088623, + "logps/chosen": -0.29247352480888367, + "logps/rejected": -0.27914655208587646, + "loss": 0.26414981484413147, + "loss/core": 0.25607991218566895, + "loss/preference_sft": 0.29247352480888367, + "loss/reference_anchor": 0.37424594163894653, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.713765859603882, + "rewards/margins": 1.5102509260177612, + "rewards/rejected": 1.2035149335861206, + "step": 165 + }, + { + "drift/chosen_abs": 0.17270924150943756, + "drift/rejected_abs": 0.1425841748714447, + "epoch": 0.16225244571701264, + "grad_norm": 1.3046875, + "ht_mnpo/logit": 0.030281126499176025, + "ht_mnpo/residual": 0.025281131267547607, + "ht_mnpo/residual_abs": 0.09685656428337097, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.4417811623641203e-07, + "logits/chosen": -2.147362470626831, + "logits/rejected": -2.008695363998413, + "logps/chosen": -0.2702954113483429, + "logps/rejected": -0.2617551386356354, + "loss": 0.09857013821601868, + "loss/core": 0.09527895599603653, + "loss/preference_sft": 0.2702954113483429, + "loss/reference_anchor": 0.13752929866313934, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.7254279851913452, + "rewards/margins": 0.30281129479408264, + "rewards/rejected": 1.4226164817810059, + "step": 170 + }, + { + "drift/chosen_abs": 0.13901950418949127, + "drift/rejected_abs": 0.06285737454891205, + "epoch": 0.16702457647339536, + "grad_norm": 27.75, + "ht_mnpo/logit": 0.07633363455533981, + "ht_mnpo/residual": 0.0713336318731308, + "ht_mnpo/residual_abs": 0.0760313868522644, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.4342458031429113e-07, + "logits/chosen": -2.2728612422943115, + "logits/rejected": -2.179896831512451, + "logps/chosen": -0.28125789761543274, + "logps/rejected": -0.2593241333961487, + "loss": 0.06590215116739273, + "loss/core": 0.06366212666034698, + "loss/preference_sft": 0.28125789761543274, + "loss/reference_anchor": 0.08387625217437744, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.3895217180252075, + "rewards/margins": 0.7633363008499146, + "rewards/rejected": 0.6261853575706482, + "step": 175 + }, + { + "drift/chosen_abs": 0.1901485025882721, + "drift/rejected_abs": 0.10939119011163712, + "epoch": 0.1717967072297781, + "grad_norm": 3.984375, + "ht_mnpo/logit": 0.08394713699817657, + "ht_mnpo/residual": 0.07894712686538696, + "ht_mnpo/residual_abs": 0.08561943471431732, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.4262650975072444e-07, + "logits/chosen": -2.1660208702087402, + "logits/rejected": -2.2384419441223145, + "logps/chosen": -0.29047879576683044, + "logps/rejected": -0.27046507596969604, + "loss": 0.04060132056474686, + "loss/core": 0.037876274436712265, + "loss/preference_sft": 0.29047879576683044, + "loss/reference_anchor": 0.10720425844192505, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.8998758792877197, + "rewards/margins": 0.8394712209701538, + "rewards/rejected": 1.0604045391082764, + "step": 180 + }, + { + "drift/chosen_abs": 0.2142634391784668, + "drift/rejected_abs": 0.09474581480026245, + "epoch": 0.17656883798616083, + "grad_norm": 2.0, + "ht_mnpo/logit": 0.12820716202259064, + "ht_mnpo/residual": 0.12320715188980103, + "ht_mnpo/residual_abs": 0.12510161101818085, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.417842046674122e-07, + "logits/chosen": -1.956552267074585, + "logits/rejected": -2.041656017303467, + "logps/chosen": -0.28232455253601074, + "logps/rejected": -0.28129279613494873, + "loss": 0.13155274093151093, + "loss/core": 0.12529942393302917, + "loss/preference_sft": 0.28232455253601074, + "loss/reference_anchor": 0.2844333052635193, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.1409764289855957, + "rewards/margins": 1.2820714712142944, + "rewards/rejected": 0.8589048385620117, + "step": 185 + }, + { + "drift/chosen_abs": 0.1476023942232132, + "drift/rejected_abs": 0.09407086670398712, + "epoch": 0.18134096874254355, + "grad_norm": 2.3125, + "ht_mnpo/logit": 0.05354578420519829, + "ht_mnpo/residual": 0.048545777797698975, + "ht_mnpo/residual_abs": 0.07173924148082733, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.4089798182084843e-07, + "logits/chosen": -2.1769192218780518, + "logits/rejected": -2.289309024810791, + "logps/chosen": -0.29876869916915894, + "logps/rejected": -0.29231029748916626, + "loss": 0.029190903529524803, + "loss/core": 0.027344604954123497, + "loss/preference_sft": 0.29876869916915894, + "loss/reference_anchor": 0.06243804842233658, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.4751077890396118, + "rewards/margins": 0.5354577302932739, + "rewards/rejected": 0.9396499395370483, + "step": 190 + }, + { + "drift/chosen_abs": 0.29235774278640747, + "drift/rejected_abs": 0.13147327303886414, + "epoch": 0.18611309949892627, + "grad_norm": 14.4375, + "ht_mnpo/logit": 0.16157792508602142, + "ht_mnpo/residual": 0.1565779149532318, + "ht_mnpo/residual_abs": 0.18083927035331726, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.3996817448320195e-07, + "logits/chosen": -2.084517240524292, + "logits/rejected": -2.0568737983703613, + "logps/chosen": -0.29337945580482483, + "logps/rejected": -0.29407113790512085, + "loss": 0.17335663735866547, + "loss/core": 0.16855856776237488, + "loss/preference_sft": 0.29337945580482483, + "loss/reference_anchor": 0.21056561172008514, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.920558214187622, + "rewards/margins": 1.6157792806625366, + "rewards/rejected": 1.304778814315796, + "step": 195 + }, + { + "drift/chosen_abs": 0.22568628191947937, + "drift/rejected_abs": 0.11775574833154678, + "epoch": 0.19088523025530899, + "grad_norm": 2.796875, + "ht_mnpo/logit": 0.10790988057851791, + "ht_mnpo/residual": 0.1029098853468895, + "ht_mnpo/residual_abs": 0.10873430967330933, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.3899513231698607e-07, + "logits/chosen": -2.2028231620788574, + "logits/rejected": -2.1879143714904785, + "logps/chosen": -0.2985078692436218, + "logps/rejected": -0.30355304479599, + "loss": 0.24885663390159607, + "loss/core": 0.24319525063037872, + "loss/preference_sft": 0.2985078692436218, + "loss/reference_anchor": 0.25322026014328003, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.256082057952881, + "rewards/margins": 1.0790985822677612, + "rewards/rejected": 1.17698335647583, + "step": 200 + }, + { + "drift/chosen_abs": 0.1635308712720871, + "drift/rejected_abs": 0.11117591708898544, + "epoch": 0.19565736101169173, + "grad_norm": 12.125, + "ht_mnpo/logit": 0.05233384296298027, + "ht_mnpo/residual": 0.047333844006061554, + "ht_mnpo/residual_abs": 0.0785122960805893, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.3797922124356506e-07, + "logits/chosen": -1.996665596961975, + "logits/rejected": -2.086698293685913, + "logps/chosen": -0.3164716362953186, + "logps/rejected": -0.3085149824619293, + "loss": 0.044618308544158936, + "loss/core": 0.040909815579652786, + "loss/preference_sft": 0.3164716362953186, + "loss/reference_anchor": 0.1537773758172989, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.631773591041565, + "rewards/margins": 0.5233384370803833, + "rewards/rejected": 1.1084349155426025, + "step": 205 + }, + { + "drift/chosen_abs": 0.3185950815677643, + "drift/rejected_abs": 0.16004785895347595, + "epoch": 0.20042949176807445, + "grad_norm": 12.875, + "ht_mnpo/logit": 0.15874508023262024, + "ht_mnpo/residual": 0.15374508500099182, + "ht_mnpo/residual_abs": 0.17046402394771576, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.3692082330554585e-07, + "logits/chosen": -1.9498332738876343, + "logits/rejected": -1.9200170040130615, + "logps/chosen": -0.2881793677806854, + "logps/rejected": -0.2933315336704254, + "loss": 0.2835574746131897, + "loss/core": 0.27446287870407104, + "loss/preference_sft": 0.2881793677806854, + "loss/reference_anchor": 0.4259108006954193, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.185950756072998, + "rewards/margins": 1.5874507427215576, + "rewards/rejected": 1.59850013256073, + "step": 210 + }, + { + "drift/chosen_abs": 0.20830459892749786, + "drift/rejected_abs": 0.12604334950447083, + "epoch": 0.20520162252445717, + "grad_norm": 1.0078125, + "ht_mnpo/logit": 0.0821927934885025, + "ht_mnpo/residual": 0.07719278335571289, + "ht_mnpo/residual_abs": 0.12102246284484863, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.3582033652310765e-07, + "logits/chosen": -2.1301636695861816, + "logits/rejected": -2.1746068000793457, + "logps/chosen": -0.2960442304611206, + "logps/rejected": -0.2779351472854614, + "loss": 0.1845921128988266, + "loss/core": 0.17939044535160065, + "loss/preference_sft": 0.2960442304611206, + "loss/reference_anchor": 0.23047873377799988, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.080594539642334, + "rewards/margins": 0.8219278454780579, + "rewards/rejected": 1.258666753768921, + "step": 215 + }, + { + "drift/chosen_abs": 0.19475789368152618, + "drift/rejected_abs": 0.13050904870033264, + "epoch": 0.2099737532808399, + "grad_norm": 0.76953125, + "ht_mnpo/logit": 0.06595344096422195, + "ht_mnpo/residual": 0.060953445732593536, + "ht_mnpo/residual_abs": 0.08129678666591644, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.346781747443227e-07, + "logits/chosen": -2.091243028640747, + "logits/rejected": -2.200920820236206, + "logps/chosen": -0.32776325941085815, + "logps/rejected": -0.3202073574066162, + "loss": 0.09159465879201889, + "loss/core": 0.08345116674900055, + "loss/preference_sft": 0.32776325941085815, + "loss/reference_anchor": 0.3743983805179596, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.9458799362182617, + "rewards/margins": 0.6595343947410583, + "rewards/rejected": 1.286345362663269, + "step": 220 + }, + { + "drift/chosen_abs": 0.3241156041622162, + "drift/rejected_abs": 0.1684974730014801, + "epoch": 0.2147458840372226, + "grad_norm": 1.0234375, + "ht_mnpo/logit": 0.15598830580711365, + "ht_mnpo/residual": 0.15098831057548523, + "ht_mnpo/residual_abs": 0.18187597393989563, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.3349476748952508e-07, + "logits/chosen": -2.1542420387268066, + "logits/rejected": -2.227787494659424, + "logps/chosen": -0.25393348932266235, + "logps/rejected": -0.23377032577991486, + "loss": 0.25889310240745544, + "loss/core": 0.24731948971748352, + "loss/preference_sft": 0.25393348932266235, + "loss/reference_anchor": 0.5532869100570679, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.2385611534118652, + "rewards/margins": 1.5598829984664917, + "rewards/rejected": 1.6786781549453735, + "step": 225 + }, + { + "drift/chosen_abs": 0.11558188498020172, + "drift/rejected_abs": 0.06361546367406845, + "epoch": 0.21951801479360533, + "grad_norm": 4.625, + "ht_mnpo/logit": 0.052035581320524216, + "ht_mnpo/residual": 0.0470355823636055, + "ht_mnpo/residual_abs": 0.05875653028488159, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.3227055978978545e-07, + "logits/chosen": -2.0761351585388184, + "logits/rejected": -2.168020009994507, + "logps/chosen": -0.319638729095459, + "logps/rejected": -0.2952752113342285, + "loss": 0.015921492129564285, + "loss/core": 0.014574600383639336, + "loss/preference_sft": 0.319638729095459, + "loss/reference_anchor": 0.03538065403699875, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.1521855592727661, + "rewards/margins": 0.5203558206558228, + "rewards/rejected": 0.6318296194076538, + "step": 230 + }, + { + "drift/chosen_abs": 0.183929443359375, + "drift/rejected_abs": 0.12602423131465912, + "epoch": 0.22429014554998808, + "grad_norm": 4.28125, + "ht_mnpo/logit": 0.05796173959970474, + "ht_mnpo/residual": 0.05296173691749573, + "ht_mnpo/residual_abs": 0.1496523916721344, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.3100601201955321e-07, + "logits/chosen": -2.0285463333129883, + "logits/rejected": -2.161771059036255, + "logps/chosen": -0.30416566133499146, + "logps/rejected": -0.29958218336105347, + "loss": 0.3189251124858856, + "loss/core": 0.3137461543083191, + "loss/preference_sft": 0.30416566133499146, + "loss/reference_anchor": 0.22853069007396698, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8382625579833984, + "rewards/margins": 0.5796173810958862, + "rewards/rejected": 1.2586452960968018, + "step": 235 + }, + { + "drift/chosen_abs": 0.15257421135902405, + "drift/rejected_abs": 0.09819750487804413, + "epoch": 0.2290622763063708, + "grad_norm": 2.9375, + "ht_mnpo/logit": 0.054583191871643066, + "ht_mnpo/residual": 0.04958319664001465, + "ht_mnpo/residual_abs": 0.06293803453445435, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.2970159972352864e-07, + "logits/chosen": -2.17802095413208, + "logits/rejected": -2.2032787799835205, + "logps/chosen": -0.29463058710098267, + "logps/rejected": -0.27752771973609924, + "loss": 0.01321292482316494, + "loss/core": 0.011416586115956306, + "loss/preference_sft": 0.29463058710098267, + "loss/reference_anchor": 0.060353875160217285, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.5236282348632812, + "rewards/margins": 0.5458319783210754, + "rewards/rejected": 0.9777960777282715, + "step": 240 + }, + { + "drift/chosen_abs": 0.15810930728912354, + "drift/rejected_abs": 0.12099383026361465, + "epoch": 0.23383440706275352, + "grad_norm": 2.34375, + "ht_mnpo/logit": 0.03711375966668129, + "ht_mnpo/residual": 0.03211376070976257, + "ht_mnpo/residual_abs": 0.11317561566829681, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.2835781343782966e-07, + "logits/chosen": -2.1228532791137695, + "logits/rejected": -2.219285011291504, + "logps/chosen": -0.2916942536830902, + "logps/rejected": -0.2701827883720398, + "loss": 0.14501670002937317, + "loss/core": 0.14180776476860046, + "loss/preference_sft": 0.2916942536830902, + "loss/reference_anchor": 0.13127687573432922, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.5794703960418701, + "rewards/margins": 0.3711375296115875, + "rewards/rejected": 1.2083330154418945, + "step": 245 + }, + { + "drift/chosen_abs": 0.3347860872745514, + "drift/rejected_abs": 0.12715332210063934, + "epoch": 0.23860653781913624, + "grad_norm": 47.75, + "ht_mnpo/logit": 0.20922037959098816, + "ht_mnpo/residual": 0.20422036945819855, + "ht_mnpo/residual_abs": 0.2054227888584137, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.2697515850552152e-07, + "logits/chosen": -1.8939529657363892, + "logits/rejected": -1.9978229999542236, + "logps/chosen": -0.30424895882606506, + "logps/rejected": -0.27813011407852173, + "loss": 0.33961981534957886, + "loss/core": 0.3286818861961365, + "loss/preference_sft": 0.30424895882606506, + "loss/reference_anchor": 0.5164695978164673, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.3477721214294434, + "rewards/margins": 2.0922036170959473, + "rewards/rejected": 1.2555686235427856, + "step": 250 + }, + { + "drift/chosen_abs": 0.2625555992126465, + "drift/rejected_abs": 0.16138434410095215, + "epoch": 0.24337866857551896, + "grad_norm": 4.625, + "ht_mnpo/logit": 0.10125492513179779, + "ht_mnpo/residual": 0.09625492990016937, + "ht_mnpo/residual_abs": 0.13495945930480957, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.2555415488657775e-07, + "logits/chosen": -1.8251349925994873, + "logits/rejected": -1.8897521495819092, + "logps/chosen": -0.299666166305542, + "logps/rejected": -0.29763779044151306, + "loss": 0.16235163807868958, + "loss/core": 0.15317954123020172, + "loss/preference_sft": 0.299666166305542, + "loss/reference_anchor": 0.42863768339157104, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.624504804611206, + "rewards/margins": 1.0125491619110107, + "rewards/rejected": 1.6119556427001953, + "step": 255 + }, + { + "drift/chosen_abs": 0.32063397765159607, + "drift/rejected_abs": 0.10358726978302002, + "epoch": 0.2481507993319017, + "grad_norm": 1368.0, + "ht_mnpo/logit": 0.21735945343971252, + "ht_mnpo/residual": 0.21235942840576172, + "ht_mnpo/residual_abs": 0.23501832783222198, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.240953369623447e-07, + "logits/chosen": -2.1691572666168213, + "logits/rejected": -2.276984691619873, + "logps/chosen": -0.3046864867210388, + "logps/rejected": -0.2934969365596771, + "loss": 1.100789189338684, + "loss/core": 1.0874181985855103, + "loss/preference_sft": 0.3046864867210388, + "loss/reference_anchor": 0.6380894780158997, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.2051138877868652, + "rewards/margins": 2.1735942363739014, + "rewards/rejected": 1.0315196514129639, + "step": 260 + }, + { + "drift/chosen_abs": 0.223398357629776, + "drift/rejected_abs": 0.20160217583179474, + "epoch": 0.2529229300882844, + "grad_norm": 32.0, + "ht_mnpo/logit": 0.03205033391714096, + "ht_mnpo/residual": 0.027050340548157692, + "ht_mnpo/residual_abs": 0.19905897974967957, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.225992533345824e-07, + "logits/chosen": -2.072179079055786, + "logits/rejected": -2.102553367614746, + "logps/chosen": -0.28394564986228943, + "logps/rejected": -0.2866360545158386, + "loss": 0.5659416317939758, + "loss/core": 0.5566951632499695, + "loss/preference_sft": 0.28394564986228943, + "loss/reference_anchor": 0.433927059173584, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.2328720092773438, + "rewards/margins": 0.32050344347953796, + "rewards/rejected": 1.9123685359954834, + "step": 265 + }, + { + "drift/chosen_abs": 0.20549187064170837, + "drift/rejected_abs": 0.16975179314613342, + "epoch": 0.25769506084466715, + "grad_norm": 204.0, + "ht_mnpo/logit": 0.035853564739227295, + "ht_mnpo/residual": 0.03085356391966343, + "ht_mnpo/residual_abs": 0.19780030846595764, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.210664666191579e-07, + "logits/chosen": -2.068603038787842, + "logits/rejected": -2.148378372192383, + "logps/chosen": -0.2916108965873718, + "logps/rejected": -0.32321587204933167, + "loss": 0.5143053531646729, + "loss/core": 0.5063118934631348, + "loss/preference_sft": 0.2916108965873718, + "loss/reference_anchor": 0.37051326036453247, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.0546793937683105, + "rewards/margins": 0.3585355281829834, + "rewards/rejected": 1.6961438655853271, + "step": 270 + }, + { + "drift/chosen_abs": 0.1933935433626175, + "drift/rejected_abs": 0.12514333426952362, + "epoch": 0.26246719160104987, + "grad_norm": 6.8125, + "ht_mnpo/logit": 0.06834165751934052, + "ht_mnpo/residual": 0.0633416622877121, + "ht_mnpo/residual_abs": 0.09675142914056778, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.1949755323446848e-07, + "logits/chosen": -2.070564031600952, + "logits/rejected": -2.177929162979126, + "logps/chosen": -0.2939522862434387, + "logps/rejected": -0.2786833941936493, + "loss": 0.05802987888455391, + "loss/core": 0.05398845672607422, + "loss/preference_sft": 0.2939522862434387, + "loss/reference_anchor": 0.17267616093158722, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.9336230754852295, + "rewards/margins": 0.6834166646003723, + "rewards/rejected": 1.2502063512802124, + "step": 275 + }, + { + "drift/chosen_abs": 0.2726713716983795, + "drift/rejected_abs": 0.15814051032066345, + "epoch": 0.2672393223574326, + "grad_norm": 99.0, + "ht_mnpo/logit": 0.1145574226975441, + "ht_mnpo/residual": 0.10955740511417389, + "ht_mnpo/residual_abs": 0.22781553864479065, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.1789310318467426e-07, + "logits/chosen": -2.0886340141296387, + "logits/rejected": -2.1369686126708984, + "logps/chosen": -0.3071291744709015, + "logps/rejected": -0.29813405871391296, + "loss": 0.5179394483566284, + "loss/core": 0.5096560716629028, + "loss/preference_sft": 0.3071291744709015, + "loss/reference_anchor": 0.3834558427333832, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.725396156311035, + "rewards/margins": 1.1455739736557007, + "rewards/rejected": 1.5798218250274658, + "step": 280 + }, + { + "drift/chosen_abs": 0.17260298132896423, + "drift/rejected_abs": 0.16383907198905945, + "epoch": 0.2720114531138153, + "grad_norm": 1.578125, + "ht_mnpo/logit": 0.008781191892921925, + "ht_mnpo/residual": 0.0037811901420354843, + "ht_mnpo/residual_abs": 0.10902787744998932, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.1625371983782182e-07, + "logits/chosen": -2.233491897583008, + "logits/rejected": -2.1172218322753906, + "logps/chosen": -0.2896055579185486, + "logps/rejected": -0.33911457657814026, + "loss": 0.08416543900966644, + "loss/core": 0.08064934611320496, + "loss/preference_sft": 0.2896055579185486, + "loss/reference_anchor": 0.1468445360660553, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.7244901657104492, + "rewards/margins": 0.0878119170665741, + "rewards/rejected": 1.6366783380508423, + "step": 285 + }, + { + "drift/chosen_abs": 0.10990118980407715, + "drift/rejected_abs": 0.046395592391490936, + "epoch": 0.276783583870198, + "grad_norm": 49.75, + "ht_mnpo/logit": 0.06795798242092133, + "ht_mnpo/residual": 0.06295798718929291, + "ht_mnpo/residual_abs": 0.07829216122627258, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.14580019698942e-07, + "logits/chosen": -2.262953281402588, + "logits/rejected": -2.218596935272217, + "logps/chosen": -0.3364924490451813, + "logps/rejected": -0.33589696884155273, + "loss": 0.10862968862056732, + "loss/core": 0.10672261565923691, + "loss/preference_sft": 0.3364924490451813, + "loss/reference_anchor": 0.061704643070697784, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.0977586507797241, + "rewards/margins": 0.6795798540115356, + "rewards/rejected": 0.41817885637283325, + "step": 290 + }, + { + "drift/chosen_abs": 0.20892763137817383, + "drift/rejected_abs": 0.07657907903194427, + "epoch": 0.28155571462658074, + "grad_norm": 152.0, + "ht_mnpo/logit": 0.1324269324541092, + "ht_mnpo/residual": 0.12742695212364197, + "ht_mnpo/residual_abs": 0.1512664258480072, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.1287263217820773e-07, + "logits/chosen": -2.0495729446411133, + "logits/rejected": -2.147361993789673, + "logps/chosen": -0.3076549172401428, + "logps/rejected": -0.28194406628608704, + "loss": 0.2917906939983368, + "loss/core": 0.28768688440322876, + "loss/preference_sft": 0.3076549172401428, + "loss/reference_anchor": 0.17442592978477478, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.0885672569274902, + "rewards/margins": 1.32426917552948, + "rewards/rejected": 0.7642977833747864, + "step": 295 + }, + { + "drift/chosen_abs": 0.2935689687728882, + "drift/rejected_abs": 0.19712750613689423, + "epoch": 0.2863278453829635, + "grad_norm": 5.90625, + "ht_mnpo/logit": 0.09644874185323715, + "ht_mnpo/residual": 0.09144874662160873, + "ht_mnpo/residual_abs": 0.16998650133609772, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.111321993542385e-07, + "logits/chosen": -2.0562853813171387, + "logits/rejected": -2.131844997406006, + "logps/chosen": -0.2610465884208679, + "logps/rejected": -0.25709906220436096, + "loss": 0.19292078912258148, + "loss/core": 0.17979469895362854, + "loss/preference_sft": 0.2610465884208679, + "loss/reference_anchor": 0.6302005052566528, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.9351634979248047, + "rewards/margins": 0.9644874334335327, + "rewards/rejected": 1.9706761837005615, + "step": 300 + }, + { + "drift/chosen_abs": 0.22880244255065918, + "drift/rejected_abs": 0.14696647226810455, + "epoch": 0.29109997613934624, + "grad_norm": 6.78125, + "ht_mnpo/logit": 0.08201225101947784, + "ht_mnpo/residual": 0.07701224833726883, + "ht_mnpo/residual_abs": 0.08669502288103104, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.0935937573264096e-07, + "logits/chosen": -1.9870030879974365, + "logits/rejected": -2.0139083862304688, + "logps/chosen": -0.30271345376968384, + "logps/rejected": -0.30806469917297363, + "loss": 0.04288022220134735, + "loss/core": 0.03761247545480728, + "loss/preference_sft": 0.30271345376968384, + "loss/reference_anchor": 0.23311614990234375, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.2861876487731934, + "rewards/margins": 0.8201225399971008, + "rewards/rejected": 1.4660651683807373, + "step": 305 + }, + { + "drift/chosen_abs": 0.28174325823783875, + "drift/rejected_abs": 0.1894901990890503, + "epoch": 0.29587210689572896, + "grad_norm": 564.0, + "ht_mnpo/logit": 0.0924399122595787, + "ht_mnpo/residual": 0.08743990957736969, + "ht_mnpo/residual_abs": 0.23076701164245605, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.0755482799987596e-07, + "logits/chosen": -1.9850540161132812, + "logits/rejected": -1.970975637435913, + "logps/chosen": -0.285608172416687, + "logps/rejected": -0.2967362403869629, + "loss": 0.7051917910575867, + "loss/core": 0.6910423040390015, + "loss/preference_sft": 0.285608172416687, + "loss/reference_anchor": 0.6789166927337646, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.8170597553253174, + "rewards/margins": 0.9243990778923035, + "rewards/rejected": 1.8926608562469482, + "step": 310 + }, + { + "drift/chosen_abs": 0.18879075348377228, + "drift/rejected_abs": 0.1482362002134323, + "epoch": 0.3006442376521117, + "grad_norm": 45.5, + "ht_mnpo/logit": 0.040583543479442596, + "ht_mnpo/residual": 0.03558354452252388, + "ht_mnpo/residual_abs": 0.1549256145954132, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.0571923477254526e-07, + "logits/chosen": -2.215289354324341, + "logits/rejected": -2.188716411590576, + "logps/chosen": -0.313434898853302, + "logps/rejected": -0.29403433203697205, + "loss": 0.23106110095977783, + "loss/core": 0.22662413120269775, + "loss/preference_sft": 0.313434898853302, + "loss/reference_anchor": 0.19050461053848267, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.8869355916976929, + "rewards/margins": 0.40583547949790955, + "rewards/rejected": 1.4811002016067505, + "step": 315 + }, + { + "drift/chosen_abs": 0.2432427704334259, + "drift/rejected_abs": 0.13727743923664093, + "epoch": 0.3054163684084944, + "grad_norm": 21.625, + "ht_mnpo/logit": 0.10607069730758667, + "ht_mnpo/residual": 0.10107071697711945, + "ht_mnpo/residual_abs": 0.1325535625219345, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.038532863421914e-07, + "logits/chosen": -2.0528945922851562, + "logits/rejected": -1.9978628158569336, + "logps/chosen": -0.28710001707077026, + "logps/rejected": -0.29180365800857544, + "loss": 0.0885167121887207, + "loss/core": 0.08476362377405167, + "loss/preference_sft": 0.28710001707077026, + "loss/reference_anchor": 0.15894415974617004, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.4323668479919434, + "rewards/margins": 1.0607070922851562, + "rewards/rejected": 1.3716596364974976, + "step": 320 + }, + { + "drift/chosen_abs": 0.17099711298942566, + "drift/rejected_abs": 0.07577967643737793, + "epoch": 0.3101884991648771, + "grad_norm": 4.28125, + "ht_mnpo/logit": 0.09501691162586212, + "ht_mnpo/residual": 0.0900169089436531, + "ht_mnpo/residual_abs": 0.09240968525409698, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.0195768441570726e-07, + "logits/chosen": -1.9887382984161377, + "logits/rejected": -2.1444485187530518, + "logps/chosen": -0.2988322377204895, + "logps/rejected": -0.2900279462337494, + "loss": 0.08121752738952637, + "loss/core": 0.07862718403339386, + "loss/preference_sft": 0.2988322377204895, + "loss/reference_anchor": 0.09963365644216537, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.7076162099838257, + "rewards/margins": 0.9501692056655884, + "rewards/rejected": 0.7574471831321716, + "step": 325 + }, + { + "drift/chosen_abs": 0.15594327449798584, + "drift/rejected_abs": 0.07677812874317169, + "epoch": 0.31496062992125984, + "grad_norm": 3.203125, + "ht_mnpo/logit": 0.07974691689014435, + "ht_mnpo/residual": 0.07474692165851593, + "ht_mnpo/residual_abs": 0.08458121865987778, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.0003314185145307e-07, + "logits/chosen": -2.0846686363220215, + "logits/rejected": -2.14108943939209, + "logps/chosen": -0.31113189458847046, + "logps/rejected": -0.2992934584617615, + "loss": 0.07248958945274353, + "loss/core": 0.0704246237874031, + "loss/preference_sft": 0.31113189458847046, + "loss/reference_anchor": 0.07213523983955383, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.5589559078216553, + "rewards/margins": 0.7974691390991211, + "rewards/rejected": 0.7614867091178894, + "step": 330 + }, + { + "drift/chosen_abs": 0.23571383953094482, + "drift/rejected_abs": 0.13885119557380676, + "epoch": 0.31973276067764256, + "grad_norm": 1.6640625, + "ht_mnpo/logit": 0.09817638248205185, + "ht_mnpo/residual": 0.09317637979984283, + "ht_mnpo/residual_abs": 0.1573963165283203, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.9808038239117913e-07, + "logits/chosen": -2.0466277599334717, + "logits/rejected": -1.9438968896865845, + "logps/chosen": -0.31259217858314514, + "logps/rejected": -0.3062000274658203, + "loss": 0.20101185142993927, + "loss/core": 0.1956087350845337, + "loss/preference_sft": 0.31259217858314514, + "loss/reference_anchor": 0.2388957291841507, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.3562703132629395, + "rewards/margins": 0.9817637205123901, + "rewards/rejected": 1.3745065927505493, + "step": 335 + }, + { + "drift/chosen_abs": 0.19340157508850098, + "drift/rejected_abs": 0.18404799699783325, + "epoch": 0.3245048914340253, + "grad_norm": 1.203125, + "ht_mnpo/logit": 0.009365958161652088, + "ht_mnpo/residual": 0.004365948028862476, + "ht_mnpo/residual_abs": 0.15316197276115417, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.9610014038785646e-07, + "logits/chosen": -2.0774543285369873, + "logits/rejected": -2.0459485054016113, + "logps/chosen": -0.3146652579307556, + "logps/rejected": -0.3233223259449005, + "loss": 0.27519553899765015, + "loss/core": 0.2662979066371918, + "loss/preference_sft": 0.3146652579307556, + "loss/reference_anchor": 0.41341686248779297, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.9339611530303955, + "rewards/margins": 0.09365954250097275, + "rewards/rejected": 1.840301513671875, + "step": 340 + }, + { + "drift/chosen_abs": 0.3130771517753601, + "drift/rejected_abs": 0.16737492382526398, + "epoch": 0.329277022190408, + "grad_norm": 129.0, + "ht_mnpo/logit": 0.14578819274902344, + "ht_mnpo/residual": 0.14078819751739502, + "ht_mnpo/residual_abs": 0.22993621230125427, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.9409316052951657e-07, + "logits/chosen": -1.9784408807754517, + "logits/rejected": -2.0493741035461426, + "logps/chosen": -0.3167511820793152, + "logps/rejected": -0.28369244933128357, + "loss": 0.50648033618927, + "loss/core": 0.49761754274368286, + "loss/preference_sft": 0.3167511820793152, + "loss/reference_anchor": 0.4114700257778168, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.130608320236206, + "rewards/margins": 1.4578818082809448, + "rewards/rejected": 1.6727266311645508, + "step": 345 + }, + { + "drift/chosen_abs": 0.3182355761528015, + "drift/rejected_abs": 0.13714584708213806, + "epoch": 0.3340491529467907, + "grad_norm": 129.0, + "ht_mnpo/logit": 0.1825825273990631, + "ht_mnpo/residual": 0.1775825321674347, + "ht_mnpo/residual_abs": 0.19932051002979279, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.920601975592047e-07, + "logits/chosen": -2.3250603675842285, + "logits/rejected": -2.3682284355163574, + "logps/chosen": -0.2681167423725128, + "logps/rejected": -0.27266380190849304, + "loss": 0.4750707745552063, + "loss/core": 0.4656985402107239, + "loss/preference_sft": 0.2681167423725128, + "loss/reference_anchor": 0.4418010115623474, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.1823558807373047, + "rewards/margins": 1.8258254528045654, + "rewards/rejected": 1.3565304279327393, + "step": 350 + }, + { + "drift/chosen_abs": 0.2780304551124573, + "drift/rejected_abs": 0.21358676254749298, + "epoch": 0.3388212837031735, + "grad_norm": 756.0, + "ht_mnpo/logit": 0.06450818479061127, + "ht_mnpo/residual": 0.05950819328427315, + "ht_mnpo/residual_abs": 0.15815815329551697, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.900020159911519e-07, + "logits/chosen": -2.156029462814331, + "logits/rejected": -2.147246837615967, + "logps/chosen": -0.27569761872291565, + "logps/rejected": -0.27141404151916504, + "loss": 0.2398795187473297, + "loss/core": 0.22512921690940857, + "loss/preference_sft": 0.27569761872291565, + "loss/reference_anchor": 0.7099437713623047, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.7803046703338623, + "rewards/margins": 0.6450818181037903, + "rewards/rejected": 2.135222911834717, + "step": 355 + }, + { + "drift/chosen_abs": 0.3071724474430084, + "drift/rejected_abs": 0.14715437591075897, + "epoch": 0.3435934144595562, + "grad_norm": 716.0, + "ht_mnpo/logit": 0.16002245247364044, + "ht_mnpo/residual": 0.15502245724201202, + "ht_mnpo/residual_abs": 0.18939676880836487, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.879193898232725e-07, + "logits/chosen": -1.8401505947113037, + "logits/rejected": -1.8808279037475586, + "logps/chosen": -0.2977268099784851, + "logps/rejected": -0.30872827768325806, + "loss": 0.424351304769516, + "loss/core": 0.41371244192123413, + "loss/preference_sft": 0.2977268099784851, + "loss/reference_anchor": 0.5021710991859436, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.0710623264312744, + "rewards/margins": 1.600224494934082, + "rewards/rejected": 1.4708378314971924, + "step": 360 + }, + { + "drift/chosen_abs": 0.13023678958415985, + "drift/rejected_abs": 0.08073797076940536, + "epoch": 0.34836554521593893, + "grad_norm": 27.25, + "ht_mnpo/logit": 0.04946478456258774, + "ht_mnpo/residual": 0.04446478560566902, + "ht_mnpo/residual_abs": 0.05108083412051201, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.8581310224609496e-07, + "logits/chosen": -2.1624913215637207, + "logits/rejected": -2.199889659881592, + "logps/chosen": -0.29786261916160583, + "logps/rejected": -0.2950283885002136, + "loss": 0.024249404668807983, + "loss/core": 0.022617992013692856, + "loss/preference_sft": 0.29786261916160583, + "loss/reference_anchor": 0.051784414798021317, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.3009334802627563, + "rewards/margins": 0.4946478307247162, + "rewards/rejected": 0.806285560131073, + "step": 365 + }, + { + "drift/chosen_abs": 0.22245018184185028, + "drift/rejected_abs": 0.1544378399848938, + "epoch": 0.35313767597232165, + "grad_norm": 8.0, + "ht_mnpo/logit": 0.06817840784788132, + "ht_mnpo/residual": 0.0631784051656723, + "ht_mnpo/residual_abs": 0.19337259232997894, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.8368394534823635e-07, + "logits/chosen": -2.1282296180725098, + "logits/rejected": -2.102940320968628, + "logps/chosen": -0.27162525057792664, + "logps/rejected": -0.25887078046798706, + "loss": 0.5373419523239136, + "loss/core": 0.5292584300041199, + "loss/preference_sft": 0.27162525057792664, + "loss/reference_anchor": 0.37701216340065, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.223367691040039, + "rewards/margins": 0.6817840933799744, + "rewards/rejected": 1.5415834188461304, + "step": 370 + }, + { + "drift/chosen_abs": 0.27047693729400635, + "drift/rejected_abs": 0.13434390723705292, + "epoch": 0.35790980672870437, + "grad_norm": 22.375, + "ht_mnpo/logit": 0.13634856045246124, + "ht_mnpo/residual": 0.13134853541851044, + "ht_mnpo/residual_abs": 0.16767247021198273, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.8153271981852968e-07, + "logits/chosen": -2.171160936355591, + "logits/rejected": -2.2762207984924316, + "logps/chosen": -0.25913965702056885, + "logps/rejected": -0.27620944380760193, + "loss": 0.2835163474082947, + "loss/core": 0.2761160731315613, + "loss/preference_sft": 0.25913965702056885, + "loss/reference_anchor": 0.3440994620323181, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.7047693729400635, + "rewards/margins": 1.36348557472229, + "rewards/rejected": 1.341283917427063, + "step": 375 + }, + { + "drift/chosen_abs": 0.36079439520835876, + "drift/rejected_abs": 0.11835791915655136, + "epoch": 0.3626819374850871, + "grad_norm": 7.90625, + "ht_mnpo/logit": 0.24685482680797577, + "ht_mnpo/residual": 0.24185481667518616, + "ht_mnpo/residual_abs": 0.25279420614242554, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.7936023464491812e-07, + "logits/chosen": -1.9889020919799805, + "logits/rejected": -2.0030906200408936, + "logps/chosen": -0.3349910378456116, + "logps/rejected": -0.28726398944854736, + "loss": 0.6925952434539795, + "loss/core": 0.6821285486221313, + "loss/preference_sft": 0.3349910378456116, + "loss/reference_anchor": 0.4898358881473541, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.605802059173584, + "rewards/margins": 2.46854829788208, + "rewards/rejected": 1.1372535228729248, + "step": 380 + }, + { + "drift/chosen_abs": 0.190070241689682, + "drift/rejected_abs": 0.11075327545404434, + "epoch": 0.3674540682414698, + "grad_norm": 15.875, + "ht_mnpo/logit": 0.07903822511434555, + "ht_mnpo/residual": 0.07403822243213654, + "ht_mnpo/residual_abs": 0.09887916594743729, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.7716730681022723e-07, + "logits/chosen": -2.1092896461486816, + "logits/rejected": -2.122014284133911, + "logps/chosen": -0.2971669137477875, + "logps/rejected": -0.28943219780921936, + "loss": 0.060740817338228226, + "loss/core": 0.05780009552836418, + "loss/preference_sft": 0.2971669137477875, + "loss/reference_anchor": 0.11731908470392227, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.897173285484314, + "rewards/margins": 0.7903822660446167, + "rewards/rejected": 1.1067907810211182, + "step": 385 + }, + { + "drift/chosen_abs": 0.1997186690568924, + "drift/rejected_abs": 0.14593148231506348, + "epoch": 0.37222619899785253, + "grad_norm": 4.0, + "ht_mnpo/logit": 0.05384016036987305, + "ht_mnpo/residual": 0.04884016513824463, + "ht_mnpo/residual_abs": 0.12053972482681274, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.7495476098493152e-07, + "logits/chosen": -2.2018589973449707, + "logits/rejected": -2.266772747039795, + "logps/chosen": -0.2876618206501007, + "logps/rejected": -0.28682073950767517, + "loss": 0.13210603594779968, + "loss/core": 0.1281919628381729, + "loss/preference_sft": 0.2876618206501007, + "loss/reference_anchor": 0.16693679988384247, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.9971866607666016, + "rewards/margins": 0.5384016036987305, + "rewards/rejected": 1.458785057067871, + "step": 390 + }, + { + "drift/chosen_abs": 0.4221646189689636, + "drift/rejected_abs": 0.19121377170085907, + "epoch": 0.37699832975423525, + "grad_norm": 200.0, + "ht_mnpo/logit": 0.23127570748329163, + "ht_mnpo/residual": 0.2262757271528244, + "ht_mnpo/residual_abs": 0.2536894679069519, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.7272342921702937e-07, + "logits/chosen": -2.016627550125122, + "logits/rejected": -2.0958032608032227, + "logps/chosen": -0.2756597399711609, + "logps/rejected": -0.27037227153778076, + "loss": 0.5089841485023499, + "loss/core": 0.4903375506401062, + "loss/preference_sft": 0.2756597399711609, + "loss/reference_anchor": 0.9047657251358032, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 4.218891620635986, + "rewards/margins": 2.3127572536468506, + "rewards/rejected": 1.9061342477798462, + "step": 395 + }, + { + "drift/chosen_abs": 0.23051393032073975, + "drift/rejected_abs": 0.13780668377876282, + "epoch": 0.38177046051061797, + "grad_norm": 6.0625, + "ht_mnpo/logit": 0.09285277873277664, + "ht_mnpo/residual": 0.08785279095172882, + "ht_mnpo/residual_abs": 0.09289351105690002, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.7047415061914393e-07, + "logits/chosen": -2.1754462718963623, + "logits/rejected": -2.227482318878174, + "logps/chosen": -0.2999895215034485, + "logps/rejected": -0.2864961326122284, + "loss": 0.08253012597560883, + "loss/core": 0.0778684988617897, + "loss/preference_sft": 0.2999895215034485, + "loss/reference_anchor": 0.2030823677778244, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.304029941558838, + "rewards/margins": 0.9285279512405396, + "rewards/rejected": 1.3755018711090088, + "step": 400 + }, + { + "drift/chosen_abs": 0.2020423412322998, + "drift/rejected_abs": 0.07275157421827316, + "epoch": 0.3865425912670007, + "grad_norm": 9.4375, + "ht_mnpo/logit": 0.1292640119791031, + "ht_mnpo/residual": 0.12426400184631348, + "ht_mnpo/residual_abs": 0.12864765524864197, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.6820777105296707e-07, + "logits/chosen": -2.068509340286255, + "logits/rejected": -2.117534875869751, + "logps/chosen": -0.2823387682437897, + "logps/rejected": -0.27292540669441223, + "loss": 0.24370598793029785, + "loss/core": 0.23992788791656494, + "loss/preference_sft": 0.2823387682437897, + "loss/reference_anchor": 0.16066968441009521, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.018876075744629, + "rewards/margins": 1.292639970779419, + "rewards/rejected": 0.7262359857559204, + "step": 405 + }, + { + "drift/chosen_abs": 0.20162303745746613, + "drift/rejected_abs": 0.1299503743648529, + "epoch": 0.39131472202338347, + "grad_norm": 17.125, + "ht_mnpo/logit": 0.07248403877019882, + "ht_mnpo/residual": 0.0674840435385704, + "ht_mnpo/residual_abs": 0.1857225000858307, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.6592514281116553e-07, + "logits/chosen": -2.032942056655884, + "logits/rejected": -2.0350537300109863, + "logps/chosen": -0.29434436559677124, + "logps/rejected": -0.294363796710968, + "loss": 0.29699331521987915, + "loss/core": 0.2922920286655426, + "loss/preference_sft": 0.29434436559677124, + "loss/reference_anchor": 0.20562903583049774, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.014000654220581, + "rewards/margins": 0.7248404026031494, + "rewards/rejected": 1.2891602516174316, + "step": 410 + }, + { + "drift/chosen_abs": 0.3014175593852997, + "drift/rejected_abs": 0.1166742816567421, + "epoch": 0.3960868527797662, + "grad_norm": 3.265625, + "ht_mnpo/logit": 0.18487218022346497, + "ht_mnpo/residual": 0.17987218499183655, + "ht_mnpo/residual_abs": 0.1962762326002121, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.636271242968684e-07, + "logits/chosen": -1.9811691045761108, + "logits/rejected": -2.0673305988311768, + "logps/chosen": -0.30998358130455017, + "logps/rejected": -0.27552932500839233, + "loss": 0.6068626642227173, + "loss/core": 0.5962422490119934, + "loss/preference_sft": 0.30998358130455017, + "loss/reference_anchor": 0.5000226497650146, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.0127615928649902, + "rewards/margins": 1.8487217426300049, + "rewards/rejected": 1.1640396118164062, + "step": 415 + }, + { + "drift/chosen_abs": 0.22677898406982422, + "drift/rejected_abs": 0.09225823730230331, + "epoch": 0.4008589835361489, + "grad_norm": 22.75, + "ht_mnpo/logit": 0.13460934162139893, + "ht_mnpo/residual": 0.1296093463897705, + "ht_mnpo/residual_abs": 0.13361980020999908, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.6131457970085684e-07, + "logits/chosen": -2.0054891109466553, + "logits/rejected": -2.080866813659668, + "logps/chosen": -0.2823740839958191, + "logps/rejected": -0.2677801251411438, + "loss": 0.1006707176566124, + "loss/core": 0.09769301116466522, + "loss/preference_sft": 0.2823740839958191, + "loss/reference_anchor": 0.12064739316701889, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.2673637866973877, + "rewards/margins": 1.3460934162139893, + "rewards/rejected": 0.9212702512741089, + "step": 420 + }, + { + "drift/chosen_abs": 0.23399285972118378, + "drift/rejected_abs": 0.07922275364398956, + "epoch": 0.4056311142925316, + "grad_norm": 580.0, + "ht_mnpo/logit": 0.15503880381584167, + "ht_mnpo/residual": 0.15003880858421326, + "ht_mnpo/residual_abs": 0.16690881550312042, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.5898837867657727e-07, + "logits/chosen": -2.0042977333068848, + "logits/rejected": -2.1140353679656982, + "logps/chosen": -0.294498085975647, + "logps/rejected": -0.28271484375, + "loss": 0.5954475402832031, + "loss/core": 0.5873497128486633, + "loss/preference_sft": 0.294498085975647, + "loss/reference_anchor": 0.3754442036151886, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.338529109954834, + "rewards/margins": 1.550387978553772, + "rewards/rejected": 0.7881409525871277, + "step": 425 + }, + { + "drift/chosen_abs": 0.2102353274822235, + "drift/rejected_abs": 0.11086465418338776, + "epoch": 0.41040324504891434, + "grad_norm": 139.0, + "ht_mnpo/logit": 0.09947403520345688, + "ht_mnpo/residual": 0.09447403252124786, + "ht_mnpo/residual_abs": 0.14160969853401184, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.5664939601310023e-07, + "logits/chosen": -2.4258694648742676, + "logits/rejected": -2.3425300121307373, + "logps/chosen": -0.28239089250564575, + "logps/rejected": -0.2863374352455139, + "loss": 0.2521866261959076, + "loss/core": 0.24730908870697021, + "loss/preference_sft": 0.28239089250564575, + "loss/reference_anchor": 0.21563799679279327, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.102165699005127, + "rewards/margins": 0.9947404861450195, + "rewards/rejected": 1.1074254512786865, + "step": 430 + }, + { + "drift/chosen_abs": 0.24716107547283173, + "drift/rejected_abs": 0.13077858090400696, + "epoch": 0.41517537580529706, + "grad_norm": 165.0, + "ht_mnpo/logit": 0.11873073875904083, + "ht_mnpo/residual": 0.1137307733297348, + "ht_mnpo/residual_abs": 0.20443549752235413, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.5429851130614807e-07, + "logits/chosen": -2.232671022415161, + "logits/rejected": -2.221527338027954, + "logps/chosen": -0.2805400788784027, + "logps/rejected": -0.29566511511802673, + "loss": 0.9471338391304016, + "loss/core": 0.936263918876648, + "loss/preference_sft": 0.2805400788784027, + "loss/reference_anchor": 0.5154294967651367, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.470388889312744, + "rewards/margins": 1.187307596206665, + "rewards/rejected": 1.2830810546875, + "step": 435 + }, + { + "drift/chosen_abs": 0.2321050614118576, + "drift/rejected_abs": 0.14237116277217865, + "epoch": 0.4199475065616798, + "grad_norm": 28.625, + "ht_mnpo/logit": 0.08972425758838654, + "ht_mnpo/residual": 0.08472425490617752, + "ht_mnpo/residual_abs": 0.1322084367275238, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.51936608627315e-07, + "logits/chosen": -2.334120750427246, + "logits/rejected": -2.3476264476776123, + "logps/chosen": -0.2910197377204895, + "logps/rejected": -0.2775648832321167, + "loss": 0.1562577337026596, + "loss/core": 0.15094859898090363, + "loss/preference_sft": 0.2910197377204895, + "loss/reference_anchor": 0.23635432124137878, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.320913076400757, + "rewards/margins": 0.8972426652908325, + "rewards/rejected": 1.4236705303192139, + "step": 440 + }, + { + "drift/chosen_abs": 0.12170737981796265, + "drift/rejected_abs": 0.0694720596075058, + "epoch": 0.4247196373180625, + "grad_norm": 6.65625, + "ht_mnpo/logit": 0.05207798629999161, + "ht_mnpo/residual": 0.04707798734307289, + "ht_mnpo/residual_abs": 0.05085582658648491, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.4956457619160375e-07, + "logits/chosen": -2.181494951248169, + "logits/rejected": -2.2427611351013184, + "logps/chosen": -0.29907384514808655, + "logps/rejected": -0.29663634300231934, + "loss": 0.015848254784941673, + "loss/core": 0.014302779920399189, + "loss/preference_sft": 0.29907384514808655, + "loss/reference_anchor": 0.04736633971333504, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.2155005931854248, + "rewards/margins": 0.5207798480987549, + "rewards/rejected": 0.6947206258773804, + "step": 445 + }, + { + "drift/chosen_abs": 0.1860462725162506, + "drift/rejected_abs": 0.11890468746423721, + "epoch": 0.4294917680744452, + "grad_norm": 10.125, + "ht_mnpo/logit": 0.06721819937229156, + "ht_mnpo/residual": 0.06221820041537285, + "ht_mnpo/residual_abs": 0.06938367336988449, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.471833060234044e-07, + "logits/chosen": -2.1414787769317627, + "logits/rejected": -2.154848337173462, + "logps/chosen": -0.283276230096817, + "logps/rejected": -0.27300840616226196, + "loss": 0.02786489762365818, + "loss/core": 0.024693552404642105, + "loss/preference_sft": 0.283276230096817, + "loss/reference_anchor": 0.13023975491523743, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.8602144718170166, + "rewards/margins": 0.6721819043159485, + "rewards/rejected": 1.1880323886871338, + "step": 450 + }, + { + "drift/chosen_abs": 0.3013097643852234, + "drift/rejected_abs": 0.10819373279809952, + "epoch": 0.43426389883082794, + "grad_norm": 0.671875, + "ht_mnpo/logit": 0.1931832730770111, + "ht_mnpo/residual": 0.1881832778453827, + "ht_mnpo/residual_abs": 0.1936052143573761, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.4479369362104037e-07, + "logits/chosen": -2.102328062057495, + "logits/rejected": -2.152161121368408, + "logps/chosen": -0.28967249393463135, + "logps/rejected": -0.2850145399570465, + "loss": 0.6813478469848633, + "loss/core": 0.6696580648422241, + "loss/preference_sft": 0.28967249393463135, + "loss/reference_anchor": 0.5555153489112854, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.011138439178467, + "rewards/margins": 1.9318326711654663, + "rewards/rejected": 1.0793057680130005, + "step": 455 + }, + { + "drift/chosen_abs": 0.28658145666122437, + "drift/rejected_abs": 0.08983840048313141, + "epoch": 0.43903602958721066, + "grad_norm": 2608.0, + "ht_mnpo/logit": 0.19802699983119965, + "ht_mnpo/residual": 0.19302698969841003, + "ht_mnpo/residual_abs": 0.22204919159412384, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.4239663762000817e-07, + "logits/chosen": -2.2929089069366455, + "logits/rejected": -2.203780174255371, + "logps/chosen": -0.29892778396606445, + "logps/rejected": -0.28588932752609253, + "loss": 0.748153567314148, + "loss/core": 0.738308846950531, + "loss/preference_sft": 0.29892778396606445, + "loss/reference_anchor": 0.46233946084976196, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.865814685821533, + "rewards/margins": 1.9802697896957397, + "rewards/rejected": 0.8855449557304382, + "step": 460 + }, + { + "drift/chosen_abs": 0.15654951333999634, + "drift/rejected_abs": 0.09482669830322266, + "epoch": 0.44380816034359344, + "grad_norm": 10.5625, + "ht_mnpo/logit": 0.061750371009111404, + "ht_mnpo/residual": 0.056750379502773285, + "ht_mnpo/residual_abs": 0.07501949369907379, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.3999303945503747e-07, + "logits/chosen": -2.1819443702697754, + "logits/rejected": -2.1801505088806152, + "logps/chosen": -0.29756730794906616, + "logps/rejected": -0.27815189957618713, + "loss": 0.029046928510069847, + "loss/core": 0.02667873539030552, + "loss/preference_sft": 0.29756730794906616, + "loss/reference_anchor": 0.08865313977003098, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.5641218423843384, + "rewards/margins": 0.6175037026405334, + "rewards/rejected": 0.946618378162384, + "step": 465 + }, + { + "drift/chosen_abs": 0.30324727296829224, + "drift/rejected_abs": 0.1545564830303192, + "epoch": 0.44858029109997616, + "grad_norm": 19.25, + "ht_mnpo/logit": 0.1486632078886032, + "ht_mnpo/residual": 0.1436631977558136, + "ht_mnpo/residual_abs": 0.232595294713974, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.3758380302109808e-07, + "logits/chosen": -1.9403120279312134, + "logits/rejected": -2.031698703765869, + "logps/chosen": -0.29452261328697205, + "logps/rejected": -0.28670722246170044, + "loss": 0.5407153367996216, + "loss/core": 0.531051516532898, + "loss/preference_sft": 0.29452261328697205, + "loss/reference_anchor": 0.45374003052711487, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.0309526920318604, + "rewards/margins": 1.4866321086883545, + "rewards/rejected": 1.5443203449249268, + "step": 470 + }, + { + "drift/chosen_abs": 0.2222728729248047, + "drift/rejected_abs": 0.09268493205308914, + "epoch": 0.4533524218563589, + "grad_norm": 46.0, + "ht_mnpo/logit": 0.12961037456989288, + "ht_mnpo/residual": 0.12461037933826447, + "ht_mnpo/residual_abs": 0.14704172313213348, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.351698343334823e-07, + "logits/chosen": -2.151616334915161, + "logits/rejected": -2.2165794372558594, + "logps/chosen": -0.30637693405151367, + "logps/rejected": -0.2831774652004242, + "loss": 0.46157604455947876, + "loss/core": 0.4549567699432373, + "loss/preference_sft": 0.30637693405151367, + "loss/reference_anchor": 0.30032485723495483, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.2221248149871826, + "rewards/margins": 1.2961037158966064, + "rewards/rejected": 0.9260212182998657, + "step": 475 + }, + { + "drift/chosen_abs": 0.20173406600952148, + "drift/rejected_abs": 0.13807883858680725, + "epoch": 0.4581245526127416, + "grad_norm": 2.203125, + "ht_mnpo/logit": 0.06972820311784744, + "ht_mnpo/residual": 0.06472820043563843, + "ht_mnpo/residual_abs": 0.19808796048164368, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.3275204118708942e-07, + "logits/chosen": -2.0868983268737793, + "logits/rejected": -2.093916416168213, + "logps/chosen": -0.2903420329093933, + "logps/rejected": -0.27758854627609253, + "loss": 0.48223456740379333, + "loss/core": 0.47636446356773376, + "loss/preference_sft": 0.2903420329093933, + "loss/reference_anchor": 0.26446932554244995, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.016468048095703, + "rewards/margins": 0.697282075881958, + "rewards/rejected": 1.3191860914230347, + "step": 480 + }, + { + "drift/chosen_abs": 0.3890398144721985, + "drift/rejected_abs": 0.14222189784049988, + "epoch": 0.4628966833691243, + "grad_norm": 1176.0, + "ht_mnpo/logit": 0.24679091572761536, + "ht_mnpo/residual": 0.24179089069366455, + "ht_mnpo/residual_abs": 0.24669161438941956, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.3033133281504132e-07, + "logits/chosen": -2.0273680686950684, + "logits/rejected": -1.986276626586914, + "logps/chosen": -0.3349688649177551, + "logps/rejected": -0.3107163608074188, + "loss": 1.0060511827468872, + "loss/core": 0.9833115339279175, + "loss/preference_sft": 0.3349688649177551, + "loss/reference_anchor": 1.1034791469573975, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.8889527320861816, + "rewards/margins": 2.4679088592529297, + "rewards/rejected": 1.4210439920425415, + "step": 485 + }, + { + "drift/chosen_abs": 0.15872669219970703, + "drift/rejected_abs": 0.0995459258556366, + "epoch": 0.46766881412550704, + "grad_norm": 1.796875, + "ht_mnpo/logit": 0.059221915900707245, + "ht_mnpo/residual": 0.05422192066907883, + "ht_mnpo/residual_abs": 0.10469312965869904, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.2790861954675702e-07, + "logits/chosen": -2.0943925380706787, + "logits/rejected": -2.1815621852874756, + "logps/chosen": -0.29587632417678833, + "logps/rejected": -0.29876887798309326, + "loss": 0.09795178472995758, + "loss/core": 0.09527580440044403, + "loss/preference_sft": 0.29587632417678833, + "loss/reference_anchor": 0.10421168804168701, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.5852258205413818, + "rewards/margins": 0.5922192335128784, + "rewards/rejected": 0.9930065870285034, + "step": 490 + }, + { + "drift/chosen_abs": 0.15152491629123688, + "drift/rejected_abs": 0.06521561741828918, + "epoch": 0.47244094488188976, + "grad_norm": 7.65625, + "ht_mnpo/logit": 0.08643212169408798, + "ht_mnpo/residual": 0.08143211901187897, + "ht_mnpo/residual_abs": 0.08807367831468582, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.2548481246561504e-07, + "logits/chosen": -2.2049636840820312, + "logits/rejected": -2.185023784637451, + "logps/chosen": -0.2812512516975403, + "logps/rejected": -0.27317750453948975, + "loss": 0.03975542262196541, + "loss/core": 0.037954915314912796, + "loss/preference_sft": 0.2812512516975403, + "loss/reference_anchor": 0.06190044432878494, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.5122514963150024, + "rewards/margins": 0.864321231842041, + "rewards/rejected": 0.6479302048683167, + "step": 495 + }, + { + "drift/chosen_abs": 0.1506694257259369, + "drift/rejected_abs": 0.05255242437124252, + "epoch": 0.4772130756382725, + "grad_norm": 43.75, + "ht_mnpo/logit": 0.09834472835063934, + "ht_mnpo/residual": 0.09334473311901093, + "ht_mnpo/residual_abs": 0.09647145867347717, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.230608230663321e-07, + "logits/chosen": -2.069584369659424, + "logits/rejected": -2.1218812465667725, + "logps/chosen": -0.33464083075523376, + "logps/rejected": -0.31745070219039917, + "loss": 0.12984250485897064, + "loss/core": 0.12702324986457825, + "loss/preference_sft": 0.33464083075523376, + "loss/reference_anchor": 0.10749910771846771, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.5064592361450195, + "rewards/margins": 0.983447253704071, + "rewards/rejected": 0.5230119824409485, + "step": 500 + }, + { + "drift/chosen_abs": 0.2962421774864197, + "drift/rejected_abs": 0.14752820134162903, + "epoch": 0.4819852063946552, + "grad_norm": 3.375, + "ht_mnpo/logit": 0.1489817202091217, + "ht_mnpo/residual": 0.1439817100763321, + "ht_mnpo/residual_abs": 0.15336847305297852, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.206375629121874e-07, + "logits/chosen": -2.281447649002075, + "logits/rejected": -2.3297507762908936, + "logps/chosen": -0.2693077027797699, + "logps/rejected": -0.24752524495124817, + "loss": 0.31294775009155273, + "loss/core": 0.3029409945011139, + "loss/preference_sft": 0.2693077027797699, + "loss/reference_anchor": 0.47340482473373413, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.9623029232025146, + "rewards/margins": 1.4898172616958618, + "rewards/rejected": 1.4724856615066528, + "step": 505 + }, + { + "drift/chosen_abs": 0.16579356789588928, + "drift/rejected_abs": 0.13272346556186676, + "epoch": 0.4867573371510379, + "grad_norm": 1.421875, + "ht_mnpo/logit": 0.033492714166641235, + "ht_mnpo/residual": 0.02849271520972252, + "ht_mnpo/residual_abs": 0.10743961483240128, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.1821594329222079e-07, + "logits/chosen": -2.1006534099578857, + "logits/rejected": -2.0847597122192383, + "logps/chosen": -0.3140560984611511, + "logps/rejected": -0.3067092299461365, + "loss": 0.10225538164377213, + "loss/core": 0.09745608270168304, + "loss/preference_sft": 0.3140560984611511, + "loss/reference_anchor": 0.2085585594177246, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.6566555500030518, + "rewards/margins": 0.33492717146873474, + "rewards/rejected": 1.3217281103134155, + "step": 510 + }, + { + "drift/chosen_abs": 0.3283791244029999, + "drift/rejected_abs": 0.14059999585151672, + "epoch": 0.49152946790742064, + "grad_norm": 4.96875, + "ht_mnpo/logit": 0.188704252243042, + "ht_mnpo/residual": 0.18370424211025238, + "ht_mnpo/residual_abs": 0.2544330060482025, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.157968748785344e-07, + "logits/chosen": -2.0187830924987793, + "logits/rejected": -2.02095365524292, + "logps/chosen": -0.27096566557884216, + "logps/rejected": -0.26615461707115173, + "loss": 0.6095878481864929, + "loss/core": 0.600264310836792, + "loss/preference_sft": 0.27096566557884216, + "loss/reference_anchor": 0.4390795826911926, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.2837913036346436, + "rewards/margins": 1.8870424032211304, + "rewards/rejected": 1.3967489004135132, + "step": 515 + }, + { + "drift/chosen_abs": 0.1408330649137497, + "drift/rejected_abs": 0.1474970579147339, + "epoch": 0.4963015986638034, + "grad_norm": 7.71875, + "ht_mnpo/logit": -0.0067166597582399845, + "ht_mnpo/residual": -0.011716673150658607, + "ht_mnpo/residual_abs": 0.13443736732006073, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.1338126738382597e-07, + "logits/chosen": -2.0800325870513916, + "logits/rejected": -2.005817413330078, + "logps/chosen": -0.3182372450828552, + "logps/rejected": -0.29156559705734253, + "loss": 0.3686665892601013, + "loss/core": 0.3629279136657715, + "loss/preference_sft": 0.3182372450828552, + "loss/reference_anchor": 0.2551078200340271, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.406805157661438, + "rewards/margins": -0.06716668605804443, + "rewards/rejected": 1.4739716053009033, + "step": 520 + }, + { + "drift/chosen_abs": 0.10101151466369629, + "drift/rejected_abs": 0.07907047867774963, + "epoch": 0.5010737294201861, + "grad_norm": 0.765625, + "ht_mnpo/logit": 0.021959036588668823, + "ht_mnpo/residual": 0.016959037631750107, + "ht_mnpo/residual_abs": 0.063358373939991, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.1097002921928316e-07, + "logits/chosen": -2.3136537075042725, + "logits/rejected": -2.301931142807007, + "logps/chosen": -0.31532853841781616, + "logps/rejected": -0.2992285192012787, + "loss": 0.032750409096479416, + "loss/core": 0.031238410621881485, + "loss/preference_sft": 0.31532853841781616, + "loss/reference_anchor": 0.04406752064824104, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.0089143514633179, + "rewards/margins": 0.21959033608436584, + "rewards/rejected": 0.789324164390564, + "step": 525 + }, + { + "drift/chosen_abs": 0.22263045608997345, + "drift/rejected_abs": 0.17575280368328094, + "epoch": 0.5058458601765689, + "grad_norm": 344.0, + "ht_mnpo/logit": 0.04713992029428482, + "ht_mnpo/residual": 0.042139921337366104, + "ht_mnpo/residual_abs": 0.13901717960834503, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.0856406715296717e-07, + "logits/chosen": -2.194549798965454, + "logits/rejected": -2.0708463191986084, + "logps/chosen": -0.2923962473869324, + "logps/rejected": -0.28995051980018616, + "loss": 0.18184489011764526, + "loss/core": 0.17629767954349518, + "loss/preference_sft": 0.2923962473869324, + "loss/reference_anchor": 0.2481221854686737, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.2255985736846924, + "rewards/margins": 0.4713992178440094, + "rewards/rejected": 1.7541992664337158, + "step": 530 + }, + { + "drift/chosen_abs": 0.3698323965072632, + "drift/rejected_abs": 0.1586240977048874, + "epoch": 0.5106179909329516, + "grad_norm": 1008.0, + "ht_mnpo/logit": 0.21117202937602997, + "ht_mnpo/residual": 0.20617201924324036, + "ht_mnpo/residual_abs": 0.2581304907798767, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.061642859688146e-07, + "logits/chosen": -2.0579802989959717, + "logits/rejected": -2.1108455657958984, + "logps/chosen": -0.29436683654785156, + "logps/rejected": -0.2859746217727661, + "loss": 1.2907676696777344, + "loss/core": 1.2714307308197021, + "loss/preference_sft": 0.29436683654785156, + "loss/reference_anchor": 0.9374252557754517, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.6979613304138184, + "rewards/margins": 2.111720323562622, + "rewards/rejected": 1.5862407684326172, + "step": 535 + }, + { + "drift/chosen_abs": 0.16676244139671326, + "drift/rejected_abs": 0.11376915872097015, + "epoch": 0.5153901216893343, + "grad_norm": 1.6953125, + "ht_mnpo/logit": 0.05305640026926994, + "ht_mnpo/residual": 0.048056405037641525, + "ht_mnpo/residual_abs": 0.07481013983488083, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.0377158812638491e-07, + "logits/chosen": -2.155945062637329, + "logits/rejected": -2.191561460494995, + "logps/chosen": -0.2934812903404236, + "logps/rejected": -0.2616683840751648, + "loss": 0.029054682701826096, + "loss/core": 0.02675912342965603, + "loss/preference_sft": 0.2934812903404236, + "loss/reference_anchor": 0.08542972803115845, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.6675370931625366, + "rewards/margins": 0.5305640697479248, + "rewards/rejected": 1.1369731426239014, + "step": 540 + }, + { + "drift/chosen_abs": 0.19116166234016418, + "drift/rejected_abs": 0.09059781581163406, + "epoch": 0.520162252445717, + "grad_norm": 0.67578125, + "ht_mnpo/logit": 0.10065528005361557, + "ht_mnpo/residual": 0.09565527737140656, + "ht_mnpo/residual_abs": 0.10974618047475815, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.0138687342148249e-07, + "logits/chosen": -2.0369791984558105, + "logits/rejected": -2.043282985687256, + "logps/chosen": -0.29164591431617737, + "logps/rejected": -0.2742694914340973, + "loss": 0.20747418701648712, + "loss/core": 0.20390582084655762, + "loss/preference_sft": 0.29164591431617737, + "loss/reference_anchor": 0.1492522656917572, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.9103397130966187, + "rewards/margins": 1.0065529346466064, + "rewards/rejected": 0.903786838054657, + "step": 545 + }, + { + "drift/chosen_abs": 0.214359313249588, + "drift/rejected_abs": 0.11399934440851212, + "epoch": 0.5249343832020997, + "grad_norm": 1.8359375, + "ht_mnpo/logit": 0.10017386823892593, + "ht_mnpo/residual": 0.09517386555671692, + "ht_mnpo/residual_abs": 0.13996323943138123, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 9.90110386477801e-08, + "logits/chosen": -2.2847418785095215, + "logits/rejected": -2.3595545291900635, + "logps/chosen": -0.3020949959754944, + "logps/rejected": -0.29033786058425903, + "loss": 0.21369631588459015, + "loss/core": 0.2089644968509674, + "loss/preference_sft": 0.3020949959754944, + "loss/reference_anchor": 0.2063818722963333, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.1406989097595215, + "rewards/margins": 1.0017387866973877, + "rewards/rejected": 1.1389601230621338, + "step": 550 + }, + { + "drift/chosen_abs": 0.18578848242759705, + "drift/rejected_abs": 0.11730077117681503, + "epoch": 0.5297065139584825, + "grad_norm": 10.6875, + "ht_mnpo/logit": 0.06949036568403244, + "ht_mnpo/residual": 0.06449036300182343, + "ht_mnpo/residual_abs": 0.10903620719909668, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 9.664497725957164e-08, + "logits/chosen": -2.161120891571045, + "logits/rejected": -2.135941982269287, + "logps/chosen": -0.2531643509864807, + "logps/rejected": -0.2631688117980957, + "loss": 0.09490617364645004, + "loss/core": 0.09214997291564941, + "loss/preference_sft": 0.2531643509864807, + "loss/reference_anchor": 0.11249303817749023, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.8578850030899048, + "rewards/margins": 0.6949037313461304, + "rewards/rejected": 1.1629812717437744, + "step": 555 + }, + { + "drift/chosen_abs": 0.34908127784729004, + "drift/rejected_abs": 0.12885652482509613, + "epoch": 0.5344786447148652, + "grad_norm": 24.5, + "ht_mnpo/logit": 0.22048592567443848, + "ht_mnpo/residual": 0.21548593044281006, + "ht_mnpo/residual_abs": 0.21782401204109192, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 9.428957903578045e-08, + "logits/chosen": -2.149848699569702, + "logits/rejected": -2.2165474891662598, + "logps/chosen": -0.2862304747104645, + "logps/rejected": -0.26955145597457886, + "loss": 0.7289299368858337, + "loss/core": 0.7147458791732788, + "loss/preference_sft": 0.2862304747104645, + "loss/reference_anchor": 0.6805819869041443, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.490812301635742, + "rewards/margins": 2.2048592567443848, + "rewards/rejected": 1.285953402519226, + "step": 560 + }, + { + "drift/chosen_abs": 0.3812643587589264, + "drift/rejected_abs": 0.156807541847229, + "epoch": 0.5392507754712479, + "grad_norm": 1024.0, + "ht_mnpo/logit": 0.2333361804485321, + "ht_mnpo/residual": 0.2283361852169037, + "ht_mnpo/residual_abs": 0.3491974174976349, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 9.194572974534976e-08, + "logits/chosen": -2.1060054302215576, + "logits/rejected": -2.117906332015991, + "logps/chosen": -0.2742339074611664, + "logps/rejected": -0.3001040816307068, + "loss": 1.6279220581054688, + "loss/core": 1.6076691150665283, + "loss/preference_sft": 0.2742339074611664, + "loss/reference_anchor": 0.9852186441421509, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.8111414909362793, + "rewards/margins": 2.3333616256713867, + "rewards/rejected": 1.4777801036834717, + "step": 565 + }, + { + "drift/chosen_abs": 0.3591713607311249, + "drift/rejected_abs": 0.15439173579216003, + "epoch": 0.5440229062276306, + "grad_norm": 0.796875, + "ht_mnpo/logit": 0.2359723299741745, + "ht_mnpo/residual": 0.23097236454486847, + "ht_mnpo/residual_abs": 0.241423562169075, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 8.96143108141412e-08, + "logits/chosen": -2.0038273334503174, + "logits/rejected": -1.9924920797348022, + "logps/chosen": -0.28397005796432495, + "logps/rejected": -0.2956791818141937, + "loss": 0.6347066164016724, + "loss/core": 0.6228178143501282, + "loss/preference_sft": 0.28397005796432495, + "loss/reference_anchor": 0.566045880317688, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.590536117553711, + "rewards/margins": 2.3597235679626465, + "rewards/rejected": 1.230812668800354, + "step": 570 + }, + { + "drift/chosen_abs": 0.22817322611808777, + "drift/rejected_abs": 0.11515506356954575, + "epoch": 0.5487950369840133, + "grad_norm": 2.5, + "ht_mnpo/logit": 0.11322301626205444, + "ht_mnpo/residual": 0.10822302103042603, + "ht_mnpo/residual_abs": 0.11970474570989609, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 8.72961989934668e-08, + "logits/chosen": -2.163687229156494, + "logits/rejected": -2.2072675228118896, + "logps/chosen": -0.30935314297676086, + "logps/rejected": -0.2823563814163208, + "loss": 0.10977880656719208, + "loss/core": 0.10459931939840317, + "loss/preference_sft": 0.30935314297676086, + "loss/reference_anchor": 0.22803907096385956, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.281670331954956, + "rewards/margins": 1.132230281829834, + "rewards/rejected": 1.1494401693344116, + "step": 575 + }, + { + "drift/chosen_abs": 0.19220417737960815, + "drift/rejected_abs": 0.0809175968170166, + "epoch": 0.553567167740396, + "grad_norm": 3.015625, + "ht_mnpo/logit": 0.11272404342889786, + "ht_mnpo/residual": 0.10772404819726944, + "ht_mnpo/residual_abs": 0.13539370894432068, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 8.499226603037854e-08, + "logits/chosen": -2.2288291454315186, + "logits/rejected": -2.188666582107544, + "logps/chosen": -0.2963632345199585, + "logps/rejected": -0.2874986529350281, + "loss": 0.29833313822746277, + "loss/core": 0.29411810636520386, + "loss/preference_sft": 0.2963632345199585, + "loss/reference_anchor": 0.18111544847488403, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.922041654586792, + "rewards/margins": 1.1272404193878174, + "rewards/rejected": 0.7948012948036194, + "step": 580 + }, + { + "drift/chosen_abs": 0.21872587502002716, + "drift/rejected_abs": 0.1608632504940033, + "epoch": 0.5583392984967788, + "grad_norm": 0.443359375, + "ht_mnpo/logit": 0.057856976985931396, + "ht_mnpo/residual": 0.05285697430372238, + "ht_mnpo/residual_abs": 0.09607581794261932, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 8.270337833983987e-08, + "logits/chosen": -1.9375085830688477, + "logits/rejected": -1.851293921470642, + "logps/chosen": -0.27032381296157837, + "logps/rejected": -0.25147318840026855, + "loss": 0.044498372822999954, + "loss/core": 0.04125569015741348, + "loss/preference_sft": 0.27032381296157837, + "loss/reference_anchor": 0.13510197401046753, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.185845375061035, + "rewards/margins": 0.5785697102546692, + "rewards/rejected": 1.6072756052017212, + "step": 585 + }, + { + "drift/chosen_abs": 0.21919722855091095, + "drift/rejected_abs": 0.16523535549640656, + "epoch": 0.5631114292531615, + "grad_norm": 8.4375, + "ht_mnpo/logit": 0.05521801859140396, + "ht_mnpo/residual": 0.05021803453564644, + "ht_mnpo/residual_abs": 0.17762021720409393, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 8.04303966789025e-08, + "logits/chosen": -2.237201452255249, + "logits/rejected": -2.175689697265625, + "logps/chosen": -0.2361256182193756, + "logps/rejected": -0.2443951815366745, + "loss": 0.3992035984992981, + "loss/core": 0.39220893383026123, + "loss/preference_sft": 0.2361256182193756, + "loss/reference_anchor": 0.3261222839355469, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.1915810108184814, + "rewards/margins": 0.5521801114082336, + "rewards/rejected": 1.6394007205963135, + "step": 590 + }, + { + "drift/chosen_abs": 0.14474406838417053, + "drift/rejected_abs": 0.07125679403543472, + "epoch": 0.5678835600095442, + "grad_norm": 0.2109375, + "ht_mnpo/logit": 0.07342871278524399, + "ht_mnpo/residual": 0.06842872500419617, + "ht_mnpo/residual_abs": 0.08838275074958801, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 7.817417582301098e-08, + "logits/chosen": -2.1800007820129395, + "logits/rejected": -2.2645437717437744, + "logps/chosen": -0.2769630551338196, + "logps/rejected": -0.27484917640686035, + "loss": 0.047025274485349655, + "loss/core": 0.04545805603265762, + "loss/preference_sft": 0.2769630551338196, + "loss/reference_anchor": 0.05066481977701187, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.4456117153167725, + "rewards/margins": 0.7342872619628906, + "rewards/rejected": 0.7113243937492371, + "step": 595 + }, + { + "drift/chosen_abs": 0.29197508096694946, + "drift/rejected_abs": 0.22344262897968292, + "epoch": 0.572655690765927, + "grad_norm": 9.1875, + "ht_mnpo/logit": 0.06846772134304047, + "ht_mnpo/residual": 0.06346772611141205, + "ht_mnpo/residual_abs": 0.2963734269142151, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 7.59355642445566e-08, + "logits/chosen": -2.0200035572052, + "logits/rejected": -2.113144636154175, + "logps/chosen": -0.27107173204421997, + "logps/rejected": -0.26764634251594543, + "loss": 1.2578901052474976, + "loss/core": 1.242497444152832, + "loss/preference_sft": 0.27107173204421997, + "loss/reference_anchor": 0.7425199747085571, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.9191036224365234, + "rewards/margins": 0.6846773624420166, + "rewards/rejected": 2.2344260215759277, + "step": 600 + }, + { + "drift/chosen_abs": 0.31894126534461975, + "drift/rejected_abs": 0.131717711687088, + "epoch": 0.5774278215223098, + "grad_norm": 113.5, + "ht_mnpo/logit": 0.1888471394777298, + "ht_mnpo/residual": 0.18384712934494019, + "ht_mnpo/residual_abs": 0.2077023684978485, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 7.37154037938015e-08, + "logits/chosen": -2.216761350631714, + "logits/rejected": -2.2498691082000732, + "logps/chosen": -0.31387758255004883, + "logps/rejected": -0.31475016474723816, + "loss": 0.5773728489875793, + "loss/core": 0.5683153867721558, + "loss/preference_sft": 0.31387758255004883, + "loss/reference_anchor": 0.42148226499557495, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.1892428398132324, + "rewards/margins": 1.8884713649749756, + "rewards/rejected": 1.3007718324661255, + "step": 605 + }, + { + "drift/chosen_abs": 0.19281364977359772, + "drift/rejected_abs": 0.10097751766443253, + "epoch": 0.5821999522786925, + "grad_norm": 2.46875, + "ht_mnpo/logit": 0.09355486929416656, + "ht_mnpo/residual": 0.08855487406253815, + "ht_mnpo/residual_abs": 0.10233534872531891, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 7.151452938229325e-08, + "logits/chosen": -2.125278949737549, + "logits/rejected": -2.109140634536743, + "logps/chosen": -0.27237892150878906, + "logps/rejected": -0.28159099817276, + "loss": 0.11355199664831161, + "loss/core": 0.11056599766016006, + "loss/preference_sft": 0.27237892150878906, + "loss/reference_anchor": 0.12206175178289413, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.9259014129638672, + "rewards/margins": 0.9355487823486328, + "rewards/rejected": 0.9903526306152344, + "step": 610 + }, + { + "drift/chosen_abs": 0.31643611192703247, + "drift/rejected_abs": 0.10367069393396378, + "epoch": 0.5869720830350752, + "grad_norm": 59.25, + "ht_mnpo/logit": 0.2132229506969452, + "ht_mnpo/residual": 0.20822295546531677, + "ht_mnpo/residual_abs": 0.2242036610841751, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 6.933376866888883e-08, + "logits/chosen": -2.1933255195617676, + "logits/rejected": -2.2344157695770264, + "logps/chosen": -0.3152548670768738, + "logps/rejected": -0.2855820655822754, + "loss": 0.7186287641525269, + "loss/core": 0.7079665660858154, + "loss/preference_sft": 0.3152548670768738, + "loss/reference_anchor": 0.5015829801559448, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.1643614768981934, + "rewards/margins": 2.1322293281555176, + "rewards/rejected": 1.0321319103240967, + "step": 615 + }, + { + "drift/chosen_abs": 0.18896803259849548, + "drift/rejected_abs": 0.14922986924648285, + "epoch": 0.5917442137914579, + "grad_norm": 35.25, + "ht_mnpo/logit": 0.039714448153972626, + "ht_mnpo/residual": 0.03471444919705391, + "ht_mnpo/residual_abs": 0.15567417442798615, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 6.717394174850605e-08, + "logits/chosen": -2.108137607574463, + "logits/rejected": -2.1779696941375732, + "logps/chosen": -0.2668481469154358, + "logps/rejected": -0.2571088671684265, + "loss": 0.14907851815223694, + "loss/core": 0.14530247449874878, + "loss/preference_sft": 0.2668481469154358, + "loss/reference_anchor": 0.1621183156967163, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.889099359512329, + "rewards/margins": 0.3971443772315979, + "rewards/rejected": 1.4919549226760864, + "step": 620 + }, + { + "drift/chosen_abs": 0.4413048326969147, + "drift/rejected_abs": 0.23714692890644073, + "epoch": 0.5965163445478406, + "grad_norm": 5.59375, + "ht_mnpo/logit": 0.20502714812755585, + "ht_mnpo/residual": 0.20002715289592743, + "ht_mnpo/residual_abs": 0.2597729563713074, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 6.503586084371926e-08, + "logits/chosen": -1.7817914485931396, + "logits/rejected": -1.8193590641021729, + "logps/chosen": -0.24516813457012177, + "logps/rejected": -0.23617276549339294, + "loss": 0.46254611015319824, + "loss/core": 0.4459630846977234, + "loss/preference_sft": 0.24516813457012177, + "loss/reference_anchor": 0.804632842540741, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 4.412737846374512, + "rewards/margins": 2.05027174949646, + "rewards/rejected": 2.3624656200408936, + "step": 625 + }, + { + "drift/chosen_abs": 0.2613522708415985, + "drift/rejected_abs": 0.12580320239067078, + "epoch": 0.6012884753042234, + "grad_norm": 1.984375, + "ht_mnpo/logit": 0.136060431599617, + "ht_mnpo/residual": 0.1310604363679886, + "ht_mnpo/residual_abs": 0.133602574467659, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 6.29203299993155e-08, + "logits/chosen": -2.093968629837036, + "logits/rejected": -2.1899936199188232, + "logps/chosen": -0.26807597279548645, + "logps/rejected": -0.26248809695243835, + "loss": 0.22901996970176697, + "loss/core": 0.22321954369544983, + "loss/preference_sft": 0.26807597279548645, + "loss/reference_anchor": 0.263213574886322, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.6133341789245605, + "rewards/margins": 1.3606044054031372, + "rewards/rejected": 1.2527296543121338, + "step": 630 + }, + { + "drift/chosen_abs": 0.14114494621753693, + "drift/rejected_abs": 0.09878765046596527, + "epoch": 0.6060606060606061, + "grad_norm": 16.625, + "ht_mnpo/logit": 0.04238896444439888, + "ht_mnpo/residual": 0.03738895803689957, + "ht_mnpo/residual_abs": 0.12258099019527435, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 6.082814477992656e-08, + "logits/chosen": -2.223665952682495, + "logits/rejected": -2.122124195098877, + "logps/chosen": -0.30093568563461304, + "logps/rejected": -0.27837997674942017, + "loss": 0.12603580951690674, + "loss/core": 0.12354050576686859, + "loss/preference_sft": 0.30093568563461304, + "loss/reference_anchor": 0.09467045962810516, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 1.410095453262329, + "rewards/margins": 0.4238895773887634, + "rewards/rejected": 0.9862058758735657, + "step": 635 + }, + { + "drift/chosen_abs": 0.2346729040145874, + "drift/rejected_abs": 0.20684631168842316, + "epoch": 0.6108327368169888, + "grad_norm": 57.0, + "ht_mnpo/logit": 0.027944982051849365, + "ht_mnpo/residual": 0.022944996133446693, + "ht_mnpo/residual_abs": 0.24229581654071808, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 5.87600919708494e-08, + "logits/chosen": -2.0714011192321777, + "logits/rejected": -2.0947563648223877, + "logps/chosen": -0.28593140840530396, + "logps/rejected": -0.2925289273262024, + "loss": 0.5171207785606384, + "loss/core": 0.5091261863708496, + "loss/preference_sft": 0.28593140840530396, + "loss/reference_anchor": 0.3711358904838562, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.345926284790039, + "rewards/margins": 0.279449999332428, + "rewards/rejected": 2.066476583480835, + "step": 640 + }, + { + "drift/chosen_abs": 0.32909509539604187, + "drift/rejected_abs": 0.15848024189472198, + "epoch": 0.6156048675733715, + "grad_norm": 2.78125, + "ht_mnpo/logit": 0.17075712978839874, + "ht_mnpo/residual": 0.16575713455677032, + "ht_mnpo/residual_abs": 0.18923448026180267, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 5.671694928216829e-08, + "logits/chosen": -2.1546435356140137, + "logits/rejected": -2.1636524200439453, + "logps/chosen": -0.2983279228210449, + "logps/rejected": -0.297065794467926, + "loss": 0.4782744348049164, + "loss/core": 0.463600218296051, + "loss/preference_sft": 0.2983279228210449, + "loss/reference_anchor": 0.7038805484771729, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.2895309925079346, + "rewards/margins": 1.707571268081665, + "rewards/rejected": 1.5819594860076904, + "step": 645 + }, + { + "drift/chosen_abs": 0.22253522276878357, + "drift/rejected_abs": 0.11168678849935532, + "epoch": 0.6203769983297542, + "grad_norm": 35.0, + "ht_mnpo/logit": 0.1109624132514, + "ht_mnpo/residual": 0.10596241056919098, + "ht_mnpo/residual_abs": 0.11568085104227066, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 5.469948505629e-08, + "logits/chosen": -2.0919768810272217, + "logits/rejected": -2.1772122383117676, + "logps/chosen": -0.2706933617591858, + "logps/rejected": -0.2707112431526184, + "loss": 0.08376672863960266, + "loss/core": 0.07922674715518951, + "loss/preference_sft": 0.2706933617591858, + "loss/reference_anchor": 0.19992953538894653, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.2252449989318848, + "rewards/margins": 1.1096241474151611, + "rewards/rejected": 1.1156206130981445, + "step": 650 + }, + { + "drift/chosen_abs": 0.39330363273620605, + "drift/rejected_abs": 0.14271840453147888, + "epoch": 0.625149129086137, + "grad_norm": 828.0, + "ht_mnpo/logit": 0.2761969566345215, + "ht_mnpo/residual": 0.2711969316005707, + "ht_mnpo/residual_abs": 0.31647664308547974, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 5.270845797900168e-08, + "logits/chosen": -2.03916335105896, + "logits/rejected": -2.0866518020629883, + "logps/chosen": -0.3057062327861786, + "logps/rejected": -0.313825398683548, + "loss": 1.5293443202972412, + "loss/core": 1.511008858680725, + "loss/preference_sft": 0.3057062327861786, + "loss/reference_anchor": 0.8862015008926392, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.932830810546875, + "rewards/margins": 2.761969804763794, + "rewards/rejected": 1.1708608865737915, + "step": 655 + }, + { + "drift/chosen_abs": 0.2743390202522278, + "drift/rejected_abs": 0.09815811365842819, + "epoch": 0.6299212598425197, + "grad_norm": 16.0, + "ht_mnpo/logit": 0.17759980261325836, + "ht_mnpo/residual": 0.17259980738162994, + "ht_mnpo/residual_abs": 0.17895349860191345, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 5.0744616794160104e-08, + "logits/chosen": -1.9069111347198486, + "logits/rejected": -1.9322376251220703, + "logps/chosen": -0.28189000487327576, + "logps/rejected": -0.2763403058052063, + "loss": 0.4070788025856018, + "loss/core": 0.4006664752960205, + "loss/preference_sft": 0.28189000487327576, + "loss/reference_anchor": 0.29242876172065735, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.7418372631073, + "rewards/margins": 1.7759981155395508, + "rewards/rejected": 0.9658395648002625, + "step": 660 + }, + { + "drift/chosen_abs": 0.19634750485420227, + "drift/rejected_abs": 0.0983700379729271, + "epoch": 0.6346933905989024, + "grad_norm": 15.6875, + "ht_mnpo/logit": 0.09792444109916687, + "ht_mnpo/residual": 0.09292443841695786, + "ht_mnpo/residual_abs": 0.12199316918849945, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 4.880870002211963e-08, + "logits/chosen": -2.0750694274902344, + "logits/rejected": -2.129474401473999, + "logps/chosen": -0.28144150972366333, + "logps/rejected": -0.27089667320251465, + "loss": 0.09351025521755219, + "loss/core": 0.09088621288537979, + "loss/preference_sft": 0.28144150972366333, + "loss/reference_anchor": 0.1030580997467041, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.9629449844360352, + "rewards/margins": 0.9792444109916687, + "rewards/rejected": 0.9837003946304321, + "step": 665 + }, + { + "drift/chosen_abs": 0.1535734385251999, + "drift/rejected_abs": 0.15276674926280975, + "epoch": 0.6394655213552851, + "grad_norm": 89.5, + "ht_mnpo/logit": 0.004352384712547064, + "ht_mnpo/residual": -0.000647619366645813, + "ht_mnpo/residual_abs": 0.15819282829761505, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 4.6901435682004996e-08, + "logits/chosen": -2.4462497234344482, + "logits/rejected": -2.402447462081909, + "logps/chosen": -0.28123825788497925, + "logps/rejected": -0.28739991784095764, + "loss": 0.2847086787223816, + "loss/core": 0.2802528142929077, + "loss/preference_sft": 0.28123825788497925, + "loss/reference_anchor": 0.19467081129550934, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.534631371498108, + "rewards/margins": 0.04352378845214844, + "rewards/rejected": 1.491107702255249, + "step": 670 + }, + { + "drift/chosen_abs": 0.25063109397888184, + "drift/rejected_abs": 0.18034696578979492, + "epoch": 0.6442376521116678, + "grad_norm": 85.0, + "ht_mnpo/logit": 0.07082972675561905, + "ht_mnpo/residual": 0.06582970917224884, + "ht_mnpo/residual_abs": 0.1985233575105667, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 4.502354101793314e-08, + "logits/chosen": -2.028636932373047, + "logits/rejected": -2.079133987426758, + "logps/chosen": -0.2808381915092468, + "logps/rejected": -0.2724851667881012, + "loss": 0.3324066400527954, + "loss/core": 0.3255481719970703, + "loss/preference_sft": 0.2808381915092468, + "loss/reference_anchor": 0.31483718752861023, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.5056729316711426, + "rewards/margins": 0.7082973122596741, + "rewards/rejected": 1.7973754405975342, + "step": 675 + }, + { + "drift/chosen_abs": 0.2721858322620392, + "drift/rejected_abs": 0.16804562509059906, + "epoch": 0.6490097828680506, + "grad_norm": 12.4375, + "ht_mnpo/logit": 0.1044061928987503, + "ht_mnpo/residual": 0.09940619766712189, + "ht_mnpo/residual_abs": 0.10363922268152237, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 4.3175722229287034e-08, + "logits/chosen": -1.9131011962890625, + "logits/rejected": -1.9577534198760986, + "logps/chosen": -0.30100974440574646, + "logps/rejected": -0.2880757451057434, + "loss": 0.09681503474712372, + "loss/core": 0.08140072226524353, + "loss/preference_sft": 0.30100974440574646, + "loss/reference_anchor": 0.7406147718429565, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.7206804752349854, + "rewards/margins": 1.0440620183944702, + "rewards/rejected": 1.6766185760498047, + "step": 680 + }, + { + "drift/chosen_abs": 0.24128063023090363, + "drift/rejected_abs": 0.1263791024684906, + "epoch": 0.6537819136244333, + "grad_norm": 16.125, + "ht_mnpo/logit": 0.11510924249887466, + "ht_mnpo/residual": 0.11010923236608505, + "ht_mnpo/residual_abs": 0.15428678691387177, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 4.135867420514276e-08, + "logits/chosen": -2.1793155670166016, + "logits/rejected": -2.2106401920318604, + "logps/chosen": -0.27883365750312805, + "logps/rejected": -0.3018958270549774, + "loss": 0.25008782744407654, + "loss/core": 0.24473276734352112, + "loss/preference_sft": 0.27883365750312805, + "loss/reference_anchor": 0.23987166583538055, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.4123270511627197, + "rewards/margins": 1.151092290878296, + "rewards/rejected": 1.2612347602844238, + "step": 685 + }, + { + "drift/chosen_abs": 0.2428806573152542, + "drift/rejected_abs": 0.12732911109924316, + "epoch": 0.658554044380816, + "grad_norm": 192.0, + "ht_mnpo/logit": 0.11564892530441284, + "ht_mnpo/residual": 0.11064890772104263, + "ht_mnpo/residual_abs": 0.1703292578458786, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 3.957308026295035e-08, + "logits/chosen": -2.0042989253997803, + "logits/rejected": -1.9776769876480103, + "logps/chosen": -0.306735634803772, + "logps/rejected": -0.28427666425704956, + "loss": 0.25161948800086975, + "loss/core": 0.2459324598312378, + "loss/preference_sft": 0.306735634803772, + "loss/reference_anchor": 0.2536788880825043, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.4288065433502197, + "rewards/margins": 1.1564891338348389, + "rewards/rejected": 1.2723174095153809, + "step": 690 + }, + { + "drift/chosen_abs": 0.2985590100288391, + "drift/rejected_abs": 0.10956205427646637, + "epoch": 0.6633261751371987, + "grad_norm": 872.0, + "ht_mnpo/logit": 0.18910077214241028, + "ht_mnpo/residual": 0.18410077691078186, + "ht_mnpo/residual_abs": 0.19521042704582214, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 3.7819611891566084e-08, + "logits/chosen": -2.094963312149048, + "logits/rejected": -2.1175599098205566, + "logps/chosen": -0.292602002620697, + "logps/rejected": -0.2823565602302551, + "loss": 0.9227179288864136, + "loss/core": 0.9114183187484741, + "loss/preference_sft": 0.292602002620697, + "loss/reference_anchor": 0.5357309579849243, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.9835476875305176, + "rewards/margins": 1.891007661819458, + "rewards/rejected": 1.0925395488739014, + "step": 695 + }, + { + "drift/chosen_abs": 0.4045167863368988, + "drift/rejected_abs": 0.1837201565504074, + "epoch": 0.6680983058935814, + "grad_norm": 716.0, + "ht_mnpo/logit": 0.2208547294139862, + "ht_mnpo/residual": 0.2158547192811966, + "ht_mnpo/residual_abs": 0.29663529992103577, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 3.609892849873286e-08, + "logits/chosen": -2.0619771480560303, + "logits/rejected": -2.0216448307037354, + "logps/chosen": -0.3050151467323303, + "logps/rejected": -0.31216365098953247, + "loss": 0.9451198577880859, + "loss/core": 0.9246786236763, + "loss/preference_sft": 0.3050151467323303, + "loss/reference_anchor": 0.991558849811554, + "rewards/accuracies": 0.875, + "rewards/chosen": 4.043200969696045, + "rewards/margins": 2.2085471153259277, + "rewards/rejected": 1.8346538543701172, + "step": 700 + }, + { + "drift/chosen_abs": 0.27926188707351685, + "drift/rejected_abs": 0.20713751018047333, + "epoch": 0.6728704366499642, + "grad_norm": 1.6875, + "ht_mnpo/logit": 0.07208569347858429, + "ht_mnpo/residual": 0.06708566844463348, + "ht_mnpo/residual_abs": 0.15458956360816956, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 3.4411677163103894e-08, + "logits/chosen": -2.2489805221557617, + "logits/rejected": -2.1912360191345215, + "logps/chosen": -0.2617301940917969, + "logps/rejected": -0.27545157074928284, + "loss": 0.31091055274009705, + "loss/core": 0.296908438205719, + "loss/preference_sft": 0.2617301940917969, + "loss/reference_anchor": 0.6739317178726196, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.791400194168091, + "rewards/margins": 0.7208568453788757, + "rewards/rejected": 2.0705432891845703, + "step": 705 + }, + { + "drift/chosen_abs": 0.30785104632377625, + "drift/rejected_abs": 0.19405075907707214, + "epoch": 0.677642567406347, + "grad_norm": 356.0, + "ht_mnpo/logit": 0.11398478597402573, + "ht_mnpo/residual": 0.10898479074239731, + "ht_mnpo/residual_abs": 0.26380276679992676, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 3.2758492390902945e-08, + "logits/chosen": -2.044776439666748, + "logits/rejected": -2.13199782371521, + "logps/chosen": -0.3172752261161804, + "logps/rejected": -0.2907639443874359, + "loss": 0.7584068775177002, + "loss/core": 0.7458577752113342, + "loss/preference_sft": 0.3172752261161804, + "loss/reference_anchor": 0.5957252979278564, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.077186346054077, + "rewards/margins": 1.139847755432129, + "rewards/rejected": 1.9373382329940796, + "step": 710 + }, + { + "drift/chosen_abs": 0.2990885376930237, + "drift/rejected_abs": 0.13457565009593964, + "epoch": 0.6824146981627297, + "grad_norm": 12.9375, + "ht_mnpo/logit": 0.16464099287986755, + "ht_mnpo/residual": 0.15964099764823914, + "ht_mnpo/residual_abs": 0.16764886677265167, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 3.11399958773126e-08, + "logits/chosen": -1.7991693019866943, + "logits/rejected": -1.8733018636703491, + "logps/chosen": -0.278984010219574, + "logps/rejected": -0.30155643820762634, + "loss": 0.2886965274810791, + "loss/core": 0.28276368975639343, + "loss/preference_sft": 0.278984010219574, + "loss/reference_anchor": 0.26874133944511414, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.9902405738830566, + "rewards/margins": 1.6464097499847412, + "rewards/rejected": 1.3438305854797363, + "step": 715 + }, + { + "drift/chosen_abs": 0.3325175642967224, + "drift/rejected_abs": 0.11430977284908295, + "epoch": 0.6871868289191124, + "grad_norm": 2.671875, + "ht_mnpo/logit": 0.21893766522407532, + "ht_mnpo/residual": 0.2139376401901245, + "ht_mnpo/residual_abs": 0.21863511204719543, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.9556796272679972e-08, + "logits/chosen": -2.025402784347534, + "logits/rejected": -2.015165090560913, + "logps/chosen": -0.2916553318500519, + "logps/rejected": -0.2767946124076843, + "loss": 0.617830753326416, + "loss/core": 0.6057730317115784, + "loss/preference_sft": 0.2916553318500519, + "loss/reference_anchor": 0.5737227201461792, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.32487416267395, + "rewards/margins": 2.1893765926361084, + "rewards/rejected": 1.135497808456421, + "step": 720 + }, + { + "drift/chosen_abs": 0.2585774064064026, + "drift/rejected_abs": 0.17033758759498596, + "epoch": 0.6919589596754951, + "grad_norm": 1.078125, + "ht_mnpo/logit": 0.08847717195749283, + "ht_mnpo/residual": 0.08347717672586441, + "ht_mnpo/residual_abs": 0.19887696206569672, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.8009488953628215e-08, + "logits/chosen": -2.201829195022583, + "logits/rejected": -2.16338849067688, + "logps/chosen": -0.2900460362434387, + "logps/rejected": -0.29323217272758484, + "loss": 0.304708331823349, + "loss/core": 0.2987585663795471, + "loss/preference_sft": 0.2900460362434387, + "loss/reference_anchor": 0.2684846520423889, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.585101366043091, + "rewards/margins": 0.8847716450691223, + "rewards/rejected": 1.7003295421600342, + "step": 725 + }, + { + "drift/chosen_abs": 0.21931128203868866, + "drift/rejected_abs": 0.11406012624502182, + "epoch": 0.6967310904318779, + "grad_norm": 338.0, + "ht_mnpo/logit": 0.10529018938541412, + "ht_mnpo/residual": 0.1002901941537857, + "ht_mnpo/residual_abs": 0.11368098109960556, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.649865579915976e-08, + "logits/chosen": -2.1011104583740234, + "logits/rejected": -2.1871607303619385, + "logps/chosen": -0.30004754662513733, + "logps/rejected": -0.28622013330459595, + "loss": 0.19001969695091248, + "loss/core": 0.18584798276424408, + "loss/preference_sft": 0.30004754662513733, + "loss/reference_anchor": 0.17857913672924042, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.191316604614258, + "rewards/margins": 1.0529019832611084, + "rewards/rejected": 1.138414978981018, + "step": 730 + }, + { + "drift/chosen_abs": 0.12048070132732391, + "drift/rejected_abs": 0.19749411940574646, + "epoch": 0.7015032211882606, + "grad_norm": 151.0, + "ht_mnpo/logit": -0.07722408324480057, + "ht_mnpo/residual": -0.08222408592700958, + "ht_mnpo/residual_abs": 0.16358783841133118, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.5024864971835507e-08, + "logits/chosen": -2.2358853816986084, + "logits/rejected": -2.1690421104431152, + "logps/chosen": -0.3068314492702484, + "logps/rejected": -0.31617873907089233, + "loss": 0.2910768389701843, + "loss/core": 0.2859953045845032, + "loss/preference_sft": 0.3068314492702484, + "loss/reference_anchor": 0.2233956754207611, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.2027002573013306, + "rewards/margins": -0.7722408771514893, + "rewards/rejected": 1.9749412536621094, + "step": 735 + }, + { + "drift/chosen_abs": 0.287181556224823, + "drift/rejected_abs": 0.21348950266838074, + "epoch": 0.7062753519446433, + "grad_norm": 9.625, + "ht_mnpo/logit": 0.07476375997066498, + "ht_mnpo/residual": 0.06976374983787537, + "ht_mnpo/residual_abs": 0.22156234085559845, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.3588670704111997e-08, + "logits/chosen": -2.060324192047119, + "logits/rejected": -1.9929609298706055, + "logps/chosen": -0.2671794891357422, + "logps/rejected": -0.28327035903930664, + "loss": 0.3700963854789734, + "loss/core": 0.36213165521621704, + "loss/preference_sft": 0.2671794891357422, + "loss/reference_anchor": 0.3715202808380127, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.8711066246032715, + "rewards/margins": 0.7476376295089722, + "rewards/rejected": 2.123469114303589, + "step": 740 + }, + { + "drift/chosen_abs": 0.2255537211894989, + "drift/rejected_abs": 0.08047862350940704, + "epoch": 0.711047482701026, + "grad_norm": 23.125, + "ht_mnpo/logit": 0.15348081290721893, + "ht_mnpo/residual": 0.14848081767559052, + "ht_mnpo/residual_abs": 0.15392430126667023, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.219061308991721e-08, + "logits/chosen": -2.09189772605896, + "logits/rejected": -2.121368885040283, + "logps/chosen": -0.27154573798179626, + "logps/rejected": -0.27765482664108276, + "loss": 0.21156100928783417, + "loss/core": 0.20837783813476562, + "loss/preference_sft": 0.27154573798179626, + "loss/reference_anchor": 0.13200418651103973, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.252340316772461, + "rewards/margins": 1.5348081588745117, + "rewards/rejected": 0.717532217502594, + "step": 745 + }, + { + "drift/chosen_abs": 0.24951839447021484, + "drift/rejected_abs": 0.0656016543507576, + "epoch": 0.7158196134574087, + "grad_norm": 14.25, + "ht_mnpo/logit": 0.1838001310825348, + "ht_mnpo/residual": 0.17880013585090637, + "ht_mnpo/residual_abs": 0.1926441490650177, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.0831217881543557e-08, + "logits/chosen": -2.2540266513824463, + "logits/rejected": -2.2791855335235596, + "logps/chosen": -0.31038394570350647, + "logps/rejected": -0.2928916811943054, + "loss": 0.9867032170295715, + "loss/core": 0.9744570851325989, + "loss/preference_sft": 0.31038394570350647, + "loss/reference_anchor": 0.5812680721282959, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.4931938648223877, + "rewards/margins": 1.8380012512207031, + "rewards/rejected": 0.6551925539970398, + "step": 750 + }, + { + "drift/chosen_abs": 0.2845051884651184, + "drift/rejected_abs": 0.11132697761058807, + "epoch": 0.7205917442137915, + "grad_norm": 53.0, + "ht_mnpo/logit": 0.17366446554660797, + "ht_mnpo/residual": 0.16866444051265717, + "ht_mnpo/residual_abs": 0.17682570219039917, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.951099629193366e-08, + "logits/chosen": -2.1384809017181396, + "logits/rejected": -2.107126235961914, + "logps/chosen": -0.278969943523407, + "logps/rejected": -0.27125102281570435, + "loss": 0.3765016496181488, + "loss/core": 0.3697974681854248, + "loss/preference_sft": 0.278969943523407, + "loss/reference_anchor": 0.3073136508464813, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.84492826461792, + "rewards/margins": 1.7366445064544678, + "rewards/rejected": 1.1082838773727417, + "step": 755 + }, + { + "drift/chosen_abs": 0.3657851219177246, + "drift/rejected_abs": 0.19124892354011536, + "epoch": 0.7253638749701742, + "grad_norm": 7.3125, + "ht_mnpo/logit": 0.1753065586090088, + "ht_mnpo/residual": 0.17030654847621918, + "ht_mnpo/residual_abs": 0.22203508019447327, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.823044480243431e-08, + "logits/chosen": -2.1926791667938232, + "logits/rejected": -2.1352715492248535, + "logps/chosen": -0.26466232538223267, + "logps/rejected": -0.26116836071014404, + "loss": 0.4025249481201172, + "loss/core": 0.39287105202674866, + "loss/preference_sft": 0.26466232538223267, + "loss/reference_anchor": 0.45622697472572327, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 3.656014919281006, + "rewards/margins": 1.7530654668807983, + "rewards/rejected": 1.902949333190918, + "step": 760 + }, + { + "drift/chosen_abs": 0.1369520127773285, + "drift/rejected_abs": 0.0672479048371315, + "epoch": 0.7301360057265569, + "grad_norm": 19.375, + "ht_mnpo/logit": 0.07037768512964249, + "ht_mnpo/residual": 0.06537768989801407, + "ht_mnpo/residual_abs": 0.10524247586727142, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.699004497608994e-08, + "logits/chosen": -2.1493887901306152, + "logits/rejected": -2.1789956092834473, + "logps/chosen": -0.28642234206199646, + "logps/rejected": -0.29152998328208923, + "loss": 0.0944807231426239, + "loss/core": 0.09230095148086548, + "loss/preference_sft": 0.28642234206199646, + "loss/reference_anchor": 0.08034622669219971, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.368091106414795, + "rewards/margins": 0.7037767767906189, + "rewards/rejected": 0.6643142700195312, + "step": 765 + }, + { + "drift/chosen_abs": 0.2831539213657379, + "drift/rejected_abs": 0.2444659173488617, + "epoch": 0.7349081364829396, + "grad_norm": 72.0, + "ht_mnpo/logit": 0.03880638629198074, + "ht_mnpo/residual": 0.033806391060352325, + "ht_mnpo/residual_abs": 0.24216315150260925, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.5790263276546658e-08, + "logits/chosen": -2.191681385040283, + "logits/rejected": -2.0962822437286377, + "logps/chosen": -0.27692681550979614, + "logps/rejected": -0.25870591402053833, + "loss": 0.33768293261528015, + "loss/core": 0.32926493883132935, + "loss/preference_sft": 0.27692681550979614, + "loss/reference_anchor": 0.3932059407234192, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.8315391540527344, + "rewards/margins": 0.38806402683258057, + "rewards/rejected": 2.4434750080108643, + "step": 770 + }, + { + "drift/chosen_abs": 0.2792401909828186, + "drift/rejected_abs": 0.08451013267040253, + "epoch": 0.7396802672393223, + "grad_norm": 7.71875, + "ht_mnpo/logit": 0.19487017393112183, + "ht_mnpo/residual": 0.18987016379833221, + "ht_mnpo/residual_abs": 0.21923527121543884, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.4631550892634126e-08, + "logits/chosen": -2.0256760120391846, + "logits/rejected": -2.054231882095337, + "logps/chosen": -0.31334948539733887, + "logps/rejected": -0.2905244827270508, + "loss": 0.7219112515449524, + "loss/core": 0.7123136520385742, + "loss/preference_sft": 0.31334948539733887, + "loss/reference_anchor": 0.4485507607460022, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.791043281555176, + "rewards/margins": 1.9487018585205078, + "rewards/rejected": 0.8423415422439575, + "step": 775 + }, + { + "drift/chosen_abs": 0.20807400345802307, + "drift/rejected_abs": 0.12098662555217743, + "epoch": 0.7444523979957051, + "grad_norm": 1.2890625, + "ht_mnpo/logit": 0.08689337223768234, + "ht_mnpo/residual": 0.08189336955547333, + "ht_mnpo/residual_abs": 0.08538765460252762, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.3514343568692132e-08, + "logits/chosen": -2.0675227642059326, + "logits/rejected": -2.1441054344177246, + "logps/chosen": -0.28695350885391235, + "logps/rejected": -0.2682899236679077, + "loss": 0.037697963416576385, + "loss/core": 0.03433579206466675, + "loss/preference_sft": 0.28695350885391235, + "loss/reference_anchor": 0.13941338658332825, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.0788002014160156, + "rewards/margins": 0.8689336776733398, + "rewards/rejected": 1.2098662853240967, + "step": 780 + }, + { + "drift/chosen_abs": 0.2656870484352112, + "drift/rejected_abs": 0.13376986980438232, + "epoch": 0.7492245287520878, + "grad_norm": 604.0, + "ht_mnpo/logit": 0.1322033405303955, + "ht_mnpo/residual": 0.1272033154964447, + "ht_mnpo/residual_abs": 0.2020864486694336, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.2439061440704724e-08, + "logits/chosen": -2.1118531227111816, + "logits/rejected": -2.0152149200439453, + "logps/chosen": -0.2860553562641144, + "logps/rejected": -0.3056414723396301, + "loss": 0.2933505177497864, + "loss/core": 0.28818944096565247, + "loss/preference_sft": 0.2860553562641144, + "loss/reference_anchor": 0.22944936156272888, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.6555895805358887, + "rewards/margins": 1.322033166885376, + "rewards/rejected": 1.3335561752319336, + "step": 785 + }, + { + "drift/chosen_abs": 0.23867455124855042, + "drift/rejected_abs": 0.11441533267498016, + "epoch": 0.7539966595084705, + "grad_norm": 14.25, + "ht_mnpo/logit": 0.12559519708156586, + "ht_mnpo/residual": 0.12059519439935684, + "ht_mnpo/residual_abs": 0.14111679792404175, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.1406108878304468e-08, + "logits/chosen": -1.9224998950958252, + "logits/rejected": -1.9974676370620728, + "logps/chosen": -0.30875641107559204, + "logps/rejected": -0.2942540645599365, + "loss": 0.2603028118610382, + "loss/core": 0.2551446557044983, + "loss/preference_sft": 0.30875641107559204, + "loss/reference_anchor": 0.2270287722349167, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.3846070766448975, + "rewards/margins": 1.2559518814086914, + "rewards/rejected": 1.128654956817627, + "step": 790 + }, + { + "drift/chosen_abs": 0.4156590402126312, + "drift/rejected_abs": 0.2223910391330719, + "epoch": 0.7587687902648532, + "grad_norm": 152.0, + "ht_mnpo/logit": 0.1967443972826004, + "ht_mnpo/residual": 0.19174440205097198, + "ht_mnpo/residual_abs": 0.25113195180892944, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.0415874332705381e-08, + "logits/chosen": -1.9679896831512451, + "logits/rejected": -1.9452831745147705, + "logps/chosen": -0.32800760865211487, + "logps/rejected": -0.32261085510253906, + "loss": 0.5044857263565063, + "loss/core": 0.4869937300682068, + "loss/preference_sft": 0.32800760865211487, + "loss/reference_anchor": 0.8417966961860657, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 4.15133810043335, + "rewards/margins": 1.9674440622329712, + "rewards/rejected": 2.183894157409668, + "step": 795 + }, + { + "drift/chosen_abs": 0.26482954621315, + "drift/rejected_abs": 0.22209832072257996, + "epoch": 0.7635409210212359, + "grad_norm": 62.5, + "ht_mnpo/logit": 0.04312277212738991, + "ht_mnpo/residual": 0.038122765719890594, + "ht_mnpo/residual_abs": 0.19203929603099823, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 9.468730190622484e-09, + "logits/chosen": -2.1530609130859375, + "logits/rejected": -2.206618070602417, + "logps/chosen": -0.29416975378990173, + "logps/rejected": -0.2798725664615631, + "loss": 0.5578433275222778, + "loss/core": 0.5430841445922852, + "loss/preference_sft": 0.29416975378990173, + "loss/reference_anchor": 0.70854252576828, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.6482741832733154, + "rewards/margins": 0.43122774362564087, + "rewards/rejected": 2.2170467376708984, + "step": 800 + }, + { + "drift/chosen_abs": 0.3293476700782776, + "drift/rejected_abs": 0.13683730363845825, + "epoch": 0.7683130517776187, + "grad_norm": 64.0, + "ht_mnpo/logit": 0.19290654361248016, + "ht_mnpo/residual": 0.18790654838085175, + "ht_mnpo/residual_abs": 0.21803894639015198, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 8.565032634232194e-09, + "logits/chosen": -2.0299782752990723, + "logits/rejected": -2.1024813652038574, + "logps/chosen": -0.2577776312828064, + "logps/rejected": -0.2615029811859131, + "loss": 0.2823915481567383, + "loss/core": 0.2752048373222351, + "loss/preference_sft": 0.2577776312828064, + "loss/reference_anchor": 0.3335556983947754, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.292163133621216, + "rewards/margins": 1.929065465927124, + "rewards/rejected": 1.3630976676940918, + "step": 805 + }, + { + "drift/chosen_abs": 0.20250026881694794, + "drift/rejected_abs": 0.09937586635351181, + "epoch": 0.7730851825340014, + "grad_norm": 7.71875, + "ht_mnpo/logit": 0.10316647589206696, + "ht_mnpo/residual": 0.09816648066043854, + "ht_mnpo/residual_abs": 0.10210967063903809, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 7.70512150722702e-09, + "logits/chosen": -2.126025676727295, + "logits/rejected": -2.2384276390075684, + "logps/chosen": -0.2960754334926605, + "logps/rejected": -0.2822602093219757, + "loss": 0.067779041826725, + "loss/core": 0.06462538242340088, + "loss/preference_sft": 0.2960754334926605, + "loss/reference_anchor": 0.12807531654834747, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.0241200923919678, + "rewards/margins": 1.0316648483276367, + "rewards/rejected": 0.9924553632736206, + "step": 810 + }, + { + "drift/chosen_abs": 0.28037869930267334, + "drift/rejected_abs": 0.09076918661594391, + "epoch": 0.7778573132903842, + "grad_norm": 11.5625, + "ht_mnpo/logit": 0.18967393040657043, + "ht_mnpo/residual": 0.18467390537261963, + "ht_mnpo/residual_abs": 0.18819065392017365, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 6.8893201870139915e-09, + "logits/chosen": -2.2424092292785645, + "logits/rejected": -2.2913436889648438, + "logps/chosen": -0.2803761065006256, + "logps/rejected": -0.2700513005256653, + "loss": 0.47603359818458557, + "loss/core": 0.46868079900741577, + "loss/preference_sft": 0.2803761065006256, + "loss/reference_anchor": 0.33960336446762085, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.8023884296417236, + "rewards/margins": 1.8967387676239014, + "rewards/rejected": 0.9056491851806641, + "step": 815 + }, + { + "drift/chosen_abs": 0.37526756525039673, + "drift/rejected_abs": 0.11981121450662613, + "epoch": 0.7826294440467669, + "grad_norm": 0.734375, + "ht_mnpo/logit": 0.25736114382743835, + "ht_mnpo/residual": 0.2523611783981323, + "ht_mnpo/residual_abs": 0.2597852051258087, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 6.117935463105808e-09, + "logits/chosen": -2.123929500579834, + "logits/rejected": -2.1171352863311768, + "logps/chosen": -0.2949315905570984, + "logps/rejected": -0.30626291036605835, + "loss": 1.008721113204956, + "loss/core": 0.9946671724319458, + "loss/preference_sft": 0.2949315905570984, + "loss/reference_anchor": 0.6732128858566284, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.751708507537842, + "rewards/margins": 2.5736114978790283, + "rewards/rejected": 1.178097128868103, + "step": 820 + }, + { + "drift/chosen_abs": 0.2473944127559662, + "drift/rejected_abs": 0.12352682650089264, + "epoch": 0.7874015748031497, + "grad_norm": 27.75, + "ht_mnpo/logit": 0.12414328753948212, + "ht_mnpo/residual": 0.11914326995611191, + "ht_mnpo/residual_abs": 0.18271097540855408, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 5.391257421749826e-09, + "logits/chosen": -2.0234365463256836, + "logits/rejected": -2.0557427406311035, + "logps/chosen": -0.27447009086608887, + "logps/rejected": -0.26940232515335083, + "loss": 0.41700777411460876, + "loss/core": 0.4105207026004791, + "loss/preference_sft": 0.27447009086608887, + "loss/reference_anchor": 0.2969074547290802, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.4735748767852783, + "rewards/margins": 1.241432785987854, + "rewards/rejected": 1.2321420907974243, + "step": 825 + }, + { + "drift/chosen_abs": 0.2917875647544861, + "drift/rejected_abs": 0.17095866799354553, + "epoch": 0.7921737055595324, + "grad_norm": 224.0, + "ht_mnpo/logit": 0.1259477138519287, + "ht_mnpo/residual": 0.1209477037191391, + "ht_mnpo/residual_abs": 0.19740599393844604, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 4.709559336838462e-09, + "logits/chosen": -2.0139071941375732, + "logits/rejected": -2.0471396446228027, + "logps/chosen": -0.2997593581676483, + "logps/rejected": -0.28049224615097046, + "loss": 0.37333980202674866, + "loss/core": 0.354358971118927, + "loss/preference_sft": 0.2997593581676483, + "loss/reference_anchor": 0.9190651774406433, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.915761947631836, + "rewards/margins": 1.259477138519287, + "rewards/rejected": 1.6562846899032593, + "step": 830 + }, + { + "drift/chosen_abs": 0.2739551067352295, + "drift/rejected_abs": 0.15794166922569275, + "epoch": 0.7969458363159151, + "grad_norm": 6.71875, + "ht_mnpo/logit": 0.11599205434322357, + "ht_mnpo/residual": 0.11099205911159515, + "ht_mnpo/residual_abs": 0.16929420828819275, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 4.073097567142025e-09, + "logits/chosen": -1.9657586812973022, + "logits/rejected": -1.9134137630462646, + "logps/chosen": -0.32205599546432495, + "logps/rejected": -0.27461498975753784, + "loss": 0.4663979411125183, + "loss/core": 0.45191946625709534, + "loss/preference_sft": 0.32205599546432495, + "loss/reference_anchor": 0.6917184591293335, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.7391560077667236, + "rewards/margins": 1.159920573234558, + "rewards/rejected": 1.5792356729507446, + "step": 835 + }, + { + "drift/chosen_abs": 0.17836840450763702, + "drift/rejected_abs": 0.25512903928756714, + "epoch": 0.8017179670722978, + "grad_norm": 476.0, + "ht_mnpo/logit": -0.07672592997550964, + "ht_mnpo/residual": -0.08172592520713806, + "ht_mnpo/residual_abs": 0.20435388386249542, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 3.482111459902695e-09, + "logits/chosen": -2.216046094894409, + "logits/rejected": -2.06156325340271, + "logps/chosen": -0.28472912311553955, + "logps/rejected": -0.2904352843761444, + "loss": 0.6602007746696472, + "loss/core": 0.6512366533279419, + "loss/preference_sft": 0.28472912311553955, + "loss/reference_anchor": 0.4197351336479187, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.781996726989746, + "rewards/margins": -0.7672592997550964, + "rewards/rejected": 2.549255847930908, + "step": 840 + }, + { + "drift/chosen_abs": 0.2331835776567459, + "drift/rejected_abs": 0.17626038193702698, + "epoch": 0.8064900978286805, + "grad_norm": 3.984375, + "ht_mnpo/logit": 0.056995175778865814, + "ht_mnpo/residual": 0.051995180547237396, + "ht_mnpo/residual_abs": 0.18106596171855927, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.9368232608258797e-09, + "logits/chosen": -2.004657506942749, + "logits/rejected": -2.017953872680664, + "logps/chosen": -0.30708542466163635, + "logps/rejected": -0.31051960587501526, + "loss": 0.3166196346282959, + "loss/core": 0.31019559502601624, + "loss/preference_sft": 0.30708542466163635, + "loss/reference_anchor": 0.2904927730560303, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.3293375968933105, + "rewards/margins": 0.5699518322944641, + "rewards/rejected": 1.7593857049942017, + "step": 845 + }, + { + "drift/chosen_abs": 0.2094683200120926, + "drift/rejected_abs": 0.10852241516113281, + "epoch": 0.8112622285850632, + "grad_norm": 1.4140625, + "ht_mnpo/logit": 0.10089180618524551, + "ht_mnpo/residual": 0.0958918035030365, + "ht_mnpo/residual_abs": 0.09958347678184509, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.4374380305025866e-09, + "logits/chosen": -2.041243314743042, + "logits/rejected": -2.049227714538574, + "logps/chosen": -0.2886284291744232, + "logps/rejected": -0.27855145931243896, + "loss": 0.08112355321645737, + "loss/core": 0.07786046713590622, + "loss/preference_sft": 0.2886284291744232, + "loss/reference_anchor": 0.13429181277751923, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.0938479900360107, + "rewards/margins": 1.008918046951294, + "rewards/rejected": 1.084930181503296, + "step": 850 + }, + { + "drift/chosen_abs": 0.16252204775810242, + "drift/rejected_abs": 0.1045575961470604, + "epoch": 0.816034359341446, + "grad_norm": 1.0390625, + "ht_mnpo/logit": 0.05791337043046951, + "ht_mnpo/residual": 0.0529133677482605, + "ht_mnpo/residual_abs": 0.07363729178905487, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.984143567294594e-09, + "logits/chosen": -2.1060547828674316, + "logits/rejected": -2.0476009845733643, + "logps/chosen": -0.29853376746177673, + "logps/rejected": -0.2876315116882324, + "loss": 0.03223501145839691, + "loss/core": 0.029459426179528236, + "loss/preference_sft": 0.29853376746177673, + "loss/reference_anchor": 0.10892575979232788, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.623846411705017, + "rewards/margins": 0.5791337490081787, + "rewards/rejected": 1.044712781906128, + "step": 855 + }, + { + "drift/chosen_abs": 0.3076639473438263, + "drift/rejected_abs": 0.15537162125110626, + "epoch": 0.8208064900978287, + "grad_norm": 16.375, + "ht_mnpo/logit": 0.1523175835609436, + "ht_mnpo/residual": 0.147317573428154, + "ht_mnpo/residual_abs": 0.1500585526227951, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.577110336711096e-09, + "logits/chosen": -2.2431137561798096, + "logits/rejected": -2.4290809631347656, + "logps/chosen": -0.2704775035381317, + "logps/rejected": -0.2497967928647995, + "loss": 0.26532191038131714, + "loss/core": 0.25764161348342896, + "loss/preference_sft": 0.2704775035381317, + "loss/reference_anchor": 0.3569665551185608, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.0762779712677, + "rewards/margins": 1.523175835609436, + "rewards/rejected": 1.5531020164489746, + "step": 860 + }, + { + "drift/chosen_abs": 0.17823238670825958, + "drift/rejected_abs": 0.09530796110630035, + "epoch": 0.8255786208542114, + "grad_norm": 1.5859375, + "ht_mnpo/logit": 0.08302712440490723, + "ht_mnpo/residual": 0.07802711427211761, + "ht_mnpo/residual_abs": 0.10461542755365372, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.2164914073037048e-09, + "logits/chosen": -2.0242111682891846, + "logits/rejected": -2.03706693649292, + "logps/chosen": -0.32036247849464417, + "logps/rejected": -0.2977069318294525, + "loss": 0.25833559036254883, + "loss/core": 0.2516372799873352, + "loss/preference_sft": 0.32036247849464417, + "loss/reference_anchor": 0.3028794229030609, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.7823235988616943, + "rewards/margins": 0.8302712440490723, + "rewards/rejected": 0.9520524740219116, + "step": 865 + }, + { + "drift/chosen_abs": 0.21682997047901154, + "drift/rejected_abs": 0.20350849628448486, + "epoch": 0.8303507516105941, + "grad_norm": 972.0, + "ht_mnpo/logit": 0.013434367254376411, + "ht_mnpo/residual": 0.008434370160102844, + "ht_mnpo/residual_abs": 0.1993631273508072, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 9.024223931035357e-10, + "logits/chosen": -2.161740779876709, + "logits/rejected": -2.1781668663024902, + "logps/chosen": -0.24697892367839813, + "logps/rejected": -0.2532227039337158, + "loss": 0.6162554025650024, + "loss/core": 0.6080134510993958, + "loss/preference_sft": 0.24697892367839813, + "loss/reference_anchor": 0.3873947560787201, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.1679275035858154, + "rewards/margins": 0.13434359431266785, + "rewards/rejected": 2.033583641052246, + "step": 870 + }, + { + "drift/chosen_abs": 0.2778306007385254, + "drift/rejected_abs": 0.1163145899772644, + "epoch": 0.8351228823669768, + "grad_norm": 1616.0, + "ht_mnpo/logit": 0.17591026425361633, + "ht_mnpo/residual": 0.17091026902198792, + "ht_mnpo/residual_abs": 0.23106594383716583, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 6.350214026223516e-10, + "logits/chosen": -2.0502777099609375, + "logits/rejected": -2.0214967727661133, + "logps/chosen": -0.27098390460014343, + "logps/rejected": -0.2793070077896118, + "loss": 0.7463462352752686, + "loss/core": 0.7381068468093872, + "loss/preference_sft": 0.27098390460014343, + "loss/reference_anchor": 0.3848689794540405, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.7779488563537598, + "rewards/margins": 1.7591025829315186, + "rewards/rejected": 1.0188462734222412, + "step": 875 + }, + { + "drift/chosen_abs": 0.12231247127056122, + "drift/rejected_abs": 0.07507012039422989, + "epoch": 0.8398950131233596, + "grad_norm": 2.71875, + "ht_mnpo/logit": 0.04689433425664902, + "ht_mnpo/residual": 0.0418943352997303, + "ht_mnpo/residual_abs": 0.051383644342422485, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 4.143889944367429e-10, + "logits/chosen": -2.061795711517334, + "logits/rejected": -2.1499526500701904, + "logps/chosen": -0.27764564752578735, + "logps/rejected": -0.27403324842453003, + "loss": 0.010318822227418423, + "loss/core": 0.009077044203877449, + "loss/preference_sft": 0.27764564752578735, + "loss/reference_anchor": 0.0343242809176445, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.2161328792572021, + "rewards/margins": 0.4689432978630066, + "rewards/rejected": 0.7471895217895508, + "step": 880 + }, + { + "drift/chosen_abs": 0.14238771796226501, + "drift/rejected_abs": 0.06504310667514801, + "epoch": 0.8446671438797423, + "grad_norm": 1.9609375, + "ht_mnpo/logit": 0.07748010009527206, + "ht_mnpo/residual": 0.07248009741306305, + "ht_mnpo/residual_abs": 0.08879692852497101, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 2.406081393722531e-10, + "logits/chosen": -2.0421433448791504, + "logits/rejected": -2.0938682556152344, + "logps/chosen": -0.30008670687675476, + "logps/rejected": -0.28426411747932434, + "loss": 0.0430099293589592, + "loss/core": 0.04152857884764671, + "loss/preference_sft": 0.30008670687675476, + "loss/reference_anchor": 0.04405852407217026, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.4236747026443481, + "rewards/margins": 0.7748010754585266, + "rewards/rejected": 0.6488736867904663, + "step": 885 + }, + { + "drift/chosen_abs": 0.34237349033355713, + "drift/rejected_abs": 0.1923760622739792, + "epoch": 0.849439274636125, + "grad_norm": 624.0, + "ht_mnpo/logit": 0.15943066775798798, + "ht_mnpo/residual": 0.15443067252635956, + "ht_mnpo/residual_abs": 0.2659396827220917, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 1.1374418930135133e-10, + "logits/chosen": -2.10282564163208, + "logits/rejected": -2.1714038848876953, + "logps/chosen": -0.2946470379829407, + "logps/rejected": -0.3199237585067749, + "loss": 0.8831459283828735, + "loss/core": 0.8672249913215637, + "loss/preference_sft": 0.2946470379829407, + "loss/reference_anchor": 0.766579806804657, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.4213645458221436, + "rewards/margins": 1.5943067073822021, + "rewards/rejected": 1.8270580768585205, + "step": 890 + }, + { + "drift/chosen_abs": 0.2455691546201706, + "drift/rejected_abs": 0.10265467315912247, + "epoch": 0.8542114053925077, + "grad_norm": 17.75, + "ht_mnpo/logit": 0.1426907479763031, + "ht_mnpo/residual": 0.1376907378435135, + "ht_mnpo/residual_abs": 0.1457034945487976, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 3.3844852567285756e-11, + "logits/chosen": -2.1894445419311523, + "logits/rejected": -2.145141124725342, + "logps/chosen": -0.3019161522388458, + "logps/rejected": -0.2727174162864685, + "loss": 0.39371412992477417, + "loss/core": 0.3860507309436798, + "loss/preference_sft": 0.3019161522388458, + "loss/reference_anchor": 0.3529819846153259, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.4513144493103027, + "rewards/margins": 1.4269073009490967, + "rewards/rejected": 1.024407148361206, + "step": 895 + }, + { + "drift/chosen_abs": 0.253998339176178, + "drift/rejected_abs": 0.14686278998851776, + "epoch": 0.8589835361488904, + "grad_norm": 0.27734375, + "ht_mnpo/logit": 0.10787560790777206, + "ht_mnpo/residual": 0.10287560522556305, + "ht_mnpo/residual_abs": 0.10665620863437653, + "ht_mnpo/target": 0.004999999888241291, + "ht_mnpo/target_abs": 0.004999999888241291, + "learning_rate": 9.401760429905703e-13, + "logits/chosen": -2.1683638095855713, + "logits/rejected": -2.2042431831359863, + "logps/chosen": -0.3270532488822937, + "logps/rejected": -0.30916860699653625, + "loss": 0.22434833645820618, + "loss/core": 0.21211738884449005, + "loss/preference_sft": 0.3270532488822937, + "loss/reference_anchor": 0.5788421034812927, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.5390026569366455, + "rewards/margins": 1.0787560939788818, + "rewards/rejected": 1.4602468013763428, + "step": 900 + }, + { + "epoch": 0.8589835361488904, + "step": 900, + "total_flos": 0.0, + "train_loss": 0.36845981503112446, + "train_runtime": 7265.877, + "train_samples_per_second": 1.982, + "train_steps_per_second": 0.124 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..99aecc2 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b759ceed42dbd8997bb8723ca8101ebdfe20682f5f347d3bfd938053a7076f54 +size 6993 diff --git a/training_status.json b/training_status.json new file mode 100644 index 0000000..1d86ad9 --- /dev/null +++ b/training_status.json @@ -0,0 +1,20 @@ +{ + "status": "completed", + "candidate_id": "ht_safety_a", + "method": "ht_mnpo_safety", + "seed": 42, + "gpu": 1, + "pid": 1096653, + "config": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/fair_demo_20260715/sweep/candidates/ht_safety_a/config.yaml", + "dataset": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/ht_mnpo_safety", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "eec5f721eea6", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/eec5f721eea6", + "started_at": "2026-07-15T15:42:04+09:00", + "returncode": 0, + "model_complete": true, + "finite_results": true, + "measured_step": null, + "completed_at": "2026-07-15T17:44:49+09:00" +}