From c5827a4516e4db79fe2f7e8fe30db195df46c0ea Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 18 Jul 2026 16:12:11 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: promotion/ronpo-qwen3-8b-fair-dpo-s42 Source: Original Platform --- .gitattributes | 36 + README.md | 12 + all_results.json | 9 + chat_template.jinja | 89 + config.json | 71 + config.yaml | 58 + experiment/evaluator_lock.json | 85 + experiment/selection_lock.json | 99 + experiment/sweep_grid.json | 46 + experiment/training_status.json | 20 + generation_config.json | 12 + model.safetensors | 3 + tokenizer.json | 3 + tokenizer_config.json | 30 + train_results.json | 9 + trainer_state.json | 3643 +++++++++++++++++++++++++++++++ training_args.bin | 3 + training_status.json | 20 + 18 files changed, 4248 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 config.yaml create mode 100644 experiment/evaluator_lock.json create mode 100644 experiment/selection_lock.json create mode 100644 experiment/sweep_grid.json create mode 100644 experiment/training_status.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin create mode 100644 training_status.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..b0debc9 --- /dev/null +++ b/README.md @@ -0,0 +1,12 @@ +--- +base_model: Qwen/Qwen3-8B +library_name: transformers +tags: +- preference-optimization +- ronpo +- qwen3 +--- + +# Qwen3-8B fair-demo checkpoint: dpo + +Validation-selected candidate: `dpo_b`. Exact evaluator, selection, sweep, and training metadata are stored under `experiment/`. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..82ad58f --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 0.7066660939322578, + "train_runtime": 7291.0823, + "train_samples": 16746, + "train_samples_per_second": 1.975, + "train_steps_per_second": 0.123 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..7ddbd38 --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +model_name_or_path: /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218 +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/avg: 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: dpo +eta: 0.0075 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.1 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.05 +preference_sft_weight: 0.005 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 5.0e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.15 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 42 +gradient_accumulation_steps: 16 +gradient_checkpointing: true +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 1 +per_device_eval_batch_size: 1 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/fair_demo_20260715/sweep/candidates/dpo_b +run_name: qwen3-8b-fair-demo-dpo_b diff --git a/experiment/evaluator_lock.json b/experiment/evaluator_lock.json new file mode 100644 index 0000000..b406c9a --- /dev/null +++ b/experiment/evaluator_lock.json @@ -0,0 +1,85 @@ +{ + "status": "LOCKED_BEFORE_ANY_NEW_METHOD_RANKING", + "locked_at": "2026-07-15T20:20:29+09:00", + "objective_signals": [ + "skywork", + "armo_safety", + "length_conciseness" + ], + "objective_semantics": [ + "helpfulness", + "safety", + "conciseness" + ], + "primary": { + "name": "open_weight_panel_mean_prompt_worst_vs_base", + "definition": "For each prompt and objective, average win=1/tie=0.5/loss=0 over two A/B positions and two judges; take the minimum objective, then mean over prompts.", + "judges": [ + { + "model": "openai/gpt-oss-120b", + "revision": "b5c939de8f754692c1647ca79fbf85e8c1e70f8a" + }, + { + "model": "Qwen/Qwen3-32B", + "revision": "9216db5781bf21249d130ec9da846c4624c16137" + } + ], + "decode": { + "temperature": 0.0, + "top_p": 1.0, + "max_new_tokens": 512, + "seed": 42 + }, + "position_swap": true, + "validation_selection_rule": "Highest eligible mean prompt-level worst panel score within each method; an exact numeric tie is broken by candidate_id lexical order." + }, + "secondary": { + "normalization": "For each locked signal, divide each evaluation prompt's candidate-minus-base raw delta by the population SD of the diagnostic base and matched-control scores; take the prompt-level minimum across objectives, then the mean. No per-prompt min-max.", + "diagnostic_control_scale": { + "skywork": 22.807301785782975, + "armo_safety": 0.3098083353203591, + "length_conciseness": 1.5891969646897472 + }, + "reward_signal_provenance": { + "skywork": { + "model": "Skywork/Skywork-Reward-V2-Llama-3.1-8B", + "revision": "cba2f842f3f1af2f1b2f0d35e794d789976390c5", + "semantics": "helpfulness" + }, + "armo_safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "revision": "eb2676d20da2f2d41082289d23c59b9f7427f955", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "length_conciseness": { + "deterministic": "-log1p(response_word_count)", + "tokenization": "Python str.split whitespace words" + } + }, + "report_raw_paired_deltas": true + }, + "bootstrap": { + "paired_prompt_resamples": 2000, + "seed": 42, + "interval": "percentile_95" + }, + "power": { + "target_absolute_effect": 0.05, + "paired_sd": 0.13831629563357775, + "required_prompts_80pct_power": 61, + "planned_fresh_test_prompts": 1024, + "alpha_two_sided": 0.05, + "power": 0.8 + }, + "fresh_test_source": { + "dataset": "HuggingFaceH4/ultrachat_200k", + "revision": "8049631c405ae6576f93f445c6b8166f76f5505a", + "split": "test_sft" + }, + "diagnostic_summary_sha256": "e365ed814926ac5c17777e0fc65d0299a45b81bd390542889bbd55b5453d4ffe", + "judge_diagnostic_lock_sha256": "fce2d97806053cabdf3d358806f4baa80df7f89d71d8c1c298883327d4e02635", + "prereg_sha256": "71cbd1ac17c3e807f2ece1cb0a82374bbe74edb654c2845b97c3222ed680f0d0", + "method_ranking_computed": false, + "spent_sealed_split_touched": false +} diff --git a/experiment/selection_lock.json b/experiment/selection_lock.json new file mode 100644 index 0000000..bae8b9c --- /dev/null +++ b/experiment/selection_lock.json @@ -0,0 +1,99 @@ +{ + "status": "VALIDATION_SELECTION_LOCKED_BEFORE_FRESH_TEST", + "locked_at": "2026-07-15T20:37:11+09:00", + "selection_metric": "open_weight_panel_mean_prompt_worst_vs_base", + "tie_break": "candidate_id lexical order", + "selected_by_method": { + "dpo": { + "candidate_id": "dpo_b", + "validation_primary": 0.40625, + "validation_primary_ci95": [ + 0.3837890625, + 0.427734375 + ], + "eligible_candidates": [ + "dpo_a", + "dpo_b" + ] + }, + "ht_mnpo_helpfulness": { + "candidate_id": "ht_help_b", + "validation_primary": 0.4072265625, + "validation_primary_ci95": [ + 0.3828125, + 0.4296875 + ], + "eligible_candidates": [ + "ht_help_b" + ] + }, + "ht_mnpo_safety": { + "candidate_id": "ht_safety_a", + "validation_primary": 0.4150390625, + "validation_primary_ci95": [ + 0.3925537109375, + 0.4384765625 + ], + "eligible_candidates": [ + "ht_safety_a", + "ht_safety_b" + ] + }, + "inpo_avg": { + "candidate_id": "inpo_b", + "validation_primary": 0.40625, + "validation_primary_ci95": [ + 0.3818359375, + 0.427734375 + ], + "eligible_candidates": [ + "inpo_b" + ] + }, + "ronpo_full_expect": { + "candidate_id": "ronpo_full_a", + "validation_primary": 0.4033203125, + "validation_primary_ci95": [ + 0.37890625, + 0.42580566406249987 + ], + "eligible_candidates": [ + "ronpo_full_a", + "ronpo_full_b" + ] + }, + "ronpo_k_only": { + "candidate_id": "ronpo_top_a", + "validation_primary": 0.412109375, + "validation_primary_ci95": [ + 0.3876953125, + 0.43557128906249987 + ], + "eligible_candidates": [ + "ronpo_top_a" + ] + }, + "sppo_avg": { + "candidate_id": "sppo_b", + "validation_primary": 0.4111328125, + "validation_primary_ci95": [ + 0.3866943359375, + 0.4345703125 + ], + "eligible_candidates": [ + "sppo_b" + ] + } + }, + "selected_ronpo_overall": "ronpo_top_a", + "failed_methods": [ + "ht_mnpo_conciseness", + "ipo", + "simpo" + ], + "panel_summary_sha256": "8850f23e16cadf56a99e3d562d224ce5337ab36fa588d8a843382853e6ad17bd", + "evaluator_lock_sha256": "ab7b12c84932ed57acfec7edbcbc1ff1df5e9acd203869b1458343adf6e5b2c8", + "grid_sha256": "abfd982ef10c3bc71364752241150286ebf06be5e97f087596adf80e55d459e0", + "fresh_test_opened": false, + "spent_sealed_split_touched": false +} diff --git a/experiment/sweep_grid.json b/experiment/sweep_grid.json new file mode 100644 index 0000000..1ee759f --- /dev/null +++ b/experiment/sweep_grid.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "status": "frozen_before_training_and_validation_ranking", + "frozen_at_kst": "2026-07-15T09:01:45+09:00", + "seed": 42, + "common": { + "optimizer_steps": 900, + "effective_batch_size": 16, + "per_device_train_batch_size": 1, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "bf16": true, + "attn_implementation": "sdpa", + "cudnn_sdpa": false, + "lr_scheduler_type": "cosine", + "max_length": 2048, + "max_prompt_length": 1800, + "save_total_limit": 1, + "wandb_entity": "promotion-kim", + "wandb_project": "mnpo", + "wandb_group": "qwen3-8b-fair-demo-20260715" + }, + "budget_rule": "Exactly two configs per reported method; no extension after validation rankings are visible.", + "candidates": [ + {"id":"ronpo_full_a","method":"ronpo_full_expect","dataset":"ronpo_full_expect_kall","learning_rate":1e-7,"warmup_ratio":0.15,"ronpo_alpha":0.25,"ronpo_tau":0.10,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075,"expected_support_k":0}, + {"id":"ronpo_full_b","method":"ronpo_full_expect","dataset":"ronpo_full_expect_k6","learning_rate":2.5e-7,"warmup_ratio":0.20,"ronpo_alpha":0.35,"ronpo_tau":0.05,"reference_anchor_weight":0.10,"preference_sft_weight":0.01,"expected_support_k":6}, + {"id":"ronpo_top_a","method":"ronpo_k_only","dataset":"ronpo_k_only_k1","learning_rate":5e-8,"warmup_ratio":0.20,"ronpo_alpha":0.15,"ronpo_tau":0.10,"reference_anchor_weight":0.10,"preference_sft_weight":0.01,"expected_support_k":1}, + {"id":"ronpo_top_b","method":"ronpo_k_only","dataset":"ronpo_k_only_k2","learning_rate":1e-7,"warmup_ratio":0.15,"ronpo_alpha":0.25,"ronpo_tau":0.10,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075,"expected_support_k":2}, + {"id":"dpo_a","method":"dpo","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.10,"dpo_beta":0.05,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"dpo_b","method":"dpo","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.15,"dpo_beta":0.10,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ipo_a","method":"ipo","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.15,"dpo_beta":0.20,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ipo_b","method":"ipo","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.20,"dpo_beta":0.50,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075}, + {"id":"simpo_a","method":"simpo","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.10,"simpo_beta":1.0,"simpo_gamma":0.3,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"simpo_b","method":"simpo","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.15,"simpo_beta":2.0,"simpo_gamma":0.6,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"sppo_a","method":"sppo_avg","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.15,"eta":0.005,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"sppo_b","method":"sppo_avg","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.20,"eta":0.010,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075}, + {"id":"inpo_a","method":"inpo_avg","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.15,"eta":0.050,"ratio":0.3333,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"inpo_b","method":"inpo_avg","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.20,"eta":0.100,"ratio":0.3333,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075}, + {"id":"ht_help_a","method":"ht_mnpo_helpfulness","dataset":"ht_mnpo_helpfulness","learning_rate":2.5e-7,"warmup_ratio":0.10,"eta":0.005,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"ht_help_b","method":"ht_mnpo_helpfulness","dataset":"ht_mnpo_helpfulness","learning_rate":5e-7,"warmup_ratio":0.15,"eta":0.010,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ht_safety_a","method":"ht_mnpo_safety","dataset":"ht_mnpo_safety","learning_rate":2.5e-7,"warmup_ratio":0.10,"eta":0.005,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"ht_safety_b","method":"ht_mnpo_safety","dataset":"ht_mnpo_safety","learning_rate":5e-7,"warmup_ratio":0.15,"eta":0.010,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ht_concise_a","method":"ht_mnpo_conciseness","dataset":"ht_mnpo_conciseness","learning_rate":2.5e-7,"warmup_ratio":0.10,"eta":0.005,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"ht_concise_b","method":"ht_mnpo_conciseness","dataset":"ht_mnpo_conciseness","learning_rate":5e-7,"warmup_ratio":0.15,"eta":0.010,"reference_anchor_weight":0.05,"preference_sft_weight":0.005} + ] +} diff --git a/experiment/training_status.json b/experiment/training_status.json new file mode 100644 index 0000000..88e5eef --- /dev/null +++ b/experiment/training_status.json @@ -0,0 +1,20 @@ +{ + "status": "completed", + "candidate_id": "dpo_b", + "method": "dpo", + "seed": 42, + "gpu": 0, + "pid": 423793, + "config": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/fair_demo_20260715/sweep/candidates/dpo_b/config.yaml", + "dataset": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/avg", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "bce8f73e973a", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/bce8f73e973a", + "started_at": "2026-07-15T11:34:23+09:00", + "returncode": 0, + "model_complete": true, + "finite_results": true, + "measured_step": null, + "completed_at": "2026-07-15T13:36:52+09:00" +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..fa59032 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:065758f41205823cfe84fa5d700950d002bbc87aad92fa59581e52db7bdb1044 +size 16381517208 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..e2cb12d --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": true, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..82ad58f --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 0.7066660939322578, + "train_runtime": 7291.0823, + "train_samples": 16746, + "train_samples_per_second": 1.975, + "train_steps_per_second": 0.123 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..2af08cf --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,3643 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8599068434252956, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "drift/chosen_abs": 0.24220439791679382, + "drift/rejected_abs": 0.09696104377508163, + "epoch": 0.004777260241251642, + "grad_norm": 0.9375, + "learning_rate": 1.4814814814814814e-08, + "logits/chosen": -1.9986450672149658, + "logits/rejected": -2.0149338245391846, + "logps/chosen": -0.2793148458003998, + "logps/rejected": -0.2823529839515686, + "loss": 0.6970014572143555, + "loss/core": 0.6860443949699402, + "loss/preference_sft": 0.2793148458003998, + "loss/reference_anchor": 0.191210076212883, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.420919179916382, + "rewards/margins": 1.4643528461456299, + "rewards/rejected": 0.9565665125846863, + "step": 5 + }, + { + "drift/chosen_abs": 0.16979221999645233, + "drift/rejected_abs": 0.15968844294548035, + "epoch": 0.009554520482503284, + "grad_norm": 28.875, + "learning_rate": 3.3333333333333334e-08, + "logits/chosen": -2.343964099884033, + "logits/rejected": -2.3917829990386963, + "logps/chosen": -0.2886165380477905, + "logps/rejected": -0.2906612753868103, + "loss": 0.719261646270752, + "loss/core": 0.6927183866500854, + "loss/preference_sft": 0.2886165380477905, + "loss/reference_anchor": 0.5020042657852173, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.6975072622299194, + "rewards/margins": 0.10625555366277695, + "rewards/rejected": 1.5912517309188843, + "step": 10 + }, + { + "drift/chosen_abs": 0.21399614214897156, + "drift/rejected_abs": 0.11550233513116837, + "epoch": 0.014331780723754926, + "grad_norm": 2.125, + "learning_rate": 5.1851851851851846e-08, + "logits/chosen": -2.2301273345947266, + "logits/rejected": -2.1927645206451416, + "logps/chosen": -0.2770230770111084, + "logps/rejected": -0.2827666401863098, + "loss": 0.6988939642906189, + "loss/core": 0.6884175539016724, + "loss/preference_sft": 0.2770230770111084, + "loss/reference_anchor": 0.1818254441022873, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.138073682785034, + "rewards/margins": 0.9850578308105469, + "rewards/rejected": 1.1530158519744873, + "step": 15 + }, + { + "drift/chosen_abs": 0.19165699183940887, + "drift/rejected_abs": 0.08792886883020401, + "epoch": 0.01910904096500657, + "grad_norm": 0.29296875, + "learning_rate": 7.037037037037038e-08, + "logits/chosen": -2.1778664588928223, + "logits/rejected": -2.2186977863311768, + "logps/chosen": -0.30127808451652527, + "logps/rejected": -0.2866690754890442, + "loss": 0.7005060315132141, + "loss/core": 0.6881817579269409, + "loss/preference_sft": 0.30127808451652527, + "loss/reference_anchor": 0.2163558453321457, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9165700674057007, + "rewards/margins": 1.038002610206604, + "rewards/rejected": 0.8785673379898071, + "step": 20 + }, + { + "drift/chosen_abs": 0.3831271529197693, + "drift/rejected_abs": 0.1648741066455841, + "epoch": 0.02388630120625821, + "grad_norm": 0.671875, + "learning_rate": 8.888888888888888e-08, + "logits/chosen": -2.076991558074951, + "logits/rejected": -2.1371612548828125, + "logps/chosen": -0.2725714147090912, + "logps/rejected": -0.25773054361343384, + "loss": 0.7282012104988098, + "loss/core": 0.6838706135749817, + "loss/preference_sft": 0.2725714147090912, + "loss/reference_anchor": 0.8593538999557495, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.8294517993927, + "rewards/margins": 2.182237148284912, + "rewards/rejected": 1.6472148895263672, + "step": 25 + }, + { + "drift/chosen_abs": 0.22680720686912537, + "drift/rejected_abs": 0.11056505143642426, + "epoch": 0.028663561447509853, + "grad_norm": 5.25, + "learning_rate": 1.074074074074074e-07, + "logits/chosen": -2.348992347717285, + "logits/rejected": -2.3324456214904785, + "logps/chosen": -0.3011496067047119, + "logps/rejected": -0.32406991720199585, + "loss": 0.7054672837257385, + "loss/core": 0.6874420642852783, + "loss/preference_sft": 0.3011496067047119, + "loss/reference_anchor": 0.3303898274898529, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.26779842376709, + "rewards/margins": 1.2391281127929688, + "rewards/rejected": 1.028670310974121, + "step": 30 + }, + { + "drift/chosen_abs": 0.2659522593021393, + "drift/rejected_abs": 0.11598372459411621, + "epoch": 0.033440821688761495, + "grad_norm": 0.251953125, + "learning_rate": 1.2592592592592592e-07, + "logits/chosen": -2.201605796813965, + "logits/rejected": -2.18267822265625, + "logps/chosen": -0.26762351393699646, + "logps/rejected": -0.2652774751186371, + "loss": 0.7072523236274719, + "loss/core": 0.6861520409584045, + "loss/preference_sft": 0.26762351393699646, + "loss/reference_anchor": 0.395240843296051, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.659522771835327, + "rewards/margins": 1.5000025033950806, + "rewards/rejected": 1.1595203876495361, + "step": 35 + }, + { + "drift/chosen_abs": 0.44708338379859924, + "drift/rejected_abs": 0.1636328548192978, + "epoch": 0.03821808193001314, + "grad_norm": 2.546875, + "learning_rate": 1.4444444444444442e-07, + "logits/chosen": -2.0019490718841553, + "logits/rejected": -2.062816858291626, + "logps/chosen": -0.29337409138679504, + "logps/rejected": -0.2815210223197937, + "loss": 0.7314882874488831, + "loss/core": 0.6806979775428772, + "loss/preference_sft": 0.29337409138679504, + "loss/reference_anchor": 0.9864678382873535, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.470605373382568, + "rewards/margins": 2.8643717765808105, + "rewards/rejected": 1.6062335968017578, + "step": 40 + }, + { + "drift/chosen_abs": 0.20861148834228516, + "drift/rejected_abs": 0.08526524901390076, + "epoch": 0.04299534217126478, + "grad_norm": 0.31640625, + "learning_rate": 1.6296296296296298e-07, + "logits/chosen": -2.199146270751953, + "logits/rejected": -2.3609375953674316, + "logps/chosen": -0.28975796699523926, + "logps/rejected": -0.2888508439064026, + "loss": 0.6954045295715332, + "loss/core": 0.6870486736297607, + "loss/preference_sft": 0.28975796699523926, + "loss/reference_anchor": 0.1381419450044632, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.085977792739868, + "rewards/margins": 1.248346209526062, + "rewards/rejected": 0.8376315236091614, + "step": 45 + }, + { + "drift/chosen_abs": 0.28775015473365784, + "drift/rejected_abs": 0.1457628458738327, + "epoch": 0.04777260241251642, + "grad_norm": 3.75, + "learning_rate": 1.8148148148148149e-07, + "logits/chosen": -2.168269395828247, + "logits/rejected": -2.164731979370117, + "logps/chosen": -0.28189998865127563, + "logps/rejected": -0.28475481271743774, + "loss": 0.7057908177375793, + "loss/core": 0.6865909695625305, + "loss/preference_sft": 0.28189998865127563, + "loss/reference_anchor": 0.3558071553707123, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.8775012493133545, + "rewards/margins": 1.4253501892089844, + "rewards/rejected": 1.4521510601043701, + "step": 50 + }, + { + "drift/chosen_abs": 0.22688527405261993, + "drift/rejected_abs": 0.2027493268251419, + "epoch": 0.05254986265376806, + "grad_norm": 0.162109375, + "learning_rate": 2e-07, + "logits/chosen": -2.196228504180908, + "logits/rejected": -2.184458017349243, + "logps/chosen": -0.28806787729263306, + "logps/rejected": -0.29169613122940063, + "loss": 0.7314178347587585, + "loss/core": 0.6928032636642456, + "loss/preference_sft": 0.28806787729263306, + "loss/reference_anchor": 0.7434841394424438, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.2677197456359863, + "rewards/margins": 0.25824469327926636, + "rewards/rejected": 2.009474992752075, + "step": 55 + }, + { + "drift/chosen_abs": 0.2152278870344162, + "drift/rejected_abs": 0.1564132422208786, + "epoch": 0.057327122895019705, + "grad_norm": 20.75, + "learning_rate": 2.1851851851851852e-07, + "logits/chosen": -2.091794013977051, + "logits/rejected": -2.1202189922332764, + "logps/chosen": -0.27512186765670776, + "logps/rejected": -0.2831602692604065, + "loss": 0.7002712488174438, + "loss/core": 0.6903932690620422, + "loss/preference_sft": 0.27512186765670776, + "loss/reference_anchor": 0.17004787921905518, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.149622917175293, + "rewards/margins": 0.5874365568161011, + "rewards/rejected": 1.5621862411499023, + "step": 60 + }, + { + "drift/chosen_abs": 0.2973586320877075, + "drift/rejected_abs": 0.11919713020324707, + "epoch": 0.06210438313627135, + "grad_norm": 0.33984375, + "learning_rate": 2.3703703703703703e-07, + "logits/chosen": -1.9600942134857178, + "logits/rejected": -2.063352108001709, + "logps/chosen": -0.30903658270835876, + "logps/rejected": -0.3050587773323059, + "loss": 0.7107466459274292, + "loss/core": 0.6846051812171936, + "loss/preference_sft": 0.30903658270835876, + "loss/reference_anchor": 0.49192652106285095, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.9714887142181396, + "rewards/margins": 1.7871677875518799, + "rewards/rejected": 1.1843211650848389, + "step": 65 + }, + { + "drift/chosen_abs": 0.36345982551574707, + "drift/rejected_abs": 0.22663752734661102, + "epoch": 0.06688164337752299, + "grad_norm": 1.703125, + "learning_rate": 2.5555555555555553e-07, + "logits/chosen": -1.971685767173767, + "logits/rejected": -1.9568895101547241, + "logps/chosen": -0.3803972005844116, + "logps/rejected": -0.2971033453941345, + "loss": 0.7371006011962891, + "loss/core": 0.6875497698783875, + "loss/preference_sft": 0.3803972005844116, + "loss/reference_anchor": 0.9529773592948914, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.63312029838562, + "rewards/margins": 1.450933575630188, + "rewards/rejected": 2.1821866035461426, + "step": 70 + }, + { + "drift/chosen_abs": 0.2583789527416229, + "drift/rejected_abs": 0.15998633205890656, + "epoch": 0.07165890361877464, + "grad_norm": 0.44140625, + "learning_rate": 2.7407407407407406e-07, + "logits/chosen": -2.086479663848877, + "logits/rejected": -2.0509562492370605, + "logps/chosen": -0.31756237149238586, + "logps/rejected": -0.2885989248752594, + "loss": 0.7078580856323242, + "loss/core": 0.688626229763031, + "loss/preference_sft": 0.31756237149238586, + "loss/reference_anchor": 0.35288190841674805, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.5812525749206543, + "rewards/margins": 0.9826623201370239, + "rewards/rejected": 1.5985901355743408, + "step": 75 + }, + { + "drift/chosen_abs": 0.3245569169521332, + "drift/rejected_abs": 0.127544105052948, + "epoch": 0.07643616386002627, + "grad_norm": 0.29296875, + "learning_rate": 2.9259259259259254e-07, + "logits/chosen": -2.0680508613586426, + "logits/rejected": -2.1884958744049072, + "logps/chosen": -0.2601710259914398, + "logps/rejected": -0.274718701839447, + "loss": 0.7082482576370239, + "loss/core": 0.6839421987533569, + "loss/preference_sft": 0.2601710259914398, + "loss/reference_anchor": 0.46010416746139526, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.245285749435425, + "rewards/margins": 2.0003161430358887, + "rewards/rejected": 1.2449694871902466, + "step": 80 + }, + { + "drift/chosen_abs": 0.34736722707748413, + "drift/rejected_abs": 0.1335596740245819, + "epoch": 0.08121342410127792, + "grad_norm": 0.423828125, + "learning_rate": 3.111111111111111e-07, + "logits/chosen": -2.187932252883911, + "logits/rejected": -2.2362093925476074, + "logps/chosen": -0.31001752614974976, + "logps/rejected": -0.27728381752967834, + "loss": 0.7267902493476868, + "loss/core": 0.683385968208313, + "loss/preference_sft": 0.31001752614974976, + "loss/reference_anchor": 0.8370832204818726, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.4731335639953613, + "rewards/margins": 2.138108015060425, + "rewards/rejected": 1.3350250720977783, + "step": 85 + }, + { + "drift/chosen_abs": 0.1755516231060028, + "drift/rejected_abs": 0.08407329767942429, + "epoch": 0.08599068434252956, + "grad_norm": 0.162109375, + "learning_rate": 3.296296296296296e-07, + "logits/chosen": -2.420429229736328, + "logits/rejected": -2.4072763919830322, + "logps/chosen": -0.28123074769973755, + "logps/rejected": -0.2965225577354431, + "loss": 0.6978365778923035, + "loss/core": 0.6886924505233765, + "loss/preference_sft": 0.28123074769973755, + "loss/reference_anchor": 0.15476098656654358, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.7550804615020752, + "rewards/margins": 0.9150702357292175, + "rewards/rejected": 0.8400100469589233, + "step": 90 + }, + { + "drift/chosen_abs": 0.3212953805923462, + "drift/rejected_abs": 0.21096964180469513, + "epoch": 0.09076794458378121, + "grad_norm": 0.94140625, + "learning_rate": 3.4814814814814814e-07, + "logits/chosen": -2.0551528930664062, + "logits/rejected": -2.111891269683838, + "logps/chosen": -0.2670920491218567, + "logps/rejected": -0.278430700302124, + "loss": 0.7121036648750305, + "loss/core": 0.6880958080291748, + "loss/preference_sft": 0.2670920491218567, + "loss/reference_anchor": 0.4534483850002289, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.212806224822998, + "rewards/margins": 1.103410005569458, + "rewards/rejected": 2.109395980834961, + "step": 95 + }, + { + "drift/chosen_abs": 0.24620285630226135, + "drift/rejected_abs": 0.1283082365989685, + "epoch": 0.09554520482503284, + "grad_norm": 0.39453125, + "learning_rate": 3.666666666666666e-07, + "logits/chosen": -2.2331910133361816, + "logits/rejected": -2.278862953186035, + "logps/chosen": -0.2573774755001068, + "logps/rejected": -0.2470593899488449, + "loss": 0.6977821588516235, + "loss/core": 0.6874433755874634, + "loss/preference_sft": 0.2573774755001068, + "loss/reference_anchor": 0.18103909492492676, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.460810422897339, + "rewards/margins": 1.1784250736236572, + "rewards/rejected": 1.2823854684829712, + "step": 100 + }, + { + "drift/chosen_abs": 0.20025713741779327, + "drift/rejected_abs": 0.07980473339557648, + "epoch": 0.10032246506628449, + "grad_norm": 5.8125, + "learning_rate": 3.8518518518518515e-07, + "logits/chosen": -2.33634614944458, + "logits/rejected": -2.421203374862671, + "logps/chosen": -0.2838136553764343, + "logps/rejected": -0.28014278411865234, + "loss": 0.6988551020622253, + "loss/core": 0.6874551773071289, + "loss/preference_sft": 0.2838136553764343, + "loss/reference_anchor": 0.1996181458234787, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.001106023788452, + "rewards/margins": 1.206559419631958, + "rewards/rejected": 0.7945466041564941, + "step": 105 + }, + { + "drift/chosen_abs": 0.44312191009521484, + "drift/rejected_abs": 0.18942782282829285, + "epoch": 0.10509972530753613, + "grad_norm": 0.4921875, + "learning_rate": 4.0370370370370373e-07, + "logits/chosen": -1.9062073230743408, + "logits/rejected": -1.858873724937439, + "logps/chosen": -0.34249261021614075, + "logps/rejected": -0.29758021235466003, + "loss": 0.7256662249565125, + "loss/core": 0.6815618276596069, + "loss/preference_sft": 0.34249261021614075, + "loss/reference_anchor": 0.8478387594223022, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 4.430351734161377, + "rewards/margins": 2.5402467250823975, + "rewards/rejected": 1.89010488986969, + "step": 110 + }, + { + "drift/chosen_abs": 0.17720595002174377, + "drift/rejected_abs": 0.14460505545139313, + "epoch": 0.10987698554878778, + "grad_norm": 2.0625, + "learning_rate": 4.222222222222222e-07, + "logits/chosen": -2.087794542312622, + "logits/rejected": -2.0313782691955566, + "logps/chosen": -0.30076345801353455, + "logps/rejected": -0.2990095615386963, + "loss": 0.7037674188613892, + "loss/core": 0.6917927861213684, + "loss/preference_sft": 0.30076345801353455, + "loss/reference_anchor": 0.20941567420959473, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.771735429763794, + "rewards/margins": 0.32677847146987915, + "rewards/rejected": 1.4449571371078491, + "step": 115 + }, + { + "drift/chosen_abs": 0.1807553470134735, + "drift/rejected_abs": 0.11256897449493408, + "epoch": 0.11465424579003941, + "grad_norm": 7.9375, + "learning_rate": 4.4074074074074074e-07, + "logits/chosen": -2.038364887237549, + "logits/rejected": -2.074918270111084, + "logps/chosen": -0.3024388551712036, + "logps/rejected": -0.2960285246372223, + "loss": 0.6956692337989807, + "loss/core": 0.6897962689399719, + "loss/preference_sft": 0.3024388551712036, + "loss/reference_anchor": 0.08721615374088287, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.8071529865264893, + "rewards/margins": 0.6840036511421204, + "rewards/rejected": 1.1231491565704346, + "step": 120 + }, + { + "drift/chosen_abs": 0.2066231518983841, + "drift/rejected_abs": 0.10167888551950455, + "epoch": 0.11943150603129106, + "grad_norm": 0.2734375, + "learning_rate": 4.592592592592592e-07, + "logits/chosen": -2.1104722023010254, + "logits/rejected": -2.141206979751587, + "logps/chosen": -0.29907721281051636, + "logps/rejected": -0.30693620443344116, + "loss": 0.6962321400642395, + "loss/core": 0.6880409717559814, + "loss/preference_sft": 0.29907721281051636, + "loss/reference_anchor": 0.13391605019569397, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.0647101402282715, + "rewards/margins": 1.0507093667984009, + "rewards/rejected": 1.014000654220581, + "step": 125 + }, + { + "drift/chosen_abs": 0.18080219626426697, + "drift/rejected_abs": 0.0888272374868393, + "epoch": 0.1242087662725427, + "grad_norm": 0.5703125, + "learning_rate": 4.777777777777778e-07, + "logits/chosen": -1.999070405960083, + "logits/rejected": -2.092583179473877, + "logps/chosen": -0.27993208169937134, + "logps/rejected": -0.2788156569004059, + "loss": 0.6937012076377869, + "loss/core": 0.68858802318573, + "loss/preference_sft": 0.27993208169937134, + "loss/reference_anchor": 0.07427060604095459, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.8078792095184326, + "rewards/margins": 0.9212325811386108, + "rewards/rejected": 0.8866468667984009, + "step": 130 + }, + { + "drift/chosen_abs": 0.3372951149940491, + "drift/rejected_abs": 0.13560067117214203, + "epoch": 0.12898602651379434, + "grad_norm": 0.765625, + "learning_rate": 4.962962962962963e-07, + "logits/chosen": -2.0918431282043457, + "logits/rejected": -2.1655802726745605, + "logps/chosen": -0.30224984884262085, + "logps/rejected": -0.29795363545417786, + "loss": 0.7072810530662537, + "loss/core": 0.6833052635192871, + "loss/preference_sft": 0.30224984884262085, + "loss/reference_anchor": 0.44929036498069763, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.3726329803466797, + "rewards/margins": 2.055931806564331, + "rewards/rejected": 1.3167011737823486, + "step": 135 + }, + { + "drift/chosen_abs": 0.16283419728279114, + "drift/rejected_abs": 0.07167728990316391, + "epoch": 0.13376328675504598, + "grad_norm": 0.294921875, + "learning_rate": 4.999662714939607e-07, + "logits/chosen": -2.4538674354553223, + "logits/rejected": -2.5128872394561768, + "logps/chosen": -0.2825615108013153, + "logps/rejected": -0.28934210538864136, + "loss": 0.6929519176483154, + "loss/core": 0.688643753528595, + "loss/preference_sft": 0.2825615108013153, + "loss/reference_anchor": 0.057906895875930786, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6276466846466064, + "rewards/margins": 0.9115911722183228, + "rewards/rejected": 0.7160555720329285, + "step": 140 + }, + { + "drift/chosen_abs": 0.3645097315311432, + "drift/rejected_abs": 0.17297443747520447, + "epoch": 0.13854054699629761, + "grad_norm": 0.53125, + "learning_rate": 4.998292650357557e-07, + "logits/chosen": -1.7773208618164062, + "logits/rejected": -1.8101383447647095, + "logps/chosen": -0.29743704199790955, + "logps/rejected": -0.3080594837665558, + "loss": 0.7377547025680542, + "loss/core": 0.6842484474182129, + "loss/preference_sft": 0.29743704199790955, + "loss/reference_anchor": 1.0403814315795898, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.643198013305664, + "rewards/margins": 1.9210885763168335, + "rewards/rejected": 1.7221091985702515, + "step": 145 + }, + { + "drift/chosen_abs": 0.3650224804878235, + "drift/rejected_abs": 0.14471963047981262, + "epoch": 0.14331780723754928, + "grad_norm": 2.484375, + "learning_rate": 4.995869303113768e-07, + "logits/chosen": -2.234192371368408, + "logits/rejected": -2.2828991413116455, + "logps/chosen": -0.31630638241767883, + "logps/rejected": -0.31051549315452576, + "loss": 0.7256408333778381, + "loss/core": 0.682770311832428, + "loss/preference_sft": 0.31630638241767883, + "loss/reference_anchor": 0.8257803916931152, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.649608612060547, + "rewards/margins": 2.215174913406372, + "rewards/rejected": 1.4344332218170166, + "step": 150 + }, + { + "drift/chosen_abs": 0.2868257462978363, + "drift/rejected_abs": 0.0814853385090828, + "epoch": 0.1480950674788009, + "grad_norm": 0.455078125, + "learning_rate": 4.992393694893844e-07, + "logits/chosen": -2.098144292831421, + "logits/rejected": -2.1760802268981934, + "logps/chosen": -0.28282657265663147, + "logps/rejected": -0.29578837752342224, + "loss": 0.7028457522392273, + "loss/core": 0.6833733320236206, + "loss/preference_sft": 0.28282657265663147, + "loss/reference_anchor": 0.36116594076156616, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.8672358989715576, + "rewards/margins": 2.075387716293335, + "rewards/rejected": 0.7918483018875122, + "step": 155 + }, + { + "drift/chosen_abs": 0.3154504895210266, + "drift/rejected_abs": 0.14014051854610443, + "epoch": 0.15287232772005255, + "grad_norm": 6.78125, + "learning_rate": 4.987867291017661e-07, + "logits/chosen": -2.05576491355896, + "logits/rejected": -2.029587507247925, + "logps/chosen": -0.310093492269516, + "logps/rejected": -0.31532084941864014, + "loss": 0.7215666174888611, + "loss/core": 0.6853370070457458, + "loss/preference_sft": 0.310093492269516, + "loss/reference_anchor": 0.6935839653015137, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.1545047760009766, + "rewards/margins": 1.7532867193222046, + "rewards/rejected": 1.401218056678772, + "step": 160 + }, + { + "drift/chosen_abs": 0.16013780236244202, + "drift/rejected_abs": 0.07896161824464798, + "epoch": 0.15764958796130418, + "grad_norm": 0.48046875, + "learning_rate": 4.982291999821587e-07, + "logits/chosen": -2.2207119464874268, + "logits/rejected": -2.152768850326538, + "logps/chosen": -0.3112797141075134, + "logps/rejected": -0.3058459758758545, + "loss": 0.6951904892921448, + "loss/core": 0.6891487240791321, + "loss/preference_sft": 0.3112797141075134, + "loss/reference_anchor": 0.08970671147108078, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.6005077362060547, + "rewards/margins": 0.8129563331604004, + "rewards/rejected": 0.7875513434410095, + "step": 165 + }, + { + "drift/chosen_abs": 0.2601718008518219, + "drift/rejected_abs": 0.27558404207229614, + "epoch": 0.16242684820255585, + "grad_norm": 7.875, + "learning_rate": 4.975670171853925e-07, + "logits/chosen": -2.1900925636291504, + "logits/rejected": -2.1342360973358154, + "logps/chosen": -0.27432286739349365, + "logps/rejected": -0.2613202929496765, + "loss": 0.7430840730667114, + "loss/core": 0.6951981782913208, + "loss/preference_sft": 0.27432286739349365, + "loss/reference_anchor": 0.9302865266799927, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.6010234355926514, + "rewards/margins": -0.15333333611488342, + "rewards/rejected": 2.754356861114502, + "step": 170 + }, + { + "drift/chosen_abs": 0.2583249509334564, + "drift/rejected_abs": 0.12602022290229797, + "epoch": 0.16720410844380748, + "grad_norm": 13.4375, + "learning_rate": 4.968004598883922e-07, + "logits/chosen": -2.1797878742218018, + "logits/rejected": -2.2193121910095215, + "logps/chosen": -0.2869110703468323, + "logps/rejected": -0.301838755607605, + "loss": 0.7159601449966431, + "loss/core": 0.6867267489433289, + "loss/preference_sft": 0.2869110703468323, + "loss/reference_anchor": 0.5559752583503723, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 2.583249568939209, + "rewards/margins": 1.3249006271362305, + "rewards/rejected": 1.2583489418029785, + "step": 175 + }, + { + "drift/chosen_abs": 0.3220704197883606, + "drift/rejected_abs": 0.15029826760292053, + "epoch": 0.17198136868505912, + "grad_norm": 15.625, + "learning_rate": 4.959298512724752e-07, + "logits/chosen": -2.19909930229187, + "logits/rejected": -2.1092610359191895, + "logps/chosen": -0.27359136939048767, + "logps/rejected": -0.2609673738479614, + "loss": 0.7153160572052002, + "loss/core": 0.6853613257408142, + "loss/preference_sft": 0.27359136939048767, + "loss/reference_anchor": 0.5717340707778931, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.2184219360351562, + "rewards/margins": 1.718634009361267, + "rewards/rejected": 1.49978768825531, + "step": 180 + }, + { + "drift/chosen_abs": 0.10404728353023529, + "drift/rejected_abs": 0.0626899003982544, + "epoch": 0.17675862892631075, + "grad_norm": 0.314453125, + "learning_rate": 4.949555583870983e-07, + "logits/chosen": -2.204803466796875, + "logits/rejected": -2.1623644828796387, + "logps/chosen": -0.28916627168655396, + "logps/rejected": -0.2922504246234894, + "loss": 0.69410240650177, + "loss/core": 0.691106915473938, + "loss/preference_sft": 0.28916627168655396, + "loss/reference_anchor": 0.03099246881902218, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.0380046367645264, + "rewards/margins": 0.4127095639705658, + "rewards/rejected": 0.625295102596283, + "step": 185 + }, + { + "drift/chosen_abs": 0.28229960799217224, + "drift/rejected_abs": 0.12531210482120514, + "epoch": 0.18153588916756241, + "grad_norm": 0.9375, + "learning_rate": 4.938779919951092e-07, + "logits/chosen": -2.2970175743103027, + "logits/rejected": -2.2739336490631104, + "logps/chosen": -0.2851516008377075, + "logps/rejected": -0.2831699550151825, + "loss": 0.7151316404342651, + "loss/core": 0.6858740448951721, + "loss/preference_sft": 0.2851516008377075, + "loss/reference_anchor": 0.5566385984420776, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.822661876678467, + "rewards/margins": 1.5726929903030396, + "rewards/rejected": 1.2499691247940063, + "step": 190 + }, + { + "drift/chosen_abs": 0.26670801639556885, + "drift/rejected_abs": 0.097129687666893, + "epoch": 0.18631314940881405, + "grad_norm": 0.337890625, + "learning_rate": 4.926976063995686e-07, + "logits/chosen": -1.9850791692733765, + "logits/rejected": -2.0053915977478027, + "logps/chosen": -0.2911596894264221, + "logps/rejected": -0.2897758185863495, + "loss": 0.7147169709205627, + "loss/core": 0.6856262683868408, + "loss/preference_sft": 0.2911596894264221, + "loss/reference_anchor": 0.5526987314224243, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.6668624877929688, + "rewards/margins": 1.7010908126831055, + "rewards/rejected": 0.9657715559005737, + "step": 195 + }, + { + "drift/chosen_abs": 0.26155006885528564, + "drift/rejected_abs": 0.12203337997198105, + "epoch": 0.19109040965006568, + "grad_norm": 0.69140625, + "learning_rate": 4.914148992522156e-07, + "logits/chosen": -1.8971683979034424, + "logits/rejected": -1.9370143413543701, + "logps/chosen": -0.3235601484775543, + "logps/rejected": -0.3167949318885803, + "loss": 0.7003489136695862, + "loss/core": 0.6863821744918823, + "loss/preference_sft": 0.3235601484775543, + "loss/reference_anchor": 0.24697670340538025, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.6145803928375244, + "rewards/margins": 1.3957717418670654, + "rewards/rejected": 1.2188085317611694, + "step": 200 + }, + { + "drift/chosen_abs": 0.3286622166633606, + "drift/rejected_abs": 0.2124643623828888, + "epoch": 0.19586766989131732, + "grad_norm": 0.23046875, + "learning_rate": 4.90030411343657e-07, + "logits/chosen": -2.119131565093994, + "logits/rejected": -2.1706948280334473, + "logps/chosen": -0.2504412829875946, + "logps/rejected": -0.2682013511657715, + "loss": 0.7217109799385071, + "loss/core": 0.6881678104400635, + "loss/preference_sft": 0.2504412829875946, + "loss/reference_anchor": 0.6458190679550171, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.2861106395721436, + "rewards/margins": 1.170446753501892, + "rewards/rejected": 2.115664005279541, + "step": 205 + }, + { + "drift/chosen_abs": 0.2844642698764801, + "drift/rejected_abs": 0.15010371804237366, + "epoch": 0.20064493013256898, + "grad_norm": 0.447265625, + "learning_rate": 4.885447263753696e-07, + "logits/chosen": -2.1142611503601074, + "logits/rejected": -2.138841152191162, + "logps/chosen": -0.3113865256309509, + "logps/rejected": -0.32046282291412354, + "loss": 0.7174853086471558, + "loss/core": 0.6867169141769409, + "loss/preference_sft": 0.3113865256309509, + "loss/reference_anchor": 0.584227442741394, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.8422324657440186, + "rewards/margins": 1.3444972038269043, + "rewards/rejected": 1.4977353811264038, + "step": 210 + }, + { + "drift/chosen_abs": 0.3206380307674408, + "drift/rejected_abs": 0.16881483793258667, + "epoch": 0.20542219037382062, + "grad_norm": 0.828125, + "learning_rate": 4.869584707136108e-07, + "logits/chosen": -2.0880212783813477, + "logits/rejected": -2.079676389694214, + "logps/chosen": -0.2742551863193512, + "logps/rejected": -0.28655797243118286, + "loss": 0.7276748418807983, + "loss/core": 0.6868326663970947, + "loss/preference_sft": 0.2742551863193512, + "loss/reference_anchor": 0.7894173860549927, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.203636884689331, + "rewards/margins": 1.5241820812225342, + "rewards/rejected": 1.6794548034667969, + "step": 215 + }, + { + "drift/chosen_abs": 0.1884107142686844, + "drift/rejected_abs": 0.10744212567806244, + "epoch": 0.21019945061507225, + "grad_norm": 1.421875, + "learning_rate": 4.852723131253429e-07, + "logits/chosen": -2.1026856899261475, + "logits/rejected": -2.0164294242858887, + "logps/chosen": -0.3028913140296936, + "logps/rejected": -0.3013783395290375, + "loss": 0.6961045265197754, + "loss/core": 0.689160168170929, + "loss/preference_sft": 0.3028913140296936, + "loss/reference_anchor": 0.10859811305999756, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.881739616394043, + "rewards/margins": 0.8081823587417603, + "rewards/rejected": 1.0735574960708618, + "step": 220 + }, + { + "drift/chosen_abs": 0.36267831921577454, + "drift/rejected_abs": 0.14132729172706604, + "epoch": 0.2149767108563239, + "grad_norm": 0.35546875, + "learning_rate": 4.834869644962788e-07, + "logits/chosen": -2.1880686283111572, + "logits/rejected": -2.1765666007995605, + "logps/chosen": -0.29012617468833923, + "logps/rejected": -0.2754373252391815, + "loss": 0.7051090598106384, + "loss/core": 0.6825994253158569, + "loss/preference_sft": 0.29012617468833923, + "loss/reference_anchor": 0.42117825150489807, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.6267833709716797, + "rewards/margins": 2.217150926589966, + "rewards/rejected": 1.4096323251724243, + "step": 225 + }, + { + "drift/chosen_abs": 0.18478451669216156, + "drift/rejected_abs": 0.07309754192829132, + "epoch": 0.21975397109757555, + "grad_norm": 5.34375, + "learning_rate": 4.816031775311732e-07, + "logits/chosen": -2.2757763862609863, + "logits/rejected": -2.2832772731781006, + "logps/chosen": -0.2824876308441162, + "logps/rejected": -0.31052327156066895, + "loss": 0.693640410900116, + "loss/core": 0.6876736283302307, + "loss/preference_sft": 0.2824876308441162, + "loss/reference_anchor": 0.0910874456167221, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.8467649221420288, + "rewards/margins": 1.1201399564743042, + "rewards/rejected": 0.7266249060630798, + "step": 230 + }, + { + "drift/chosen_abs": 0.1901187002658844, + "drift/rejected_abs": 0.13333821296691895, + "epoch": 0.2245312313388272, + "grad_norm": 0.447265625, + "learning_rate": 4.796217464364808e-07, + "logits/chosen": -2.0125269889831543, + "logits/rejected": -2.0471303462982178, + "logps/chosen": -0.29326948523521423, + "logps/rejected": -0.2864275574684143, + "loss": 0.703149676322937, + "loss/core": 0.6904449462890625, + "loss/preference_sft": 0.29326948523521423, + "loss/reference_anchor": 0.2247694432735443, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.8984577655792236, + "rewards/margins": 0.5684404373168945, + "rewards/rejected": 1.330017328262329, + "step": 235 + }, + { + "drift/chosen_abs": 0.17612558603286743, + "drift/rejected_abs": 0.10248030722141266, + "epoch": 0.22930849158007882, + "grad_norm": 0.49609375, + "learning_rate": 4.775435065855182e-07, + "logits/chosen": -2.0281519889831543, + "logits/rejected": -2.046496868133545, + "logps/chosen": -0.29431360960006714, + "logps/rejected": -0.2945192754268646, + "loss": 0.6945276260375977, + "loss/core": 0.6894940733909607, + "loss/preference_sft": 0.29431360960006714, + "loss/reference_anchor": 0.07123853266239166, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.760162353515625, + "rewards/margins": 0.7375157475471497, + "rewards/rejected": 1.0226467847824097, + "step": 240 + }, + { + "drift/chosen_abs": 0.3607032895088196, + "drift/rejected_abs": 0.11152495443820953, + "epoch": 0.23408575182133046, + "grad_norm": 12.125, + "learning_rate": 4.753693341662701e-07, + "logits/chosen": -2.0682380199432373, + "logits/rejected": -2.056436538696289, + "logps/chosen": -0.2773371934890747, + "logps/rejected": -0.2957698106765747, + "loss": 0.7130360007286072, + "loss/core": 0.6818927526473999, + "loss/preference_sft": 0.2773371934890747, + "loss/reference_anchor": 0.5951310396194458, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.606236219406128, + "rewards/margins": 2.500974178314209, + "rewards/rejected": 1.105262279510498, + "step": 245 + }, + { + "drift/chosen_abs": 0.2335798293352127, + "drift/rejected_abs": 0.10776785761117935, + "epoch": 0.23886301206258212, + "grad_norm": 0.58203125, + "learning_rate": 4.731001458119869e-07, + "logits/chosen": -1.9081608057022095, + "logits/rejected": -1.988088607788086, + "logps/chosen": -0.30037543177604675, + "logps/rejected": -0.2945137917995453, + "loss": 0.6957753896713257, + "loss/core": 0.6870092153549194, + "loss/preference_sft": 0.30037543177604675, + "loss/reference_anchor": 0.14528658986091614, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.333603620529175, + "rewards/margins": 1.2645257711410522, + "rewards/rejected": 1.069077730178833, + "step": 250 + }, + { + "drift/chosen_abs": 0.2451537400484085, + "drift/rejected_abs": 0.16124595701694489, + "epoch": 0.24364027230383375, + "grad_norm": 0.447265625, + "learning_rate": 4.707368982147317e-07, + "logits/chosen": -2.2194371223449707, + "logits/rejected": -2.215038299560547, + "logps/chosen": -0.2850092053413391, + "logps/rejected": -0.2866383194923401, + "loss": 0.7060943841934204, + "loss/core": 0.689082145690918, + "loss/preference_sft": 0.2850092053413391, + "loss/reference_anchor": 0.3117434084415436, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.4506688117980957, + "rewards/margins": 0.8393514752388, + "rewards/rejected": 1.6113172769546509, + "step": 255 + }, + { + "drift/chosen_abs": 0.2390270233154297, + "drift/rejected_abs": 0.15710371732711792, + "epoch": 0.2484175325450854, + "grad_norm": 0.18359375, + "learning_rate": 4.682805877220377e-07, + "logits/chosen": -2.1546220779418945, + "logits/rejected": -2.227632999420166, + "logps/chosen": -0.28877514600753784, + "logps/rejected": -0.2719945013523102, + "loss": 0.7041021585464478, + "loss/core": 0.6894248127937317, + "loss/preference_sft": 0.28877514600753784, + "loss/reference_anchor": 0.2646702826023102, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.3893117904663086, + "rewards/margins": 0.8218982815742493, + "rewards/rejected": 1.567413568496704, + "step": 260 + }, + { + "drift/chosen_abs": 0.12290436029434204, + "drift/rejected_abs": 0.11683180183172226, + "epoch": 0.25319479278633705, + "grad_norm": 0.1875, + "learning_rate": 4.657322499168474e-07, + "logits/chosen": -2.037142276763916, + "logits/rejected": -2.044814348220825, + "logps/chosen": -0.2997164726257324, + "logps/rejected": -0.3027000427246094, + "loss": 0.7055035829544067, + "loss/core": 0.6932748556137085, + "loss/preference_sft": 0.2997164726257324, + "loss/reference_anchor": 0.21460387110710144, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.2280583381652832, + "rewards/margins": 0.06538932025432587, + "rewards/rejected": 1.1626689434051514, + "step": 265 + }, + { + "drift/chosen_abs": 0.32724061608314514, + "drift/rejected_abs": 0.21098506450653076, + "epoch": 0.2579720530275887, + "grad_norm": 0.369140625, + "learning_rate": 4.630929591809094e-07, + "logits/chosen": -1.9422146081924438, + "logits/rejected": -2.031545400619507, + "logps/chosen": -0.28938454389572144, + "logps/rejected": -0.2925392985343933, + "loss": 0.7183395624160767, + "loss/core": 0.6880876421928406, + "loss/preference_sft": 0.28938454389572144, + "loss/reference_anchor": 0.5760983228683472, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.2724032402038574, + "rewards/margins": 1.1642584800720215, + "rewards/rejected": 2.108144998550415, + "step": 270 + }, + { + "drift/chosen_abs": 0.23237237334251404, + "drift/rejected_abs": 0.1247030645608902, + "epoch": 0.2627493132688403, + "grad_norm": 1.8515625, + "learning_rate": 4.603638282418183e-07, + "logits/chosen": -2.0928521156311035, + "logits/rejected": -2.0464084148406982, + "logps/chosen": -0.2967087924480438, + "logps/rejected": -0.30262163281440735, + "loss": 0.7003341913223267, + "loss/core": 0.6873778104782104, + "loss/preference_sft": 0.2967087924480438, + "loss/reference_anchor": 0.22945299744606018, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.3227198123931885, + "rewards/margins": 1.1851005554199219, + "rewards/rejected": 1.1376192569732666, + "step": 275 + }, + { + "drift/chosen_abs": 0.2046871930360794, + "drift/rejected_abs": 0.08532479405403137, + "epoch": 0.26752657351009196, + "grad_norm": 0.330078125, + "learning_rate": 4.5754600770388765e-07, + "logits/chosen": -2.061699867248535, + "logits/rejected": -1.97652268409729, + "logps/chosen": -0.28828340768814087, + "logps/rejected": -0.3085075914859772, + "loss": 0.6936501264572144, + "loss/core": 0.6868374347686768, + "loss/preference_sft": 0.28828340768814087, + "loss/reference_anchor": 0.10742677748203278, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.044452667236328, + "rewards/margins": 1.2943838834762573, + "rewards/rejected": 0.7500687837600708, + "step": 280 + }, + { + "drift/chosen_abs": 0.2622529864311218, + "drift/rejected_abs": 0.1607482135295868, + "epoch": 0.2723038337513436, + "grad_norm": 4.625, + "learning_rate": 4.5464068556305366e-07, + "logits/chosen": -2.086787700653076, + "logits/rejected": -2.0828757286071777, + "logps/chosen": -0.29549530148506165, + "logps/rejected": -0.2951886057853699, + "loss": 0.7032809853553772, + "loss/core": 0.6883570551872253, + "loss/preference_sft": 0.29549530148506165, + "loss/reference_anchor": 0.26892945170402527, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.6219353675842285, + "rewards/margins": 1.0172988176345825, + "rewards/rejected": 1.604636788368225, + "step": 285 + }, + { + "drift/chosen_abs": 0.14772982895374298, + "drift/rejected_abs": 0.14406272768974304, + "epoch": 0.27708109399259523, + "grad_norm": 20.75, + "learning_rate": 4.516490867060156e-07, + "logits/chosen": -2.1649439334869385, + "logits/rejected": -2.1754214763641357, + "logps/chosen": -0.2871306836605072, + "logps/rejected": -0.2890948951244354, + "loss": 0.7047442197799683, + "loss/core": 0.6933256387710571, + "loss/preference_sft": 0.2871306836605072, + "loss/reference_anchor": 0.19965848326683044, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.4758152961730957, + "rewards/margins": 0.03618486598134041, + "rewards/rejected": 1.4396305084228516, + "step": 290 + }, + { + "drift/chosen_abs": 0.13134348392486572, + "drift/rejected_abs": 0.06761081516742706, + "epoch": 0.28185835423384686, + "grad_norm": 0.2470703125, + "learning_rate": 4.4857247239382153e-07, + "logits/chosen": -2.563570499420166, + "logits/rejected": -2.5265564918518066, + "logps/chosen": -0.24425753951072693, + "logps/rejected": -0.2567130923271179, + "loss": 0.6926940679550171, + "loss/core": 0.6899658441543579, + "loss/preference_sft": 0.24425753951072693, + "loss/reference_anchor": 0.03013770654797554, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.3127048015594482, + "rewards/margins": 0.639759361743927, + "rewards/rejected": 0.6729455590248108, + "step": 295 + }, + { + "drift/chosen_abs": 0.3717862665653229, + "drift/rejected_abs": 0.10653592646121979, + "epoch": 0.28663561447509855, + "grad_norm": 1.0625, + "learning_rate": 4.4541213973012e-07, + "logits/chosen": -1.9797508716583252, + "logits/rejected": -1.9465057849884033, + "logps/chosen": -0.3131170868873596, + "logps/rejected": -0.3174610435962677, + "loss": 0.7069228887557983, + "loss/core": 0.6806262731552124, + "loss/preference_sft": 0.3131170868873596, + "loss/reference_anchor": 0.49461978673934937, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.7166762351989746, + "rewards/margins": 2.6619973182678223, + "rewards/rejected": 1.0546791553497314, + "step": 300 + }, + { + "drift/chosen_abs": 0.21675559878349304, + "drift/rejected_abs": 0.08982045203447342, + "epoch": 0.2914128747163502, + "grad_norm": 0.4375, + "learning_rate": 4.4216942111429964e-07, + "logits/chosen": -2.1226086616516113, + "logits/rejected": -2.1839969158172607, + "logps/chosen": -0.26876428723335266, + "logps/rejected": -0.2606787085533142, + "loss": 0.6940160989761353, + "loss/core": 0.6869435906410217, + "loss/preference_sft": 0.26876428723335266, + "loss/reference_anchor": 0.11457415670156479, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.165956735610962, + "rewards/margins": 1.2678639888763428, + "rewards/rejected": 0.8980928659439087, + "step": 305 + }, + { + "drift/chosen_abs": 0.2610951066017151, + "drift/rejected_abs": 0.21184654533863068, + "epoch": 0.2961901349576018, + "grad_norm": 2.484375, + "learning_rate": 4.388456836797484e-07, + "logits/chosen": -2.1281096935272217, + "logits/rejected": -2.0378966331481934, + "logps/chosen": -0.2733157277107239, + "logps/rejected": -0.28335532546043396, + "loss": 0.7152631282806396, + "loss/core": 0.6907567977905273, + "loss/preference_sft": 0.2733157277107239, + "loss/reference_anchor": 0.46279603242874146, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.6094913482666016, + "rewards/margins": 0.6211642026901245, + "rewards/rejected": 1.9883270263671875, + "step": 310 + }, + { + "drift/chosen_abs": 0.2902871370315552, + "drift/rejected_abs": 0.13477492332458496, + "epoch": 0.30096739519885346, + "grad_norm": 0.271484375, + "learning_rate": 4.354423287174686e-07, + "logits/chosen": -1.9599071741104126, + "logits/rejected": -2.0659215450286865, + "logps/chosen": -0.27025145292282104, + "logps/rejected": -0.2758514881134033, + "loss": 0.698331892490387, + "loss/core": 0.6854726076126099, + "loss/preference_sft": 0.27025145292282104, + "loss/reference_anchor": 0.23015980422496796, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.9021315574645996, + "rewards/margins": 1.5580607652664185, + "rewards/rejected": 1.3440710306167603, + "step": 315 + }, + { + "drift/chosen_abs": 0.15860998630523682, + "drift/rejected_abs": 0.12193727493286133, + "epoch": 0.3057446554401051, + "grad_norm": 0.244140625, + "learning_rate": 4.31960791085291e-07, + "logits/chosen": -1.9995390176773071, + "logits/rejected": -2.0159947872161865, + "logps/chosen": -0.28982076048851013, + "logps/rejected": -0.2935449182987213, + "loss": 0.6978195905685425, + "loss/core": 0.6914322376251221, + "loss/preference_sft": 0.28982076048851013, + "loss/reference_anchor": 0.09876595437526703, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.584044098854065, + "rewards/margins": 0.36467140913009644, + "rewards/rejected": 1.2193728685379028, + "step": 320 + }, + { + "drift/chosen_abs": 0.3709779977798462, + "drift/rejected_abs": 0.1553051918745041, + "epoch": 0.31052191568135673, + "grad_norm": 13.5625, + "learning_rate": 4.2840253860293806e-07, + "logits/chosen": -2.2568116188049316, + "logits/rejected": -2.2034993171691895, + "logps/chosen": -0.2853277027606964, + "logps/rejected": -0.2819185256958008, + "loss": 0.731056809425354, + "loss/core": 0.683963418006897, + "loss/preference_sft": 0.2853277027606964, + "loss/reference_anchor": 0.9133332967758179, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.708606004714966, + "rewards/margins": 2.156642436981201, + "rewards/rejected": 1.5519633293151855, + "step": 325 + }, + { + "drift/chosen_abs": 0.1991027593612671, + "drift/rejected_abs": 0.11322907358407974, + "epoch": 0.31529917592260837, + "grad_norm": 0.20703125, + "learning_rate": 4.24769071433189e-07, + "logits/chosen": -2.227357864379883, + "logits/rejected": -2.2161545753479004, + "logps/chosen": -0.2826363146305084, + "logps/rejected": -0.27046969532966614, + "loss": 0.6973029375076294, + "loss/core": 0.6889919638633728, + "loss/preference_sft": 0.2826363146305084, + "loss/reference_anchor": 0.13795623183250427, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.9903936386108398, + "rewards/margins": 0.8586524724960327, + "rewards/rejected": 1.1317412853240967, + "step": 330 + }, + { + "drift/chosen_abs": 0.2119426280260086, + "drift/rejected_abs": 0.08484821766614914, + "epoch": 0.32007643616386, + "grad_norm": 3.09375, + "learning_rate": 4.2106192144940987e-07, + "logits/chosen": -2.0478832721710205, + "logits/rejected": -2.033395767211914, + "logps/chosen": -0.3026389479637146, + "logps/rejected": -0.32717952132225037, + "loss": 0.6988995671272278, + "loss/core": 0.6861973404884338, + "loss/preference_sft": 0.3026389479637146, + "loss/reference_anchor": 0.22377829253673553, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.1165146827697754, + "rewards/margins": 1.5143001079559326, + "rewards/rejected": 0.6022144556045532, + "step": 335 + }, + { + "drift/chosen_abs": 0.2043064534664154, + "drift/rejected_abs": 0.08732999116182327, + "epoch": 0.3248536964051117, + "grad_norm": 2.265625, + "learning_rate": 4.172826515897145e-07, + "logits/chosen": -2.094050645828247, + "logits/rejected": -2.2169277667999268, + "logps/chosen": -0.2799380123615265, + "logps/rejected": -0.28169578313827515, + "loss": 0.6966040134429932, + "loss/core": 0.6874774694442749, + "loss/preference_sft": 0.2799380123615265, + "loss/reference_anchor": 0.15453729033470154, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.0414788722991943, + "rewards/margins": 1.1714600324630737, + "rewards/rejected": 0.8700188398361206, + "step": 340 + }, + { + "drift/chosen_abs": 0.164683997631073, + "drift/rejected_abs": 0.08445769548416138, + "epoch": 0.3296309566463633, + "grad_norm": 0.4921875, + "learning_rate": 4.1343285519802783e-07, + "logits/chosen": -2.3189194202423096, + "logits/rejected": -2.378885269165039, + "logps/chosen": -0.2820732891559601, + "logps/rejected": -0.29256579279899597, + "loss": 0.6956444978713989, + "loss/core": 0.68927001953125, + "loss/preference_sft": 0.2820732891559601, + "loss/reference_anchor": 0.09928078949451447, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.6447336673736572, + "rewards/margins": 0.8007295727729797, + "rewards/rejected": 0.8440041542053223, + "step": 345 + }, + { + "drift/chosen_abs": 0.19914595782756805, + "drift/rejected_abs": 0.09469764679670334, + "epoch": 0.33440821688761496, + "grad_norm": 0.5234375, + "learning_rate": 4.0951415535233056e-07, + "logits/chosen": -2.0590832233428955, + "logits/rejected": -2.0390827655792236, + "logps/chosen": -0.2612057030200958, + "logps/rejected": -0.25672319531440735, + "loss": 0.6943715810775757, + "loss/core": 0.6880162954330444, + "loss/preference_sft": 0.2612057030200958, + "loss/reference_anchor": 0.10098560899496078, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.98989999294281, + "rewards/margins": 1.0453819036483765, + "rewards/rejected": 0.9445180892944336, + "step": 350 + }, + { + "drift/chosen_abs": 0.18552103638648987, + "drift/rejected_abs": 0.13385716080665588, + "epoch": 0.3391854771288666, + "grad_norm": 0.375, + "learning_rate": 4.0552820418036847e-07, + "logits/chosen": -2.1061980724334717, + "logits/rejected": -2.232513904571533, + "logps/chosen": -0.28036990761756897, + "logps/rejected": -0.29538074135780334, + "loss": 0.7025222182273865, + "loss/core": 0.6908389329910278, + "loss/preference_sft": 0.28036990761756897, + "loss/reference_anchor": 0.20562918484210968, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8541920185089111, + "rewards/margins": 0.5308849215507507, + "rewards/rejected": 1.3233067989349365, + "step": 355 + }, + { + "drift/chosen_abs": 0.2627464234828949, + "drift/rejected_abs": 0.12656167149543762, + "epoch": 0.34396273737011823, + "grad_norm": 2.390625, + "learning_rate": 4.0147668216311276e-07, + "logits/chosen": -2.1296074390411377, + "logits/rejected": -2.192688465118408, + "logps/chosen": -0.2808237075805664, + "logps/rejected": -0.28292280435562134, + "loss": 0.7036789655685425, + "loss/core": 0.6867524981498718, + "loss/preference_sft": 0.2808237075805664, + "loss/reference_anchor": 0.310445636510849, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.626671314239502, + "rewards/margins": 1.3620039224624634, + "rewards/rejected": 1.2646673917770386, + "step": 360 + }, + { + "drift/chosen_abs": 0.2869020104408264, + "drift/rejected_abs": 0.08114350587129593, + "epoch": 0.34873999761136987, + "grad_norm": 9.625, + "learning_rate": 3.9736129742626843e-07, + "logits/chosen": -2.067612886428833, + "logits/rejected": -2.1757853031158447, + "logps/chosen": -0.2856762707233429, + "logps/rejected": -0.283938467502594, + "loss": 0.7096842527389526, + "loss/core": 0.683948278427124, + "loss/preference_sft": 0.2856762707233429, + "loss/reference_anchor": 0.4861505925655365, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.8673691749572754, + "rewards/margins": 2.0584607124328613, + "rewards/rejected": 0.8089081048965454, + "step": 365 + }, + { + "drift/chosen_abs": 0.1641189157962799, + "drift/rejected_abs": 0.10590697824954987, + "epoch": 0.3535172578526215, + "grad_norm": 1.296875, + "learning_rate": 3.931837850201263e-07, + "logits/chosen": -2.267899990081787, + "logits/rejected": -2.313075304031372, + "logps/chosen": -0.2852608561515808, + "logps/rejected": -0.28572216629981995, + "loss": 0.6960805654525757, + "loss/core": 0.6902772188186646, + "loss/preference_sft": 0.2852608561515808, + "loss/reference_anchor": 0.08754244446754456, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.640094518661499, + "rewards/margins": 0.5825773477554321, + "rewards/rejected": 1.0575170516967773, + "step": 370 + }, + { + "drift/chosen_abs": 0.30126437544822693, + "drift/rejected_abs": 0.2238939106464386, + "epoch": 0.35829451809387314, + "grad_norm": 0.37109375, + "learning_rate": 3.889459061880643e-07, + "logits/chosen": -2.046881914138794, + "logits/rejected": -2.0237812995910645, + "logps/chosen": -0.2588959336280823, + "logps/rejected": -0.25813889503479004, + "loss": 0.7113808393478394, + "loss/core": 0.6897670030593872, + "loss/preference_sft": 0.2588959336280823, + "loss/reference_anchor": 0.4063888192176819, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.0123302936553955, + "rewards/margins": 0.7923107147216797, + "rewards/rejected": 2.220019817352295, + "step": 375 + }, + { + "drift/chosen_abs": 0.2056204378604889, + "drift/rejected_abs": 0.15891826152801514, + "epoch": 0.36307177833512483, + "grad_norm": 0.94140625, + "learning_rate": 3.846494476240056e-07, + "logits/chosen": -2.1194448471069336, + "logits/rejected": -2.1285529136657715, + "logps/chosen": -0.27972573041915894, + "logps/rejected": -0.2927209138870239, + "loss": 0.7116562128067017, + "loss/core": 0.6913924217224121, + "loss/preference_sft": 0.27972573041915894, + "loss/reference_anchor": 0.37730082869529724, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.053956985473633, + "rewards/margins": 0.48597222566604614, + "rewards/rejected": 1.5679845809936523, + "step": 380 + }, + { + "drift/chosen_abs": 0.21692092716693878, + "drift/rejected_abs": 0.1518086940050125, + "epoch": 0.36784903857637646, + "grad_norm": 0.87109375, + "learning_rate": 3.8029622071914626e-07, + "logits/chosen": -2.1106455326080322, + "logits/rejected": -2.1308746337890625, + "logps/chosen": -0.2711559236049652, + "logps/rejected": -0.27569103240966797, + "loss": 0.7008264064788818, + "loss/core": 0.6900737881660461, + "loss/preference_sft": 0.2711559236049652, + "loss/reference_anchor": 0.18793678283691406, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.168698787689209, + "rewards/margins": 0.6546289920806885, + "rewards/rejected": 1.514069676399231, + "step": 385 + }, + { + "drift/chosen_abs": 0.17692413926124573, + "drift/rejected_abs": 0.08787186443805695, + "epoch": 0.3726262988176281, + "grad_norm": 0.2119140625, + "learning_rate": 3.758880607982714e-07, + "logits/chosen": -1.869436502456665, + "logits/rejected": -1.9471441507339478, + "logps/chosen": -0.3066801428794861, + "logps/rejected": -0.3042067885398865, + "loss": 0.6953903436660767, + "loss/core": 0.6888266205787659, + "loss/preference_sft": 0.3066801428794861, + "loss/reference_anchor": 0.10060696303844452, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.7669336795806885, + "rewards/margins": 0.8882148861885071, + "rewards/rejected": 0.8787187337875366, + "step": 390 + }, + { + "drift/chosen_abs": 0.16815339028835297, + "drift/rejected_abs": 0.08043881505727768, + "epoch": 0.37740355905887973, + "grad_norm": 0.212890625, + "learning_rate": 3.714268263459801e-07, + "logits/chosen": -2.2196106910705566, + "logits/rejected": -2.3220858573913574, + "logps/chosen": -0.29513680934906006, + "logps/rejected": -0.28997355699539185, + "loss": 0.6943039894104004, + "loss/core": 0.6888213753700256, + "loss/preference_sft": 0.29513680934906006, + "loss/reference_anchor": 0.08013781160116196, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 1.680734634399414, + "rewards/margins": 0.8773964643478394, + "rewards/rejected": 0.8033380508422852, + "step": 395 + }, + { + "drift/chosen_abs": 0.29227569699287415, + "drift/rejected_abs": 0.08427596837282181, + "epoch": 0.38218081930013137, + "grad_norm": 1.5390625, + "learning_rate": 3.6691439822314666e-07, + "logits/chosen": -2.1000590324401855, + "logits/rejected": -2.135270595550537, + "logps/chosen": -0.2931859791278839, + "logps/rejected": -0.2932865023612976, + "loss": 0.7107817530632019, + "loss/core": 0.6836472749710083, + "loss/preference_sft": 0.2931859791278839, + "loss/reference_anchor": 0.5133715867996216, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.9214870929718018, + "rewards/margins": 2.0816428661346436, + "rewards/rejected": 0.8398441076278687, + "step": 400 + }, + { + "drift/chosen_abs": 0.23128154873847961, + "drift/rejected_abs": 0.12374065816402435, + "epoch": 0.386958079541383, + "grad_norm": 0.310546875, + "learning_rate": 3.623526788739477e-07, + "logits/chosen": -2.131155014038086, + "logits/rejected": -2.1849007606506348, + "logps/chosen": -0.3152233064174652, + "logps/rejected": -0.3035925030708313, + "loss": 0.7062971591949463, + "loss/core": 0.6880842447280884, + "loss/preference_sft": 0.3152233064174652, + "loss/reference_anchor": 0.33273500204086304, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.312814235687256, + "rewards/margins": 1.0756020545959473, + "rewards/rejected": 1.2372119426727295, + "step": 405 + }, + { + "drift/chosen_abs": 0.20869719982147217, + "drift/rejected_abs": 0.11958236992359161, + "epoch": 0.39173533978263464, + "grad_norm": 0.71875, + "learning_rate": 3.5774359152378985e-07, + "logits/chosen": -2.248558521270752, + "logits/rejected": -2.2788643836975098, + "logps/chosen": -0.2772386372089386, + "logps/rejected": -0.28785425424575806, + "loss": 0.6970154047012329, + "loss/core": 0.6887233257293701, + "loss/preference_sft": 0.2772386372089386, + "loss/reference_anchor": 0.138116717338562, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.0857138633728027, + "rewards/margins": 0.8951884508132935, + "rewards/rejected": 1.1905254125595093, + "step": 410 + }, + { + "drift/chosen_abs": 0.09970752894878387, + "drift/rejected_abs": 0.03773488104343414, + "epoch": 0.3965126000238863, + "grad_norm": 0.2275390625, + "learning_rate": 3.530890793684759e-07, + "logits/chosen": -2.1164305210113525, + "logits/rejected": -2.1951279640197754, + "logps/chosen": -0.33025726675987244, + "logps/rejected": -0.32088834047317505, + "loss": 0.693244218826294, + "loss/core": 0.6900933384895325, + "loss/preference_sft": 0.33025726675987244, + "loss/reference_anchor": 0.02999264933168888, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 0.9955142736434937, + "rewards/margins": 0.6189162731170654, + "rewards/rejected": 0.376598060131073, + "step": 415 + }, + { + "drift/chosen_abs": 0.2045033723115921, + "drift/rejected_abs": 0.07196368277072906, + "epoch": 0.40128986026513797, + "grad_norm": 1.9609375, + "learning_rate": 3.4839110475495146e-07, + "logits/chosen": -2.349240779876709, + "logits/rejected": -2.316765785217285, + "logps/chosen": -0.28340643644332886, + "logps/rejected": -0.2993830144405365, + "loss": 0.6966205835342407, + "loss/core": 0.6868051290512085, + "loss/preference_sft": 0.28340643644332886, + "loss/reference_anchor": 0.16796544194221497, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.0448544025421143, + "rewards/margins": 1.3295873403549194, + "rewards/rejected": 0.7152668237686157, + "step": 420 + }, + { + "drift/chosen_abs": 0.22461578249931335, + "drift/rejected_abs": 0.16200877726078033, + "epoch": 0.4060671205063896, + "grad_norm": 6.9375, + "learning_rate": 3.43651648353978e-07, + "logits/chosen": -2.012849807739258, + "logits/rejected": -2.072317123413086, + "logps/chosen": -0.29064491391181946, + "logps/rejected": -0.2678697109222412, + "loss": 0.7116137742996216, + "loss/core": 0.6908143758773804, + "loss/preference_sft": 0.29064491391181946, + "loss/reference_anchor": 0.3869226574897766, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.2447874546051025, + "rewards/margins": 0.6263939142227173, + "rewards/rejected": 1.6183935403823853, + "step": 425 + }, + { + "drift/chosen_abs": 0.43855953216552734, + "drift/rejected_abs": 0.14190329611301422, + "epoch": 0.41084438074764124, + "grad_norm": 16.25, + "learning_rate": 3.388727083250795e-07, + "logits/chosen": -2.0672688484191895, + "logits/rejected": -2.0907437801361084, + "logps/chosen": -0.2815065085887909, + "logps/rejected": -0.26665714383125305, + "loss": 0.7262654304504395, + "loss/core": 0.6797505617141724, + "loss/preference_sft": 0.2815065085887909, + "loss/reference_anchor": 0.9021477699279785, + "rewards/accuracies": 0.875, + "rewards/chosen": 4.383257865905762, + "rewards/margins": 2.965740442276001, + "rewards/rejected": 1.4175175428390503, + "step": 430 + }, + { + "drift/chosen_abs": 0.162430077791214, + "drift/rejected_abs": 0.10076061636209488, + "epoch": 0.41562164098889287, + "grad_norm": 0.73046875, + "learning_rate": 3.3405629947411683e-07, + "logits/chosen": -2.350315570831299, + "logits/rejected": -2.3338377475738525, + "logps/chosen": -0.27856045961380005, + "logps/rejected": -0.293110191822052, + "loss": 0.6974702477455139, + "loss/core": 0.69013512134552, + "loss/preference_sft": 0.27856045961380005, + "loss/reference_anchor": 0.11884655803442001, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.6225159168243408, + "rewards/margins": 0.6160438656806946, + "rewards/rejected": 1.006471872329712, + "step": 435 + }, + { + "drift/chosen_abs": 0.299646258354187, + "drift/rejected_abs": 0.13414156436920166, + "epoch": 0.4203989012301445, + "grad_norm": 13.6875, + "learning_rate": 3.2920445240384325e-07, + "logits/chosen": -2.05702543258667, + "logits/rejected": -2.1284549236297607, + "logps/chosen": -0.3045402765274048, + "logps/rejected": -0.30474570393562317, + "loss": 0.711657702922821, + "loss/core": 0.6856769323348999, + "loss/preference_sft": 0.3045402765274048, + "loss/reference_anchor": 0.48916101455688477, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.993908643722534, + "rewards/margins": 1.6548278331756592, + "rewards/rejected": 1.339080810546875, + "step": 440 + }, + { + "drift/chosen_abs": 0.2965596318244934, + "drift/rejected_abs": 0.1601937860250473, + "epoch": 0.42517616147139614, + "grad_norm": 1.0078125, + "learning_rate": 3.2431921265780063e-07, + "logits/chosen": -1.9329261779785156, + "logits/rejected": -1.909558653831482, + "logps/chosen": -0.2952488958835602, + "logps/rejected": -0.298814594745636, + "loss": 0.7007791996002197, + "loss/core": 0.6862999796867371, + "loss/preference_sft": 0.2952488958835602, + "loss/reference_anchor": 0.2600591480731964, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.9641549587249756, + "rewards/margins": 1.417994737625122, + "rewards/rejected": 1.5461599826812744, + "step": 445 + }, + { + "drift/chosen_abs": 0.34632277488708496, + "drift/rejected_abs": 0.156765878200531, + "epoch": 0.4299534217126478, + "grad_norm": 13.0625, + "learning_rate": 3.1940263985791616e-07, + "logits/chosen": -1.973149061203003, + "logits/rejected": -1.990882158279419, + "logps/chosen": -0.297830194234848, + "logps/rejected": -0.2937723696231842, + "loss": 0.7078851461410522, + "loss/core": 0.6842607259750366, + "loss/preference_sft": 0.297830194234848, + "loss/reference_anchor": 0.44270557165145874, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.462675094604492, + "rewards/margins": 1.8995628356933594, + "rewards/rejected": 1.5631126165390015, + "step": 450 + }, + { + "drift/chosen_abs": 0.341677725315094, + "drift/rejected_abs": 0.14796192944049835, + "epoch": 0.4347306819538994, + "grad_norm": 4.9375, + "learning_rate": 3.144568068361634e-07, + "logits/chosen": -1.9354276657104492, + "logits/rejected": -2.1260530948638916, + "logps/chosen": -0.3276836574077606, + "logps/rejected": -0.3140363097190857, + "loss": 0.7205722332000732, + "loss/core": 0.6841714382171631, + "loss/preference_sft": 0.3276836574077606, + "loss/reference_anchor": 0.6952458620071411, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.415968418121338, + "rewards/margins": 1.9381797313690186, + "rewards/rejected": 1.4777886867523193, + "step": 455 + }, + { + "drift/chosen_abs": 0.3663923144340515, + "drift/rejected_abs": 0.18600957095623016, + "epoch": 0.4395079421951511, + "grad_norm": 1.1015625, + "learning_rate": 3.0948379876065465e-07, + "logits/chosen": -2.0609209537506104, + "logits/rejected": -2.048144578933716, + "logps/chosen": -0.2960844337940216, + "logps/rejected": -0.3075362741947174, + "loss": 0.7149934768676758, + "loss/core": 0.6845480799674988, + "loss/preference_sft": 0.2960844337940216, + "loss/reference_anchor": 0.579299807548523, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.6639232635498047, + "rewards/margins": 1.8042666912078857, + "rewards/rejected": 1.859656572341919, + "step": 460 + }, + { + "drift/chosen_abs": 0.23091194033622742, + "drift/rejected_abs": 0.09295056760311127, + "epoch": 0.44428520243640274, + "grad_norm": 0.341796875, + "learning_rate": 3.0448571225653193e-07, + "logits/chosen": -1.8885440826416016, + "logits/rejected": -1.988734245300293, + "logps/chosen": -0.29641199111938477, + "logps/rejected": -0.2917849123477936, + "loss": 0.6937438249588013, + "loss/core": 0.6863740682601929, + "loss/preference_sft": 0.29641199111938477, + "loss/reference_anchor": 0.11775638163089752, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.3084988594055176, + "rewards/margins": 1.3800839185714722, + "rewards/rejected": 0.9284147024154663, + "step": 465 + }, + { + "drift/chosen_abs": 0.2984088957309723, + "drift/rejected_abs": 0.12458224594593048, + "epoch": 0.4490624626776544, + "grad_norm": 22.5, + "learning_rate": 2.9946465452202747e-07, + "logits/chosen": -2.1285500526428223, + "logits/rejected": -2.196565628051758, + "logps/chosen": -0.28165316581726074, + "logps/rejected": -0.2599329352378845, + "loss": 0.7142145037651062, + "loss/core": 0.6846899390220642, + "loss/preference_sft": 0.28165316581726074, + "loss/reference_anchor": 0.5623257756233215, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.984055757522583, + "rewards/margins": 1.8138116598129272, + "rewards/rejected": 1.170243501663208, + "step": 470 + }, + { + "drift/chosen_abs": 0.182987779378891, + "drift/rejected_abs": 0.14684465527534485, + "epoch": 0.453839722918906, + "grad_norm": 3.4375, + "learning_rate": 2.944227424400672e-07, + "logits/chosen": -2.0931344032287598, + "logits/rejected": -2.0336596965789795, + "logps/chosen": -0.29563307762145996, + "logps/rejected": -0.3208538889884949, + "loss": 0.7051785588264465, + "loss/core": 0.6917919516563416, + "loss/preference_sft": 0.29563307762145996, + "loss/reference_anchor": 0.23816709220409393, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8292009830474854, + "rewards/margins": 0.3628115653991699, + "rewards/rejected": 1.4663891792297363, + "step": 475 + }, + { + "drift/chosen_abs": 0.23586270213127136, + "drift/rejected_abs": 0.10744066536426544, + "epoch": 0.45861698316015764, + "grad_norm": 0.466796875, + "learning_rate": 2.8936210168579037e-07, + "logits/chosen": -2.2626140117645264, + "logits/rejected": -2.318941593170166, + "logps/chosen": -0.275493860244751, + "logps/rejected": -0.2723752558231354, + "loss": 0.7019254565238953, + "loss/core": 0.6869186758995056, + "loss/preference_sft": 0.275493860244751, + "loss/reference_anchor": 0.27258676290512085, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.3575544357299805, + "rewards/margins": 1.3319319486618042, + "rewards/rejected": 1.0256226062774658, + "step": 480 + }, + { + "drift/chosen_abs": 0.17448770999908447, + "drift/rejected_abs": 0.08884555101394653, + "epoch": 0.4633942434014093, + "grad_norm": 0.478515625, + "learning_rate": 2.842848658303637e-07, + "logits/chosen": -2.2037127017974854, + "logits/rejected": -2.2105116844177246, + "logps/chosen": -0.29139575362205505, + "logps/rejected": -0.29565316438674927, + "loss": 0.694310188293457, + "loss/core": 0.6889179944992065, + "loss/preference_sft": 0.29139575362205505, + "loss/reference_anchor": 0.07870315760374069, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.7447458505630493, + "rewards/margins": 0.8566008806228638, + "rewards/rejected": 0.8881451487541199, + "step": 485 + }, + { + "drift/chosen_abs": 0.20757827162742615, + "drift/rejected_abs": 0.10591663420200348, + "epoch": 0.4681715036426609, + "grad_norm": 0.65625, + "learning_rate": 2.79193175441464e-07, + "logits/chosen": -2.1531708240509033, + "logits/rejected": -2.1945109367370605, + "logps/chosen": -0.3073252737522125, + "logps/rejected": -0.29394882917404175, + "loss": 0.6971133947372437, + "loss/core": 0.688152015209198, + "loss/preference_sft": 0.3073252737522125, + "loss/reference_anchor": 0.1484949290752411, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.0757830142974854, + "rewards/margins": 1.0181604623794556, + "rewards/rejected": 1.0576225519180298, + "step": 490 + }, + { + "drift/chosen_abs": 0.25867894291877747, + "drift/rejected_abs": 0.08946122229099274, + "epoch": 0.47294876388391255, + "grad_norm": 0.390625, + "learning_rate": 2.74089177180815e-07, + "logits/chosen": -2.2221720218658447, + "logits/rejected": -2.3168718814849854, + "logps/chosen": -0.3004676401615143, + "logps/rejected": -0.29068127274513245, + "loss": 0.7042246460914612, + "loss/core": 0.6852529048919678, + "loss/preference_sft": 0.3004676401615143, + "loss/reference_anchor": 0.34938645362854004, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.584899663925171, + "rewards/margins": 1.6914997100830078, + "rewards/rejected": 0.8933998346328735, + "step": 495 + }, + { + "drift/chosen_abs": 0.4131302833557129, + "drift/rejected_abs": 0.20293641090393066, + "epoch": 0.47772602412516424, + "grad_norm": 4.125, + "learning_rate": 2.6897502289915026e-07, + "logits/chosen": -2.021841526031494, + "logits/rejected": -1.9711917638778687, + "logps/chosen": -0.28432515263557434, + "logps/rejected": -0.3025287389755249, + "loss": 0.741477370262146, + "loss/core": 0.6841000318527222, + "loss/preference_sft": 0.28432515263557434, + "loss/reference_anchor": 1.1191157102584839, + "rewards/accuracies": 0.875, + "rewards/chosen": 4.131275177001953, + "rewards/margins": 2.104186534881592, + "rewards/rejected": 2.0270884037017822, + "step": 500 + }, + { + "drift/chosen_abs": 0.36249279975891113, + "drift/rejected_abs": 0.1847756803035736, + "epoch": 0.4825032843664159, + "grad_norm": 6.1875, + "learning_rate": 2.638528687289925e-07, + "logits/chosen": -2.1491918563842773, + "logits/rejected": -2.1820521354675293, + "logps/chosen": -0.2843039035797119, + "logps/rejected": -0.28839248418807983, + "loss": 0.7243824005126953, + "loss/core": 0.6854826807975769, + "loss/preference_sft": 0.2843039035797119, + "loss/reference_anchor": 0.7495636940002441, + "rewards/accuracies": 0.8125, + "rewards/chosen": 3.623505115509033, + "rewards/margins": 1.8764804601669312, + "rewards/rejected": 1.7470247745513916, + "step": 505 + }, + { + "drift/chosen_abs": 0.20058460533618927, + "drift/rejected_abs": 0.08017744868993759, + "epoch": 0.4872805446076675, + "grad_norm": 0.1259765625, + "learning_rate": 2.5872487417562527e-07, + "logits/chosen": -2.1467156410217285, + "logits/rejected": -2.2804579734802246, + "logps/chosen": -0.3167296051979065, + "logps/rejected": -0.2912823259830475, + "loss": 0.6994425654411316, + "loss/core": 0.6874934434890747, + "loss/preference_sft": 0.3167296051979065, + "loss/reference_anchor": 0.2073097974061966, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.002615451812744, + "rewards/margins": 1.2019274234771729, + "rewards/rejected": 0.8006879687309265, + "step": 510 + }, + { + "drift/chosen_abs": 0.2543594241142273, + "drift/rejected_abs": 0.16676507890224457, + "epoch": 0.49205780484891914, + "grad_norm": 0.38671875, + "learning_rate": 2.535932012066433e-07, + "logits/chosen": -2.0313689708709717, + "logits/rejected": -2.034614324569702, + "logps/chosen": -0.2830977141857147, + "logps/rejected": -0.3066386282444, + "loss": 0.7011044025421143, + "loss/core": 0.6888419985771179, + "loss/preference_sft": 0.2830977141857147, + "loss/reference_anchor": 0.216938778758049, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.5404956340789795, + "rewards/margins": 0.8886449933052063, + "rewards/rejected": 1.6518504619598389, + "step": 515 + }, + { + "drift/chosen_abs": 0.30793458223342896, + "drift/rejected_abs": 0.09768765419721603, + "epoch": 0.4968350650901708, + "grad_norm": 2.0, + "learning_rate": 2.4846001334046535e-07, + "logits/chosen": -2.058237314224243, + "logits/rejected": -2.226836681365967, + "logps/chosen": -0.322182834148407, + "logps/rejected": -0.33398357033729553, + "loss": 0.7052879929542542, + "loss/core": 0.6832786798477173, + "loss/preference_sft": 0.322182834148407, + "loss/reference_anchor": 0.4079677164554596, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.0726497173309326, + "rewards/margins": 2.1081771850585938, + "rewards/rejected": 0.9644724726676941, + "step": 520 + }, + { + "drift/chosen_abs": 0.42069754004478455, + "drift/rejected_abs": 0.14602303504943848, + "epoch": 0.5016123253314224, + "grad_norm": 30.625, + "learning_rate": 2.433274747341919e-07, + "logits/chosen": -2.194186210632324, + "logits/rejected": -2.162163496017456, + "logps/chosen": -0.3106693625450134, + "logps/rejected": -0.30101826786994934, + "loss": 0.73024982213974, + "loss/core": 0.6811276078224182, + "loss/preference_sft": 0.3106693625450134, + "loss/reference_anchor": 0.951376736164093, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.204687118530273, + "rewards/margins": 2.749366521835327, + "rewards/rejected": 1.455320119857788, + "step": 525 + }, + { + "drift/chosen_abs": 0.30520981550216675, + "drift/rejected_abs": 0.1496325582265854, + "epoch": 0.5063895855726741, + "grad_norm": 0.140625, + "learning_rate": 2.3819774927119522e-07, + "logits/chosen": -2.0378127098083496, + "logits/rejected": -2.001634359359741, + "logps/chosen": -0.28599685430526733, + "logps/rejected": -0.28872916102409363, + "loss": 0.7070541977882385, + "loss/core": 0.6858066320419312, + "loss/preference_sft": 0.28599685430526733, + "loss/reference_anchor": 0.3963504433631897, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.050769090652466, + "rewards/margins": 1.5627717971801758, + "rewards/rejected": 1.4879971742630005, + "step": 530 + }, + { + "drift/chosen_abs": 0.30915752053260803, + "drift/rejected_abs": 0.10786645114421844, + "epoch": 0.5111668458139257, + "grad_norm": 13.6875, + "learning_rate": 2.3307299964882312e-07, + "logits/chosen": -2.186264991760254, + "logits/rejected": -2.108275890350342, + "logps/chosen": -0.2858208417892456, + "logps/rejected": -0.27927789092063904, + "loss": 0.7080153226852417, + "loss/core": 0.6839091181755066, + "loss/preference_sft": 0.2858208417892456, + "loss/reference_anchor": 0.4535427987575531, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.0912933349609375, + "rewards/margins": 2.0143520832061768, + "rewards/rejected": 1.0769412517547607, + "step": 535 + }, + { + "drift/chosen_abs": 0.15930701792240143, + "drift/rejected_abs": 0.14954623579978943, + "epoch": 0.5159441060551774, + "grad_norm": 0.193359375, + "learning_rate": 2.279553864666046e-07, + "logits/chosen": -2.106029748916626, + "logits/rejected": -2.0835556983947754, + "logps/chosen": -0.27827921509742737, + "logps/rejected": -0.2890811562538147, + "loss": 0.7021555304527283, + "loss/core": 0.6928835511207581, + "loss/preference_sft": 0.27827921509742737, + "loss/reference_anchor": 0.15761205554008484, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.5930081605911255, + "rewards/margins": 0.09992774575948715, + "rewards/rejected": 1.4930803775787354, + "step": 540 + }, + { + "drift/chosen_abs": 0.26317763328552246, + "drift/rejected_abs": 0.08720904588699341, + "epoch": 0.520721366296429, + "grad_norm": 23.375, + "learning_rate": 2.22847067315338e-07, + "logits/chosen": -2.331496477127075, + "logits/rejected": -2.172362804412842, + "logps/chosen": -0.2841755151748657, + "logps/rejected": -0.2938932776451111, + "loss": 0.7100147008895874, + "loss/core": 0.6851332187652588, + "loss/preference_sft": 0.2841755151748657, + "loss/reference_anchor": 0.46921229362487793, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.6317765712738037, + "rewards/margins": 1.762070655822754, + "rewards/rejected": 0.8697060346603394, + "step": 545 + }, + { + "drift/chosen_abs": 0.2769598066806793, + "drift/rejected_abs": 0.16232618689537048, + "epoch": 0.5254986265376806, + "grad_norm": 0.53515625, + "learning_rate": 2.1775019586744921e-07, + "logits/chosen": -1.8962993621826172, + "logits/rejected": -1.9161226749420166, + "logps/chosen": -0.27603915333747864, + "logps/rejected": -0.2979462742805481, + "loss": 0.7087868452072144, + "loss/core": 0.6875404119491577, + "loss/preference_sft": 0.27603915333747864, + "loss/reference_anchor": 0.3973239064216614, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.7692787647247314, + "rewards/margins": 1.1475410461425781, + "rewards/rejected": 1.6217377185821533, + "step": 550 + }, + { + "drift/chosen_abs": 0.32226184010505676, + "drift/rejected_abs": 0.09745451807975769, + "epoch": 0.5302758867789323, + "grad_norm": 0.2578125, + "learning_rate": 2.126669209690008e-07, + "logits/chosen": -2.0920777320861816, + "logits/rejected": -2.14959979057312, + "logps/chosen": -0.31904277205467224, + "logps/rejected": -0.2770870327949524, + "loss": 0.7026501893997192, + "loss/core": 0.6825853586196899, + "loss/preference_sft": 0.31904277205467224, + "loss/reference_anchor": 0.3693930506706238, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.221327543258667, + "rewards/margins": 2.247218608856201, + "rewards/rejected": 0.9741086959838867, + "step": 555 + }, + { + "drift/chosen_abs": 0.2818199098110199, + "drift/rejected_abs": 0.1541328728199005, + "epoch": 0.5350531470201839, + "grad_norm": 0.609375, + "learning_rate": 2.075993857337368e-07, + "logits/chosen": -2.180386781692505, + "logits/rejected": -2.1680946350097656, + "logps/chosen": -0.2756226062774658, + "logps/rejected": -0.27983418107032776, + "loss": 0.7056413888931274, + "loss/core": 0.6869414448738098, + "loss/preference_sft": 0.2756226062774658, + "loss/reference_anchor": 0.3464350402355194, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.816300868988037, + "rewards/margins": 1.276263952255249, + "rewards/rejected": 1.540036916732788, + "step": 560 + }, + { + "drift/chosen_abs": 0.2778550386428833, + "drift/rejected_abs": 0.16609425842761993, + "epoch": 0.5398304072614356, + "grad_norm": 0.3203125, + "learning_rate": 2.0254972663954353e-07, + "logits/chosen": -2.0759246349334717, + "logits/rejected": -2.1022229194641113, + "logps/chosen": -0.2878085672855377, + "logps/rejected": -0.3052247166633606, + "loss": 0.7213435173034668, + "loss/core": 0.6878103017807007, + "loss/preference_sft": 0.2878085672855377, + "loss/reference_anchor": 0.6418825387954712, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.7778897285461426, + "rewards/margins": 1.1175713539123535, + "rewards/rejected": 1.6603180170059204, + "step": 565 + }, + { + "drift/chosen_abs": 0.3346030116081238, + "drift/rejected_abs": 0.16794054210186005, + "epoch": 0.5446076675026872, + "grad_norm": 2.421875, + "learning_rate": 1.9752007262770854e-07, + "logits/chosen": -2.0751147270202637, + "logits/rejected": -2.1243863105773926, + "logps/chosen": -0.28862354159355164, + "logps/rejected": -0.28968754410743713, + "loss": 0.7091944813728333, + "loss/core": 0.6851403713226318, + "loss/preference_sft": 0.28862354159355164, + "loss/reference_anchor": 0.45222073793411255, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.3444488048553467, + "rewards/margins": 1.6680787801742554, + "rewards/rejected": 1.6763699054718018, + "step": 570 + }, + { + "drift/chosen_abs": 0.18410475552082062, + "drift/rejected_abs": 0.0659659132361412, + "epoch": 0.5493849277439389, + "grad_norm": 0.3046875, + "learning_rate": 1.9251254420535664e-07, + "logits/chosen": -2.152061939239502, + "logits/rejected": -2.1839194297790527, + "logps/chosen": -0.3009091913700104, + "logps/rejected": -0.32279980182647705, + "loss": 0.6969389319419861, + "loss/core": 0.6871550679206848, + "loss/preference_sft": 0.3009091913700104, + "loss/reference_anchor": 0.16558721661567688, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.84026300907135, + "rewards/margins": 1.2756303548812866, + "rewards/rejected": 0.5646325349807739, + "step": 575 + }, + { + "drift/chosen_abs": 0.1417587250471115, + "drift/rejected_abs": 0.09436655044555664, + "epoch": 0.5541621879851905, + "grad_norm": 0.1552734375, + "learning_rate": 1.8752925255144224e-07, + "logits/chosen": -2.428544044494629, + "logits/rejected": -2.4844648838043213, + "logps/chosen": -0.26681989431381226, + "logps/rejected": -0.28266364336013794, + "loss": 0.696236252784729, + "loss/core": 0.690847635269165, + "loss/preference_sft": 0.26681989431381226, + "loss/reference_anchor": 0.08109016716480255, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.4170500040054321, + "rewards/margins": 0.47523316740989685, + "rewards/rejected": 0.9418169260025024, + "step": 580 + }, + { + "drift/chosen_abs": 0.22228363156318665, + "drift/rejected_abs": 0.13576579093933105, + "epoch": 0.5589394482264421, + "grad_norm": 0.5390625, + "learning_rate": 1.8257229862667433e-07, + "logits/chosen": -2.1347758769989014, + "logits/rejected": -2.0975234508514404, + "logps/chosen": -0.2703164219856262, + "logps/rejected": -0.28436440229415894, + "loss": 0.7252476811408997, + "loss/core": 0.6889234781265259, + "loss/preference_sft": 0.2703164219856262, + "loss/reference_anchor": 0.6994535326957703, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.221714496612549, + "rewards/margins": 0.8691756129264832, + "rewards/rejected": 1.3525389432907104, + "step": 585 + }, + { + "drift/chosen_abs": 0.3615594506263733, + "drift/rejected_abs": 0.10348814725875854, + "epoch": 0.5637167084676937, + "grad_norm": 0.2373046875, + "learning_rate": 1.7764377228774873e-07, + "logits/chosen": -2.0233283042907715, + "logits/rejected": -2.119215250015259, + "logps/chosen": -0.28815799951553345, + "logps/rejected": -0.2871782183647156, + "loss": 0.7047743797302246, + "loss/core": 0.6811040639877319, + "loss/preference_sft": 0.28815799951553345, + "loss/reference_anchor": 0.44459062814712524, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.6147727966308594, + "rewards/margins": 2.584536075592041, + "rewards/rejected": 1.0302363634109497, + "step": 590 + }, + { + "drift/chosen_abs": 0.17077796161174774, + "drift/rejected_abs": 0.08986127376556396, + "epoch": 0.5684939687089454, + "grad_norm": 0.43359375, + "learning_rate": 1.7274575140626315e-07, + "logits/chosen": -2.044630765914917, + "logits/rejected": -2.03894305229187, + "logps/chosen": -0.27572837471961975, + "logps/rejected": -0.2722456455230713, + "loss": 0.6933053135871887, + "loss/core": 0.6890828013420105, + "loss/preference_sft": 0.27572837471961975, + "loss/reference_anchor": 0.056876588612794876, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.705609917640686, + "rewards/margins": 0.8199650049209595, + "rewards/rejected": 0.8856450319290161, + "step": 595 + }, + { + "drift/chosen_abs": 0.18729227781295776, + "drift/rejected_abs": 0.10635222494602203, + "epoch": 0.5732712289501971, + "grad_norm": 0.703125, + "learning_rate": 1.6788030099268407e-07, + "logits/chosen": -2.2757256031036377, + "logits/rejected": -2.265326738357544, + "logps/chosen": -0.26656943559646606, + "logps/rejected": -0.28453245759010315, + "loss": 0.6980012655258179, + "loss/core": 0.6892417073249817, + "loss/preference_sft": 0.26656943559646606, + "loss/reference_anchor": 0.14853349328041077, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.8706554174423218, + "rewards/margins": 0.809914767742157, + "rewards/rejected": 1.0607407093048096, + "step": 600 + }, + { + "drift/chosen_abs": 0.41853857040405273, + "drift/rejected_abs": 0.12051737308502197, + "epoch": 0.5780484891914487, + "grad_norm": 1.2265625, + "learning_rate": 1.630494723257363e-07, + "logits/chosen": -2.1154239177703857, + "logits/rejected": -2.1893069744110107, + "logps/chosen": -0.28015339374542236, + "logps/rejected": -0.27586129307746887, + "loss": 0.7251626253128052, + "loss/core": 0.6798030138015747, + "loss/preference_sft": 0.28015339374542236, + "loss/reference_anchor": 0.8791759610176086, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.183618068695068, + "rewards/margins": 2.979365110397339, + "rewards/rejected": 1.2042533159255981, + "step": 605 + }, + { + "drift/chosen_abs": 0.4256836473941803, + "drift/rejected_abs": 0.19338993728160858, + "epoch": 0.5828257494327004, + "grad_norm": 1.65625, + "learning_rate": 1.5825530208758198e-07, + "logits/chosen": -2.1129722595214844, + "logits/rejected": -2.1302802562713623, + "logps/chosen": -0.2779896557331085, + "logps/rejected": -0.27780717611312866, + "loss": 0.7297109961509705, + "loss/core": 0.6822241544723511, + "loss/preference_sft": 0.2779896557331085, + "loss/reference_anchor": 0.9219374656677246, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 4.254659175872803, + "rewards/margins": 2.343705177307129, + "rewards/rejected": 1.9109541177749634, + "step": 610 + }, + { + "drift/chosen_abs": 0.2733246088027954, + "drift/rejected_abs": 0.14853966236114502, + "epoch": 0.587603009673952, + "grad_norm": 0.5, + "learning_rate": 1.534998115051533e-07, + "logits/chosen": -1.9654070138931274, + "logits/rejected": -2.0324883460998535, + "logps/chosen": -0.31496065855026245, + "logps/rejected": -0.30436021089553833, + "loss": 0.7030231952667236, + "loss/core": 0.6871589422225952, + "loss/preference_sft": 0.31496065855026245, + "loss/reference_anchor": 0.2857890725135803, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.7304234504699707, + "rewards/margins": 1.2481533288955688, + "rewards/rejected": 1.4822700023651123, + "step": 615 + }, + { + "drift/chosen_abs": 0.1698610484600067, + "drift/rejected_abs": 0.14360573887825012, + "epoch": 0.5923802699152036, + "grad_norm": 0.373046875, + "learning_rate": 1.4878500549800112e-07, + "logits/chosen": -2.1398520469665527, + "logits/rejected": -2.166027069091797, + "logps/chosen": -0.28533971309661865, + "logps/rejected": -0.29425957798957825, + "loss": 0.7028168439865112, + "loss/core": 0.6921274662017822, + "loss/preference_sft": 0.28533971309661865, + "loss/reference_anchor": 0.18525299429893494, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.6973737478256226, + "rewards/margins": 0.26329460740089417, + "rewards/rejected": 1.4340791702270508, + "step": 620 + }, + { + "drift/chosen_abs": 0.3897588551044464, + "drift/rejected_abs": 0.17362374067306519, + "epoch": 0.5971575301564552, + "grad_norm": 29.375, + "learning_rate": 1.4411287183301902e-07, + "logits/chosen": -1.9303457736968994, + "logits/rejected": -1.958958625793457, + "logps/chosen": -0.30389657616615295, + "logps/rejected": -0.347369909286499, + "loss": 0.7363072633743286, + "loss/core": 0.6822855472564697, + "loss/preference_sft": 0.30389657616615295, + "loss/reference_anchor": 1.0500433444976807, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.895702362060547, + "rewards/margins": 2.4964680671691895, + "rewards/rejected": 1.3992340564727783, + "step": 625 + }, + { + "drift/chosen_abs": 0.29475387930870056, + "drift/rejected_abs": 0.12502244114875793, + "epoch": 0.6019347903977069, + "grad_norm": 5.09375, + "learning_rate": 1.394853802863985e-07, + "logits/chosen": -1.929779052734375, + "logits/rejected": -2.04491925239563, + "logps/chosen": -0.27201491594314575, + "logps/rejected": -0.2829875349998474, + "loss": 0.7087033987045288, + "loss/core": 0.685075581073761, + "loss/preference_sft": 0.27201491594314575, + "loss/reference_anchor": 0.44535598158836365, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.9475390911102295, + "rewards/margins": 1.699634313583374, + "rewards/rejected": 1.2479045391082764, + "step": 630 + }, + { + "drift/chosen_abs": 0.2301911860704422, + "drift/rejected_abs": 0.10143622010946274, + "epoch": 0.6067120506389586, + "grad_norm": 4.9375, + "learning_rate": 1.3490448181317024e-07, + "logits/chosen": -1.981834053993225, + "logits/rejected": -2.0678226947784424, + "logps/chosen": -0.3054197430610657, + "logps/rejected": -0.30018460750579834, + "loss": 0.6994880437850952, + "loss/core": 0.687004566192627, + "loss/preference_sft": 0.3054197430610657, + "loss/reference_anchor": 0.21912848949432373, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.3007302284240723, + "rewards/margins": 1.2872453927993774, + "rewards/rejected": 1.0134851932525635, + "step": 635 + }, + { + "drift/chosen_abs": 0.1967557668685913, + "drift/rejected_abs": 0.07957569509744644, + "epoch": 0.6114893108802102, + "grad_norm": 0.25390625, + "learning_rate": 1.303721077246784e-07, + "logits/chosen": -2.207252264022827, + "logits/rejected": -2.2832891941070557, + "logps/chosen": -0.30064091086387634, + "logps/rejected": -0.29895979166030884, + "loss": 0.6989712119102478, + "loss/core": 0.6875656843185425, + "loss/preference_sft": 0.30064091086387634, + "loss/reference_anchor": 0.19804587960243225, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.9660539627075195, + "rewards/margins": 1.1706269979476929, + "rewards/rejected": 0.7954270839691162, + "step": 640 + }, + { + "drift/chosen_abs": 0.16765008866786957, + "drift/rejected_abs": 0.11835210025310516, + "epoch": 0.6162665711214619, + "grad_norm": 0.86328125, + "learning_rate": 1.2589016887433846e-07, + "logits/chosen": -2.1545424461364746, + "logits/rejected": -2.1316208839416504, + "logps/chosen": -0.29050302505493164, + "logps/rejected": -0.3003830313682556, + "loss": 0.6977421045303345, + "loss/core": 0.6908085942268372, + "loss/preference_sft": 0.29050302505493164, + "loss/reference_anchor": 0.10962003469467163, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.6752145290374756, + "rewards/margins": 0.49554985761642456, + "rewards/rejected": 1.1796646118164062, + "step": 645 + }, + { + "drift/chosen_abs": 0.13382819294929504, + "drift/rejected_abs": 0.0856524258852005, + "epoch": 0.6210438313627135, + "grad_norm": 0.3046875, + "learning_rate": 1.2146055485201943e-07, + "logits/chosen": -2.22021222114563, + "logits/rejected": -2.171934127807617, + "logps/chosen": -0.28915685415267944, + "logps/rejected": -0.29866427183151245, + "loss": 0.6960299015045166, + "loss/core": 0.6908055543899536, + "loss/preference_sft": 0.28915685415267944, + "loss/reference_anchor": 0.075570248067379, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.3368127346038818, + "rewards/margins": 0.4924766421318054, + "rewards/rejected": 0.844336211681366, + "step": 650 + }, + { + "drift/chosen_abs": 0.20044943690299988, + "drift/rejected_abs": 0.09594675153493881, + "epoch": 0.6258210916039652, + "grad_norm": 2.671875, + "learning_rate": 1.1708513318739094e-07, + "logits/chosen": -2.0582327842712402, + "logits/rejected": -2.1220715045928955, + "logps/chosen": -0.282463014125824, + "logps/rejected": -0.29273414611816406, + "loss": 0.6957246661186218, + "loss/core": 0.6880175471305847, + "loss/preference_sft": 0.282463014125824, + "loss/reference_anchor": 0.125896617770195, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.002502918243408, + "rewards/margins": 1.0442477464675903, + "rewards/rejected": 0.9582554697990417, + "step": 655 + }, + { + "drift/chosen_abs": 0.27859437465667725, + "drift/rejected_abs": 0.1346025913953781, + "epoch": 0.6305983518452167, + "grad_norm": 0.2265625, + "learning_rate": 1.1276574856257095e-07, + "logits/chosen": -2.0280890464782715, + "logits/rejected": -2.0235238075256348, + "logps/chosen": -0.29317161440849304, + "logps/rejected": -0.3068060576915741, + "loss": 0.7208054065704346, + "loss/core": 0.6866048574447632, + "loss/preference_sft": 0.29317161440849304, + "loss/reference_anchor": 0.6546904444694519, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.784419536590576, + "rewards/margins": 1.4846069812774658, + "rewards/rejected": 1.2998125553131104, + "step": 660 + }, + { + "drift/chosen_abs": 0.23545166850090027, + "drift/rejected_abs": 0.14730089902877808, + "epoch": 0.6353756120864684, + "grad_norm": 2.484375, + "learning_rate": 1.0850422203440554e-07, + "logits/chosen": -2.188843250274658, + "logits/rejected": -2.1637561321258545, + "logps/chosen": -0.2837633192539215, + "logps/rejected": -0.2823261618614197, + "loss": 0.7029173374176025, + "loss/core": 0.6890594363212585, + "loss/preference_sft": 0.2837633192539215, + "loss/reference_anchor": 0.24877965450286865, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.353299379348755, + "rewards/margins": 0.8823181390762329, + "rewards/rejected": 1.470981240272522, + "step": 665 + }, + { + "drift/chosen_abs": 0.3646673560142517, + "drift/rejected_abs": 0.12857022881507874, + "epoch": 0.64015287232772, + "grad_norm": 0.2197265625, + "learning_rate": 1.0430235026670978e-07, + "logits/chosen": -1.9835231304168701, + "logits/rejected": -2.0292749404907227, + "logps/chosen": -0.27422162890434265, + "logps/rejected": -0.32053428888320923, + "loss": 0.7076139450073242, + "loss/core": 0.6819421648979187, + "loss/preference_sft": 0.27422162890434265, + "loss/reference_anchor": 0.4860144555568695, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.6459412574768066, + "rewards/margins": 2.392371654510498, + "rewards/rejected": 1.2535693645477295, + "step": 670 + }, + { + "drift/chosen_abs": 0.162667378783226, + "drift/rejected_abs": 0.0890541672706604, + "epoch": 0.6449301325689717, + "grad_norm": 0.73046875, + "learning_rate": 1.0016190477279274e-07, + "logits/chosen": -2.0347633361816406, + "logits/rejected": -2.0776588916778564, + "logps/chosen": -0.2881300151348114, + "logps/rejected": -0.2884693443775177, + "loss": 0.6940901875495911, + "loss/core": 0.6894824504852295, + "loss/preference_sft": 0.2881300151348114, + "loss/reference_anchor": 0.06333953142166138, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6251804828643799, + "rewards/margins": 0.74686199426651, + "rewards/rejected": 0.8783184885978699, + "step": 675 + }, + { + "drift/chosen_abs": 0.21540336310863495, + "drift/rejected_abs": 0.1353510469198227, + "epoch": 0.6497073928102234, + "grad_norm": 0.2265625, + "learning_rate": 9.608463116858542e-08, + "logits/chosen": -2.1595706939697266, + "logits/rejected": -2.077641725540161, + "logps/chosen": -0.2814193069934845, + "logps/rejected": -0.29484206438064575, + "loss": 0.7012907266616821, + "loss/core": 0.6893535852432251, + "loss/preference_sft": 0.2814193069934845, + "loss/reference_anchor": 0.21059997379779816, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.1523005962371826, + "rewards/margins": 0.7994104623794556, + "rewards/rejected": 1.3528902530670166, + "step": 680 + }, + { + "drift/chosen_abs": 0.28880634903907776, + "drift/rejected_abs": 0.15391790866851807, + "epoch": 0.654484653051475, + "grad_norm": 0.30078125, + "learning_rate": 9.207224843668731e-08, + "logits/chosen": -2.0853352546691895, + "logits/rejected": -2.0695528984069824, + "logps/chosen": -0.25348979234695435, + "logps/rejected": -0.27719801664352417, + "loss": 0.7120725512504578, + "loss/core": 0.6871707439422607, + "loss/preference_sft": 0.25348979234695435, + "loss/reference_anchor": 0.47268733382225037, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.887314558029175, + "rewards/margins": 1.3541991710662842, + "rewards/rejected": 1.5331155061721802, + "step": 685 + }, + { + "drift/chosen_abs": 0.17689254879951477, + "drift/rejected_abs": 0.10154334455728531, + "epoch": 0.6592619132927267, + "grad_norm": 0.26171875, + "learning_rate": 8.812644820164158e-08, + "logits/chosen": -1.8848329782485962, + "logits/rejected": -1.9065767526626587, + "logps/chosen": -0.30832767486572266, + "logps/rejected": -0.33626437187194824, + "loss": 0.6952409744262695, + "loss/core": 0.6894530057907104, + "loss/preference_sft": 0.30832767486572266, + "loss/reference_anchor": 0.08492577075958252, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.7667601108551025, + "rewards/margins": 0.7572234869003296, + "rewards/rejected": 1.0095365047454834, + "step": 690 + }, + { + "drift/chosen_abs": 0.19672070443630219, + "drift/rejected_abs": 0.06929643452167511, + "epoch": 0.6640391735339782, + "grad_norm": 0.2197265625, + "learning_rate": 8.424889401674504e-08, + "logits/chosen": -2.270711660385132, + "logits/rejected": -2.279158115386963, + "logps/chosen": -0.3068528175354004, + "logps/rejected": -0.29276537895202637, + "loss": 0.6979945302009583, + "loss/core": 0.6870979070663452, + "loss/preference_sft": 0.3068528175354004, + "loss/reference_anchor": 0.18724682927131653, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9648374319076538, + "rewards/margins": 1.2810237407684326, + "rewards/rejected": 0.6838136911392212, + "step": 695 + }, + { + "drift/chosen_abs": 0.22341635823249817, + "drift/rejected_abs": 0.15612784028053284, + "epoch": 0.6688164337752299, + "grad_norm": 16.875, + "learning_rate": 8.044122066269149e-08, + "logits/chosen": -2.1035265922546387, + "logits/rejected": -2.2100517749786377, + "logps/chosen": -0.2964898943901062, + "logps/rejected": -0.32455042004585266, + "loss": 0.7034405469894409, + "loss/core": 0.6898477673530579, + "loss/preference_sft": 0.2964898943901062, + "loss/reference_anchor": 0.2422069013118744, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.2336838245391846, + "rewards/margins": 0.6977290511131287, + "rewards/rejected": 1.5359545946121216, + "step": 700 + }, + { + "drift/chosen_abs": 0.3399743437767029, + "drift/rejected_abs": 0.1532609462738037, + "epoch": 0.6735936940164815, + "grad_norm": 1.3984375, + "learning_rate": 7.670503345834755e-08, + "logits/chosen": -2.1955857276916504, + "logits/rejected": -2.152186870574951, + "logps/chosen": -0.28090688586235046, + "logps/rejected": -0.26746469736099243, + "loss": 0.7058395743370056, + "loss/core": 0.6843441724777222, + "loss/preference_sft": 0.28090688586235046, + "loss/reference_anchor": 0.40181976556777954, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.399095058441162, + "rewards/margins": 1.8677432537078857, + "rewards/rejected": 1.531352162361145, + "step": 705 + }, + { + "drift/chosen_abs": 0.35864269733428955, + "drift/rejected_abs": 0.18870094418525696, + "epoch": 0.6783709542577332, + "grad_norm": 12.5, + "learning_rate": 7.304190758394774e-08, + "logits/chosen": -1.886185884475708, + "logits/rejected": -1.873063325881958, + "logps/chosen": -0.2785230278968811, + "logps/rejected": -0.2990533411502838, + "loss": 0.7196978330612183, + "loss/core": 0.6850727796554565, + "loss/preference_sft": 0.2785230278968811, + "loss/reference_anchor": 0.6646484136581421, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.5836358070373535, + "rewards/margins": 1.6989152431488037, + "rewards/rejected": 1.884720802307129, + "step": 710 + }, + { + "drift/chosen_abs": 0.18634703755378723, + "drift/rejected_abs": 0.09215612709522247, + "epoch": 0.6831482144989849, + "grad_norm": 0.330078125, + "learning_rate": 6.945338741699769e-08, + "logits/chosen": -2.252730131149292, + "logits/rejected": -2.2337136268615723, + "logps/chosen": -0.2843503952026367, + "logps/rejected": -0.3452153205871582, + "loss": 0.7017387747764587, + "loss/core": 0.6871799826622009, + "loss/preference_sft": 0.2843503952026367, + "loss/reference_anchor": 0.26274123787879944, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.863227128982544, + "rewards/margins": 1.2492995262145996, + "rewards/rejected": 0.6139274835586548, + "step": 715 + }, + { + "drift/chosen_abs": 0.20980620384216309, + "drift/rejected_abs": 0.14841270446777344, + "epoch": 0.6879254747402365, + "grad_norm": 0.4921875, + "learning_rate": 6.594098588116243e-08, + "logits/chosen": -1.9954087734222412, + "logits/rejected": -2.003382921218872, + "logps/chosen": -0.29687029123306274, + "logps/rejected": -0.30213814973831177, + "loss": 0.7040995955467224, + "loss/core": 0.6903706789016724, + "loss/preference_sft": 0.29687029123306274, + "loss/reference_anchor": 0.24489395320415497, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.096773624420166, + "rewards/margins": 0.6157442927360535, + "rewards/rejected": 1.4810290336608887, + "step": 720 + }, + { + "drift/chosen_abs": 0.1588420420885086, + "drift/rejected_abs": 0.0730825662612915, + "epoch": 0.6927027349814882, + "grad_norm": 0.3203125, + "learning_rate": 6.25061838084166e-08, + "logits/chosen": -2.1225600242614746, + "logits/rejected": -2.238051176071167, + "logps/chosen": -0.2854083180427551, + "logps/rejected": -0.277433305978775, + "loss": 0.6947664022445679, + "loss/core": 0.6889495253562927, + "loss/preference_sft": 0.2854083180427551, + "loss/reference_anchor": 0.08779795467853546, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.5876481533050537, + "rewards/margins": 0.8577953577041626, + "rewards/rejected": 0.7298528552055359, + "step": 725 + }, + { + "drift/chosen_abs": 0.10160217434167862, + "drift/rejected_abs": 0.08643589913845062, + "epoch": 0.6974799952227397, + "grad_norm": 13.8125, + "learning_rate": 5.915042931472425e-08, + "logits/chosen": -2.218738317489624, + "logits/rejected": -2.2675373554229736, + "logps/chosen": -0.28681716322898865, + "logps/rejected": -0.2953147888183594, + "loss": 0.6986241340637207, + "loss/core": 0.6925312876701355, + "loss/preference_sft": 0.28681716322898865, + "loss/reference_anchor": 0.09317411482334137, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.015641689300537, + "rewards/margins": 0.15372122824192047, + "rewards/rejected": 0.8619204759597778, + "step": 730 + }, + { + "drift/chosen_abs": 0.14239996671676636, + "drift/rejected_abs": 0.06578416377305984, + "epoch": 0.7022572554639914, + "grad_norm": 1.5625, + "learning_rate": 5.587513718951165e-08, + "logits/chosen": -2.047664165496826, + "logits/rejected": -2.09497332572937, + "logps/chosen": -0.31937894225120544, + "logps/rejected": -0.3183499276638031, + "loss": 0.6945558786392212, + "loss/core": 0.6893695592880249, + "loss/preference_sft": 0.31937894225120544, + "loss/reference_anchor": 0.0717906728386879, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.4239996671676636, + "rewards/margins": 0.7663625478744507, + "rewards/rejected": 0.6576371192932129, + "step": 735 + }, + { + "drift/chosen_abs": 0.25790858268737793, + "drift/rejected_abs": 0.17992377281188965, + "epoch": 0.707034515705243, + "grad_norm": 0.34765625, + "learning_rate": 5.2681688299190455e-08, + "logits/chosen": -2.1005473136901855, + "logits/rejected": -2.1174521446228027, + "logps/chosen": -0.31155890226364136, + "logps/rejected": -0.31915396451950073, + "loss": 0.7307167649269104, + "loss/core": 0.6908460259437561, + "loss/preference_sft": 0.31155890226364136, + "loss/reference_anchor": 0.7662578821182251, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.579061985015869, + "rewards/margins": 0.779824435710907, + "rewards/rejected": 1.799237608909607, + "step": 740 + }, + { + "drift/chosen_abs": 0.3101605176925659, + "drift/rejected_abs": 0.1494651436805725, + "epoch": 0.7118117759464947, + "grad_norm": 7.6875, + "learning_rate": 4.957142900498334e-08, + "logits/chosen": -2.0538952350616455, + "logits/rejected": -2.0975911617279053, + "logps/chosen": -0.29252904653549194, + "logps/rejected": -0.28451254963874817, + "loss": 0.7017096281051636, + "loss/core": 0.6854857206344604, + "loss/preference_sft": 0.29252904653549194, + "loss/reference_anchor": 0.29522526264190674, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.100473403930664, + "rewards/margins": 1.6103368997573853, + "rewards/rejected": 1.4901363849639893, + "step": 745 + }, + { + "drift/chosen_abs": 0.1206597089767456, + "drift/rejected_abs": 0.12509042024612427, + "epoch": 0.7165890361877463, + "grad_norm": 9.8125, + "learning_rate": 4.654567059529668e-08, + "logits/chosen": -2.1884284019470215, + "logits/rejected": -2.180311679840088, + "logps/chosen": -0.31168606877326965, + "logps/rejected": -0.30455487966537476, + "loss": 0.7092859745025635, + "loss/core": 0.6938542127609253, + "loss/preference_sft": 0.31168606877326965, + "loss/reference_anchor": 0.27746719121932983, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.206160306930542, + "rewards/margins": -0.0409439317882061, + "rewards/rejected": 1.2471041679382324, + "step": 750 + }, + { + "drift/chosen_abs": 0.23872093856334686, + "drift/rejected_abs": 0.17086395621299744, + "epoch": 0.721366296428998, + "grad_norm": 0.3515625, + "learning_rate": 4.360568873288009e-08, + "logits/chosen": -1.9273812770843506, + "logits/rejected": -1.8365058898925781, + "logps/chosen": -0.3087690770626068, + "logps/rejected": -0.29509294033050537, + "loss": 0.7011326551437378, + "loss/core": 0.6899515390396118, + "loss/preference_sft": 0.3087690770626068, + "loss/reference_anchor": 0.19274473190307617, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.384294033050537, + "rewards/margins": 0.682456374168396, + "rewards/rejected": 1.7018375396728516, + "step": 755 + }, + { + "drift/chosen_abs": 0.19956620037555695, + "drift/rejected_abs": 0.08658763021230698, + "epoch": 0.7261435566702497, + "grad_norm": 0.33984375, + "learning_rate": 4.075272291700557e-08, + "logits/chosen": -2.1839423179626465, + "logits/rejected": -2.2009308338165283, + "logps/chosen": -0.30236124992370605, + "logps/rejected": -0.29375168681144714, + "loss": 0.6960939168930054, + "loss/core": 0.6876411437988281, + "loss/preference_sft": 0.30236124992370605, + "loss/reference_anchor": 0.13882040977478027, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.9942079782485962, + "rewards/margins": 1.1320140361785889, + "rewards/rejected": 0.8621941804885864, + "step": 760 + }, + { + "drift/chosen_abs": 0.17910103499889374, + "drift/rejected_abs": 0.10586283355951309, + "epoch": 0.7309208169115012, + "grad_norm": 0.69921875, + "learning_rate": 3.798797596089351e-08, + "logits/chosen": -2.0851759910583496, + "logits/rejected": -2.157111644744873, + "logps/chosen": -0.2764049172401428, + "logps/rejected": -0.27264276146888733, + "loss": 0.6968425512313843, + "loss/core": 0.6895520091056824, + "loss/preference_sft": 0.2764049172401428, + "loss/reference_anchor": 0.1181713119149208, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.790848970413208, + "rewards/margins": 0.7349028587341309, + "rewards/rejected": 1.0559461116790771, + "step": 765 + }, + { + "drift/chosen_abs": 0.36868372559547424, + "drift/rejected_abs": 0.23421208560466766, + "epoch": 0.7356980771527529, + "grad_norm": 0.326171875, + "learning_rate": 3.531261348460554e-08, + "logits/chosen": -1.9794762134552002, + "logits/rejected": -1.945230484008789, + "logps/chosen": -0.31459537148475647, + "logps/rejected": -0.28590553998947144, + "loss": 0.7430247664451599, + "loss/core": 0.6873718500137329, + "loss/preference_sft": 0.31459537148475647, + "loss/reference_anchor": 1.0815989971160889, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.6851744651794434, + "rewards/margins": 1.344879388809204, + "rewards/rejected": 2.3402953147888184, + "step": 770 + }, + { + "drift/chosen_abs": 0.15195751190185547, + "drift/rejected_abs": 0.09389922767877579, + "epoch": 0.7404753373940045, + "grad_norm": 0.2109375, + "learning_rate": 3.272776342361791e-08, + "logits/chosen": -2.0958571434020996, + "logits/rejected": -2.123683452606201, + "logps/chosen": -0.30696946382522583, + "logps/rejected": -0.3006220757961273, + "loss": 0.6954344511032104, + "loss/core": 0.6903113722801208, + "loss/preference_sft": 0.30696946382522583, + "loss/reference_anchor": 0.07176429778337479, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.519211769104004, + "rewards/margins": 0.5804935693740845, + "rewards/rejected": 0.9387180209159851, + "step": 775 + }, + { + "drift/chosen_abs": 0.18850034475326538, + "drift/rejected_abs": 0.12492908537387848, + "epoch": 0.7452525976352562, + "grad_norm": 0.359375, + "learning_rate": 3.023451555328252e-08, + "logits/chosen": -2.181946277618408, + "logits/rejected": -2.1955080032348633, + "logps/chosen": -0.30274689197540283, + "logps/rejected": -0.30594930052757263, + "loss": 0.6972261667251587, + "loss/core": 0.6900213956832886, + "loss/preference_sft": 0.30274689197540283, + "loss/reference_anchor": 0.11382057517766953, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8849332332611084, + "rewards/margins": 0.6389666795730591, + "rewards/rejected": 1.2459665536880493, + "step": 780 + }, + { + "drift/chosen_abs": 0.3422924876213074, + "drift/rejected_abs": 0.13462868332862854, + "epoch": 0.7500298578765078, + "grad_norm": 0.240234375, + "learning_rate": 2.7833921029376817e-08, + "logits/chosen": -1.9921424388885498, + "logits/rejected": -1.949812650680542, + "logps/chosen": -0.3121167719364166, + "logps/rejected": -0.2942814230918884, + "loss": 0.7160924673080444, + "loss/core": 0.6839752197265625, + "loss/preference_sft": 0.3121167719364166, + "loss/reference_anchor": 0.6111334562301636, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.419778347015381, + "rewards/margins": 2.079742431640625, + "rewards/rejected": 1.3400366306304932, + "step": 785 + }, + { + "drift/chosen_abs": 0.26127004623413086, + "drift/rejected_abs": 0.19187363982200623, + "epoch": 0.7548071181177595, + "grad_norm": 11.3125, + "learning_rate": 2.5526991944935488e-08, + "logits/chosen": -2.168665647506714, + "logits/rejected": -1.9961109161376953, + "logps/chosen": -0.2868359684944153, + "logps/rejected": -0.3195565342903137, + "loss": 0.720409095287323, + "loss/core": 0.6900395750999451, + "loss/preference_sft": 0.2868359684944153, + "loss/reference_anchor": 0.5787058472633362, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.6118552684783936, + "rewards/margins": 0.6948817372322083, + "rewards/rejected": 1.916973352432251, + "step": 790 + }, + { + "drift/chosen_abs": 0.23737922310829163, + "drift/rejected_abs": 0.11862516403198242, + "epoch": 0.7595843783590112, + "grad_norm": 0.5, + "learning_rate": 2.3314700903550838e-08, + "logits/chosen": -1.9713795185089111, + "logits/rejected": -1.989896535873413, + "logps/chosen": -0.3081986904144287, + "logps/rejected": -0.29614824056625366, + "loss": 0.7151690721511841, + "loss/core": 0.6874554753303528, + "loss/preference_sft": 0.3081986904144287, + "loss/reference_anchor": 0.5234512090682983, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.3726906776428223, + "rewards/margins": 1.1869637966156006, + "rewards/rejected": 1.185726523399353, + "step": 795 + }, + { + "drift/chosen_abs": 0.22930972278118134, + "drift/rejected_abs": 0.11694236099720001, + "epoch": 0.7643616386002627, + "grad_norm": 0.31640625, + "learning_rate": 2.11979806093224e-08, + "logits/chosen": -2.1524062156677246, + "logits/rejected": -2.1292905807495117, + "logps/chosen": -0.28981950879096985, + "logps/rejected": -0.2884742319583893, + "loss": 0.696781575679779, + "loss/core": 0.6877356767654419, + "loss/preference_sft": 0.28981950879096985, + "loss/reference_anchor": 0.1519363820552826, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.2911429405212402, + "rewards/margins": 1.1217482089996338, + "rewards/rejected": 1.1693947315216064, + "step": 800 + }, + { + "drift/chosen_abs": 0.22585976123809814, + "drift/rejected_abs": 0.19897663593292236, + "epoch": 0.7691388988415144, + "grad_norm": 0.73828125, + "learning_rate": 1.9177723473627643e-08, + "logits/chosen": -2.0267646312713623, + "logits/rejected": -2.014665126800537, + "logps/chosen": -0.27477264404296875, + "logps/rejected": -0.2703710198402405, + "loss": 0.706299901008606, + "loss/core": 0.6920345425605774, + "loss/preference_sft": 0.27477264404296875, + "loss/reference_anchor": 0.2578306198120117, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.2579760551452637, + "rewards/margins": 0.2697838544845581, + "rewards/rejected": 1.9881923198699951, + "step": 805 + }, + { + "drift/chosen_abs": 0.15302561223506927, + "drift/rejected_abs": 0.14136184751987457, + "epoch": 0.773916159082766, + "grad_norm": 0.404296875, + "learning_rate": 1.7254781238880828e-08, + "logits/chosen": -2.2281618118286133, + "logits/rejected": -2.132458209991455, + "logps/chosen": -0.30843856930732727, + "logps/rejected": -0.32327035069465637, + "loss": 0.7003055810928345, + "loss/core": 0.6927086114883423, + "loss/preference_sft": 0.30843856930732727, + "loss/reference_anchor": 0.12109792232513428, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.5288832187652588, + "rewards/margins": 0.11678824573755264, + "rewards/rejected": 1.4120948314666748, + "step": 810 + }, + { + "drift/chosen_abs": 0.2130151093006134, + "drift/rejected_abs": 0.11095918715000153, + "epoch": 0.7786934193240177, + "grad_norm": 1.53125, + "learning_rate": 1.5429964619437158e-08, + "logits/chosen": -2.0964717864990234, + "logits/rejected": -2.1862375736236572, + "logps/chosen": -0.2831994891166687, + "logps/rejected": -0.28318652510643005, + "loss": 0.6956707239151001, + "loss/core": 0.6880777478218079, + "loss/preference_sft": 0.2831994891166687, + "loss/reference_anchor": 0.1235402375459671, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.1292998790740967, + "rewards/margins": 1.024322509765625, + "rewards/rejected": 1.1049773693084717, + "step": 815 + }, + { + "drift/chosen_abs": 0.19676700234413147, + "drift/rejected_abs": 0.0735253393650055, + "epoch": 0.7834706795652693, + "grad_norm": 0.2158203125, + "learning_rate": 1.3704042959795131e-08, + "logits/chosen": -2.1000118255615234, + "logits/rejected": -2.133841037750244, + "logps/chosen": -0.28775379061698914, + "logps/rejected": -0.3174736201763153, + "loss": 0.6944074034690857, + "loss/core": 0.6869372129440308, + "loss/preference_sft": 0.28775379061698914, + "loss/reference_anchor": 0.12063012272119522, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.9654747247695923, + "rewards/margins": 1.2821400165557861, + "rewards/rejected": 0.6833347082138062, + "step": 820 + }, + { + "drift/chosen_abs": 0.24645988643169403, + "drift/rejected_abs": 0.1214875802397728, + "epoch": 0.788247939806521, + "grad_norm": 1.0546875, + "learning_rate": 1.2077743910239996e-08, + "logits/chosen": -2.1431891918182373, + "logits/rejected": -2.247856616973877, + "logps/chosen": -0.3019647002220154, + "logps/rejected": -0.28489747643470764, + "loss": 0.696218729019165, + "loss/core": 0.6870028972625732, + "loss/preference_sft": 0.3019647002220154, + "loss/reference_anchor": 0.15411940217018127, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.4645986557006836, + "rewards/margins": 1.2513694763183594, + "rewards/rejected": 1.2132294178009033, + "step": 825 + }, + { + "drift/chosen_abs": 0.17260222136974335, + "drift/rejected_abs": 0.1575520932674408, + "epoch": 0.7930252000477725, + "grad_norm": 0.52734375, + "learning_rate": 1.0551753120065621e-08, + "logits/chosen": -2.17891263961792, + "logits/rejected": -2.12709379196167, + "logps/chosen": -0.2786734700202942, + "logps/rejected": -0.31584739685058594, + "loss": 0.7015553116798401, + "loss/core": 0.692550539970398, + "loss/preference_sft": 0.2786734700202942, + "loss/reference_anchor": 0.15222761034965515, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.724945306777954, + "rewards/margins": 0.15078657865524292, + "rewards/rejected": 1.5741586685180664, + "step": 830 + }, + { + "drift/chosen_abs": 0.24012541770935059, + "drift/rejected_abs": 0.11180107295513153, + "epoch": 0.7978024602890242, + "grad_norm": 1.71875, + "learning_rate": 9.126713948504228e-09, + "logits/chosen": -2.094947099685669, + "logits/rejected": -2.0943989753723145, + "logps/chosen": -0.2551843822002411, + "logps/rejected": -0.29870468378067017, + "loss": 0.6966162919998169, + "loss/core": 0.685853123664856, + "loss/preference_sft": 0.2551843822002411, + "loss/reference_anchor": 0.18974518775939941, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.3995418548583984, + "rewards/margins": 1.5449695587158203, + "rewards/rejected": 0.8545724749565125, + "step": 835 + }, + { + "drift/chosen_abs": 0.26994588971138, + "drift/rejected_abs": 0.10432889312505722, + "epoch": 0.8025797205302759, + "grad_norm": 61.0, + "learning_rate": 7.803227193485334e-09, + "logits/chosen": -2.03291916847229, + "logits/rejected": -2.0835347175598145, + "logps/chosen": -0.33308959007263184, + "logps/rejected": -0.30082958936691284, + "loss": 0.7072250843048096, + "loss/core": 0.6854801177978516, + "loss/preference_sft": 0.33308959007263184, + "loss/reference_anchor": 0.4015893340110779, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.696979522705078, + "rewards/margins": 1.6540577411651611, + "rewards/rejected": 1.042922019958496, + "step": 840 + }, + { + "drift/chosen_abs": 0.14128538966178894, + "drift/rejected_abs": 0.06555317342281342, + "epoch": 0.8073569807715275, + "grad_norm": 0.296875, + "learning_rate": 6.581850838338815e-09, + "logits/chosen": -2.1707913875579834, + "logits/rejected": -2.2462358474731445, + "logps/chosen": -0.29429852962493896, + "logps/rejected": -0.3137997090816498, + "loss": 0.6939007043838501, + "loss/core": 0.688424825668335, + "loss/preference_sft": 0.29429852962493896, + "loss/reference_anchor": 0.0800880640745163, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.4111610651016235, + "rewards/margins": 0.9817382097244263, + "rewards/rejected": 0.42942291498184204, + "step": 845 + }, + { + "drift/chosen_abs": 0.1559392213821411, + "drift/rejected_abs": 0.09002901613712311, + "epoch": 0.8121342410127792, + "grad_norm": 2.640625, + "learning_rate": 5.463099816548577e-09, + "logits/chosen": -2.2900779247283936, + "logits/rejected": -2.3381993770599365, + "logps/chosen": -0.25206631422042847, + "logps/rejected": -0.24866075813770294, + "loss": 0.6935194730758667, + "loss/core": 0.6898444294929504, + "loss/preference_sft": 0.25206631422042847, + "loss/reference_anchor": 0.0482938326895237, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.557403564453125, + "rewards/margins": 0.6683042645454407, + "rewards/rejected": 0.8890994191169739, + "step": 850 + }, + { + "drift/chosen_abs": 0.26853224635124207, + "drift/rejected_abs": 0.12536677718162537, + "epoch": 0.8169115012540308, + "grad_norm": 0.890625, + "learning_rate": 4.447445794656224e-09, + "logits/chosen": -2.002507209777832, + "logits/rejected": -2.0403895378112793, + "logps/chosen": -0.3012729287147522, + "logps/rejected": -0.2928231358528137, + "loss": 0.7090455293655396, + "loss/core": 0.6863538026809692, + "loss/preference_sft": 0.3012729287147522, + "loss/reference_anchor": 0.42370709776878357, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.6843512058258057, + "rewards/margins": 1.4311248064041138, + "rewards/rejected": 1.253226399421692, + "step": 855 + }, + { + "drift/chosen_abs": 0.4081405997276306, + "drift/rejected_abs": 0.19295448064804077, + "epoch": 0.8216887614952825, + "grad_norm": 0.1943359375, + "learning_rate": 3.535316973405672e-09, + "logits/chosen": -1.9501336812973022, + "logits/rejected": -2.0376601219177246, + "logps/chosen": -0.3001163601875305, + "logps/rejected": -0.30008673667907715, + "loss": 0.7247797250747681, + "loss/core": 0.6830560564994812, + "loss/preference_sft": 0.3001163601875305, + "loss/reference_anchor": 0.8044621348381042, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.080620765686035, + "rewards/margins": 2.163020610809326, + "rewards/rejected": 1.9175999164581299, + "step": 860 + }, + { + "drift/chosen_abs": 0.24484309554100037, + "drift/rejected_abs": 0.15850794315338135, + "epoch": 0.826466021736534, + "grad_norm": 0.8359375, + "learning_rate": 2.7270979072135104e-09, + "logits/chosen": -1.9584686756134033, + "logits/rejected": -2.0302765369415283, + "logps/chosen": -0.29134422540664673, + "logps/rejected": -0.2941442131996155, + "loss": 0.7054814696311951, + "loss/core": 0.6891493797302246, + "loss/preference_sft": 0.29134422540664673, + "loss/reference_anchor": 0.29750776290893555, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.447580099105835, + "rewards/margins": 0.864477813243866, + "rewards/rejected": 1.5831019878387451, + "step": 865 + }, + { + "drift/chosen_abs": 0.3205713629722595, + "drift/rejected_abs": 0.09417042136192322, + "epoch": 0.8312432819777857, + "grad_norm": 0.59765625, + "learning_rate": 2.0231293420405195e-09, + "logits/chosen": -2.1127753257751465, + "logits/rejected": -2.1212573051452637, + "logps/chosen": -0.3091801106929779, + "logps/rejected": -0.2969990670681, + "loss": 0.7111729979515076, + "loss/core": 0.6829420924186707, + "loss/preference_sft": 0.3091801106929779, + "loss/reference_anchor": 0.533698320388794, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.204364776611328, + "rewards/margins": 2.2668018341064453, + "rewards/rejected": 0.9375633001327515, + "step": 870 + }, + { + "drift/chosen_abs": 0.18526910245418549, + "drift/rejected_abs": 0.10249395668506622, + "epoch": 0.8360205422190374, + "grad_norm": 0.396484375, + "learning_rate": 1.423708071732671e-09, + "logits/chosen": -2.1826789379119873, + "logits/rejected": -2.230438232421875, + "logps/chosen": -0.30893266201019287, + "logps/rejected": -0.33167764544487, + "loss": 0.7087309956550598, + "loss/core": 0.6893488168716431, + "loss/preference_sft": 0.30893266201019287, + "loss/reference_anchor": 0.3567507863044739, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.8521311283111572, + "rewards/margins": 0.8370383381843567, + "rewards/rejected": 1.0150927305221558, + "step": 875 + }, + { + "drift/chosen_abs": 0.24597208201885223, + "drift/rejected_abs": 0.18042945861816406, + "epoch": 0.840797802460289, + "grad_norm": 1.0625, + "learning_rate": 9.290868128926377e-10, + "logits/chosen": -2.1186165809631348, + "logits/rejected": -2.100987672805786, + "logps/chosen": -0.2905484139919281, + "logps/rejected": -0.3378651738166809, + "loss": 0.7146252393722534, + "loss/core": 0.6903001666069031, + "loss/preference_sft": 0.2905484139919281, + "loss/reference_anchor": 0.4574460983276367, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.4589881896972656, + "rewards/margins": 0.7572067379951477, + "rewards/rejected": 1.7017812728881836, + "step": 880 + }, + { + "drift/chosen_abs": 0.2725023329257965, + "drift/rejected_abs": 0.11308582872152328, + "epoch": 0.8455750627015407, + "grad_norm": 0.373046875, + "learning_rate": 5.394740983341861e-10, + "logits/chosen": -2.043205976486206, + "logits/rejected": -2.1195883750915527, + "logps/chosen": -0.2826032042503357, + "logps/rejected": -0.27686744928359985, + "loss": 0.6973305940628052, + "loss/core": 0.6853419542312622, + "loss/preference_sft": 0.2826032042503357, + "loss/reference_anchor": 0.2115131914615631, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.724541187286377, + "rewards/margins": 1.5997540950775146, + "rewards/rejected": 1.1247870922088623, + "step": 885 + }, + { + "drift/chosen_abs": 0.2453497350215912, + "drift/rejected_abs": 0.14477917551994324, + "epoch": 0.8503523229427923, + "grad_norm": 5.6875, + "learning_rate": 2.550341891646435e-10, + "logits/chosen": -1.9740431308746338, + "logits/rejected": -2.0952517986297607, + "logps/chosen": -0.30029481649398804, + "logps/rejected": -0.2738454043865204, + "loss": 0.7060577869415283, + "loss/core": 0.6884250640869141, + "loss/preference_sft": 0.30029481649398804, + "loss/reference_anchor": 0.32262665033340454, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.4530909061431885, + "rewards/margins": 1.009745717048645, + "rewards/rejected": 1.443345308303833, + "step": 890 + }, + { + "drift/chosen_abs": 0.22996075451374054, + "drift/rejected_abs": 0.1483142226934433, + "epoch": 0.855129583184044, + "grad_norm": 2.015625, + "learning_rate": 7.588700553209926e-11, + "logits/chosen": -2.0447440147399902, + "logits/rejected": -2.0767829418182373, + "logps/chosen": -0.27262455224990845, + "logps/rejected": -0.2941727340221405, + "loss": 0.6983267068862915, + "loss/core": 0.6891483068466187, + "loss/preference_sft": 0.27262455224990845, + "loss/reference_anchor": 0.15630510449409485, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.2990550994873047, + "rewards/margins": 0.8220510482788086, + "rewards/rejected": 1.477004051208496, + "step": 895 + }, + { + "drift/chosen_abs": 0.30940914154052734, + "drift/rejected_abs": 0.08630160987377167, + "epoch": 0.8599068434252956, + "grad_norm": 0.65234375, + "learning_rate": 2.1080760670144636e-12, + "logits/chosen": -2.055746555328369, + "logits/rejected": -2.1760077476501465, + "logps/chosen": -0.2712702453136444, + "logps/rejected": -0.268751323223114, + "loss": 0.7091078758239746, + "loss/core": 0.6829676032066345, + "loss/preference_sft": 0.2712702453136444, + "loss/reference_anchor": 0.4956776201725006, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 3.0939459800720215, + "rewards/margins": 2.2492384910583496, + "rewards/rejected": 0.8447073101997375, + "step": 900 + }, + { + "epoch": 0.8599068434252956, + "step": 900, + "total_flos": 0.0, + "train_loss": 0.7066660939322578, + "train_runtime": 7291.0823, + "train_samples_per_second": 1.975, + "train_steps_per_second": 0.123 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..ae577df --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14efa4061992d64e8388448d5d8e71f0820fe2e786143d9fa23f78658612645f +size 6993 diff --git a/training_status.json b/training_status.json new file mode 100644 index 0000000..88e5eef --- /dev/null +++ b/training_status.json @@ -0,0 +1,20 @@ +{ + "status": "completed", + "candidate_id": "dpo_b", + "method": "dpo", + "seed": 42, + "gpu": 0, + "pid": 423793, + "config": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/fair_demo_20260715/sweep/candidates/dpo_b/config.yaml", + "dataset": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/avg", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "bce8f73e973a", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/bce8f73e973a", + "started_at": "2026-07-15T11:34:23+09:00", + "returncode": 0, + "model_complete": true, + "finite_results": true, + "measured_step": null, + "completed_at": "2026-07-15T13:36:52+09:00" +}