From 9315ced0ffa534de9484247e29510b97073b7df3 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 18 Jul 2026 13:23:11 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: promotion/ronpo-qwen3-8b-fair-sppo-avg-s42 Source: Original Platform --- .gitattributes | 36 + README.md | 12 + all_results.json | 9 + chat_template.jinja | 89 + config.json | 71 + config.yaml | 58 + experiment/evaluator_lock.json | 85 + experiment/selection_lock.json | 99 + experiment/sweep_grid.json | 46 + experiment/training_status.json | 20 + generation_config.json | 12 + model.safetensors | 3 + tokenizer.json | 3 + tokenizer_config.json | 30 + train_results.json | 9 + trainer_state.json | 3643 +++++++++++++++++++++++++++++++ training_args.bin | 3 + training_status.json | 20 + 18 files changed, 4248 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 config.yaml create mode 100644 experiment/evaluator_lock.json create mode 100644 experiment/selection_lock.json create mode 100644 experiment/sweep_grid.json create mode 100644 experiment/training_status.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin create mode 100644 training_status.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..1dcbfff --- /dev/null +++ b/README.md @@ -0,0 +1,12 @@ +--- +base_model: Qwen/Qwen3-8B +library_name: transformers +tags: +- preference-optimization +- ronpo +- qwen3 +--- + +# Qwen3-8B fair-demo checkpoint: sppo_avg + +Validation-selected candidate: `sppo_b`. Exact evaluator, selection, sweep, and training metadata are stored under `experiment/`. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..fab812d --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 0.7444233691361215, + "train_runtime": 7251.7494, + "train_samples": 16746, + "train_samples_per_second": 1.986, + "train_steps_per_second": 0.124 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..1fb7875 --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +model_name_or_path: /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218 +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/avg: 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: sppo +eta: 0.01 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.075 +preference_sft_weight: 0.0075 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 5.0e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.2 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 42 +gradient_accumulation_steps: 16 +gradient_checkpointing: true +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 1 +per_device_eval_batch_size: 1 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/fair_demo_20260715/sweep/candidates/sppo_b +run_name: qwen3-8b-fair-demo-sppo_b diff --git a/experiment/evaluator_lock.json b/experiment/evaluator_lock.json new file mode 100644 index 0000000..b406c9a --- /dev/null +++ b/experiment/evaluator_lock.json @@ -0,0 +1,85 @@ +{ + "status": "LOCKED_BEFORE_ANY_NEW_METHOD_RANKING", + "locked_at": "2026-07-15T20:20:29+09:00", + "objective_signals": [ + "skywork", + "armo_safety", + "length_conciseness" + ], + "objective_semantics": [ + "helpfulness", + "safety", + "conciseness" + ], + "primary": { + "name": "open_weight_panel_mean_prompt_worst_vs_base", + "definition": "For each prompt and objective, average win=1/tie=0.5/loss=0 over two A/B positions and two judges; take the minimum objective, then mean over prompts.", + "judges": [ + { + "model": "openai/gpt-oss-120b", + "revision": "b5c939de8f754692c1647ca79fbf85e8c1e70f8a" + }, + { + "model": "Qwen/Qwen3-32B", + "revision": "9216db5781bf21249d130ec9da846c4624c16137" + } + ], + "decode": { + "temperature": 0.0, + "top_p": 1.0, + "max_new_tokens": 512, + "seed": 42 + }, + "position_swap": true, + "validation_selection_rule": "Highest eligible mean prompt-level worst panel score within each method; an exact numeric tie is broken by candidate_id lexical order." + }, + "secondary": { + "normalization": "For each locked signal, divide each evaluation prompt's candidate-minus-base raw delta by the population SD of the diagnostic base and matched-control scores; take the prompt-level minimum across objectives, then the mean. No per-prompt min-max.", + "diagnostic_control_scale": { + "skywork": 22.807301785782975, + "armo_safety": 0.3098083353203591, + "length_conciseness": 1.5891969646897472 + }, + "reward_signal_provenance": { + "skywork": { + "model": "Skywork/Skywork-Reward-V2-Llama-3.1-8B", + "revision": "cba2f842f3f1af2f1b2f0d35e794d789976390c5", + "semantics": "helpfulness" + }, + "armo_safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "revision": "eb2676d20da2f2d41082289d23c59b9f7427f955", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "length_conciseness": { + "deterministic": "-log1p(response_word_count)", + "tokenization": "Python str.split whitespace words" + } + }, + "report_raw_paired_deltas": true + }, + "bootstrap": { + "paired_prompt_resamples": 2000, + "seed": 42, + "interval": "percentile_95" + }, + "power": { + "target_absolute_effect": 0.05, + "paired_sd": 0.13831629563357775, + "required_prompts_80pct_power": 61, + "planned_fresh_test_prompts": 1024, + "alpha_two_sided": 0.05, + "power": 0.8 + }, + "fresh_test_source": { + "dataset": "HuggingFaceH4/ultrachat_200k", + "revision": "8049631c405ae6576f93f445c6b8166f76f5505a", + "split": "test_sft" + }, + "diagnostic_summary_sha256": "e365ed814926ac5c17777e0fc65d0299a45b81bd390542889bbd55b5453d4ffe", + "judge_diagnostic_lock_sha256": "fce2d97806053cabdf3d358806f4baa80df7f89d71d8c1c298883327d4e02635", + "prereg_sha256": "71cbd1ac17c3e807f2ece1cb0a82374bbe74edb654c2845b97c3222ed680f0d0", + "method_ranking_computed": false, + "spent_sealed_split_touched": false +} diff --git a/experiment/selection_lock.json b/experiment/selection_lock.json new file mode 100644 index 0000000..bae8b9c --- /dev/null +++ b/experiment/selection_lock.json @@ -0,0 +1,99 @@ +{ + "status": "VALIDATION_SELECTION_LOCKED_BEFORE_FRESH_TEST", + "locked_at": "2026-07-15T20:37:11+09:00", + "selection_metric": "open_weight_panel_mean_prompt_worst_vs_base", + "tie_break": "candidate_id lexical order", + "selected_by_method": { + "dpo": { + "candidate_id": "dpo_b", + "validation_primary": 0.40625, + "validation_primary_ci95": [ + 0.3837890625, + 0.427734375 + ], + "eligible_candidates": [ + "dpo_a", + "dpo_b" + ] + }, + "ht_mnpo_helpfulness": { + "candidate_id": "ht_help_b", + "validation_primary": 0.4072265625, + "validation_primary_ci95": [ + 0.3828125, + 0.4296875 + ], + "eligible_candidates": [ + "ht_help_b" + ] + }, + "ht_mnpo_safety": { + "candidate_id": "ht_safety_a", + "validation_primary": 0.4150390625, + "validation_primary_ci95": [ + 0.3925537109375, + 0.4384765625 + ], + "eligible_candidates": [ + "ht_safety_a", + "ht_safety_b" + ] + }, + "inpo_avg": { + "candidate_id": "inpo_b", + "validation_primary": 0.40625, + "validation_primary_ci95": [ + 0.3818359375, + 0.427734375 + ], + "eligible_candidates": [ + "inpo_b" + ] + }, + "ronpo_full_expect": { + "candidate_id": "ronpo_full_a", + "validation_primary": 0.4033203125, + "validation_primary_ci95": [ + 0.37890625, + 0.42580566406249987 + ], + "eligible_candidates": [ + "ronpo_full_a", + "ronpo_full_b" + ] + }, + "ronpo_k_only": { + "candidate_id": "ronpo_top_a", + "validation_primary": 0.412109375, + "validation_primary_ci95": [ + 0.3876953125, + 0.43557128906249987 + ], + "eligible_candidates": [ + "ronpo_top_a" + ] + }, + "sppo_avg": { + "candidate_id": "sppo_b", + "validation_primary": 0.4111328125, + "validation_primary_ci95": [ + 0.3866943359375, + 0.4345703125 + ], + "eligible_candidates": [ + "sppo_b" + ] + } + }, + "selected_ronpo_overall": "ronpo_top_a", + "failed_methods": [ + "ht_mnpo_conciseness", + "ipo", + "simpo" + ], + "panel_summary_sha256": "8850f23e16cadf56a99e3d562d224ce5337ab36fa588d8a843382853e6ad17bd", + "evaluator_lock_sha256": "ab7b12c84932ed57acfec7edbcbc1ff1df5e9acd203869b1458343adf6e5b2c8", + "grid_sha256": "abfd982ef10c3bc71364752241150286ebf06be5e97f087596adf80e55d459e0", + "fresh_test_opened": false, + "spent_sealed_split_touched": false +} diff --git a/experiment/sweep_grid.json b/experiment/sweep_grid.json new file mode 100644 index 0000000..1ee759f --- /dev/null +++ b/experiment/sweep_grid.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "status": "frozen_before_training_and_validation_ranking", + "frozen_at_kst": "2026-07-15T09:01:45+09:00", + "seed": 42, + "common": { + "optimizer_steps": 900, + "effective_batch_size": 16, + "per_device_train_batch_size": 1, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "bf16": true, + "attn_implementation": "sdpa", + "cudnn_sdpa": false, + "lr_scheduler_type": "cosine", + "max_length": 2048, + "max_prompt_length": 1800, + "save_total_limit": 1, + "wandb_entity": "promotion-kim", + "wandb_project": "mnpo", + "wandb_group": "qwen3-8b-fair-demo-20260715" + }, + "budget_rule": "Exactly two configs per reported method; no extension after validation rankings are visible.", + "candidates": [ + {"id":"ronpo_full_a","method":"ronpo_full_expect","dataset":"ronpo_full_expect_kall","learning_rate":1e-7,"warmup_ratio":0.15,"ronpo_alpha":0.25,"ronpo_tau":0.10,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075,"expected_support_k":0}, + {"id":"ronpo_full_b","method":"ronpo_full_expect","dataset":"ronpo_full_expect_k6","learning_rate":2.5e-7,"warmup_ratio":0.20,"ronpo_alpha":0.35,"ronpo_tau":0.05,"reference_anchor_weight":0.10,"preference_sft_weight":0.01,"expected_support_k":6}, + {"id":"ronpo_top_a","method":"ronpo_k_only","dataset":"ronpo_k_only_k1","learning_rate":5e-8,"warmup_ratio":0.20,"ronpo_alpha":0.15,"ronpo_tau":0.10,"reference_anchor_weight":0.10,"preference_sft_weight":0.01,"expected_support_k":1}, + {"id":"ronpo_top_b","method":"ronpo_k_only","dataset":"ronpo_k_only_k2","learning_rate":1e-7,"warmup_ratio":0.15,"ronpo_alpha":0.25,"ronpo_tau":0.10,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075,"expected_support_k":2}, + {"id":"dpo_a","method":"dpo","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.10,"dpo_beta":0.05,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"dpo_b","method":"dpo","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.15,"dpo_beta":0.10,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ipo_a","method":"ipo","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.15,"dpo_beta":0.20,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ipo_b","method":"ipo","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.20,"dpo_beta":0.50,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075}, + {"id":"simpo_a","method":"simpo","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.10,"simpo_beta":1.0,"simpo_gamma":0.3,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"simpo_b","method":"simpo","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.15,"simpo_beta":2.0,"simpo_gamma":0.6,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"sppo_a","method":"sppo_avg","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.15,"eta":0.005,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"sppo_b","method":"sppo_avg","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.20,"eta":0.010,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075}, + {"id":"inpo_a","method":"inpo_avg","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.15,"eta":0.050,"ratio":0.3333,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"inpo_b","method":"inpo_avg","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.20,"eta":0.100,"ratio":0.3333,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075}, + {"id":"ht_help_a","method":"ht_mnpo_helpfulness","dataset":"ht_mnpo_helpfulness","learning_rate":2.5e-7,"warmup_ratio":0.10,"eta":0.005,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"ht_help_b","method":"ht_mnpo_helpfulness","dataset":"ht_mnpo_helpfulness","learning_rate":5e-7,"warmup_ratio":0.15,"eta":0.010,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ht_safety_a","method":"ht_mnpo_safety","dataset":"ht_mnpo_safety","learning_rate":2.5e-7,"warmup_ratio":0.10,"eta":0.005,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"ht_safety_b","method":"ht_mnpo_safety","dataset":"ht_mnpo_safety","learning_rate":5e-7,"warmup_ratio":0.15,"eta":0.010,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ht_concise_a","method":"ht_mnpo_conciseness","dataset":"ht_mnpo_conciseness","learning_rate":2.5e-7,"warmup_ratio":0.10,"eta":0.005,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"ht_concise_b","method":"ht_mnpo_conciseness","dataset":"ht_mnpo_conciseness","learning_rate":5e-7,"warmup_ratio":0.15,"eta":0.010,"reference_anchor_weight":0.05,"preference_sft_weight":0.005} + ] +} diff --git a/experiment/training_status.json b/experiment/training_status.json new file mode 100644 index 0000000..41cf43c --- /dev/null +++ b/experiment/training_status.json @@ -0,0 +1,20 @@ +{ + "status": "completed", + "candidate_id": "sppo_b", + "method": "sppo_avg", + "seed": 42, + "gpu": 3, + "pid": 762647, + "config": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/fair_demo_20260715/sweep/candidates/sppo_b/config.yaml", + "dataset": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/avg", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "a2bd72efe9d9", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/a2bd72efe9d9", + "started_at": "2026-07-15T13:38:53+09:00", + "returncode": 0, + "model_complete": true, + "finite_results": true, + "measured_step": null, + "completed_at": "2026-07-15T15:40:43+09:00" +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..bbee1a0 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b94cd9477d93b424a30a411493ac9edee08e1d6ce3caac9d3f95c2a164a09efd +size 16381517208 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..e2cb12d --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": true, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..fab812d --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 0.7444233691361215, + "train_runtime": 7251.7494, + "train_samples": 16746, + "train_samples_per_second": 1.986, + "train_steps_per_second": 0.124 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..ab8ca9f --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,3643 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8599068434252956, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "drift/chosen_abs": 0.241798996925354, + "drift/rejected_abs": 0.09728006273508072, + "epoch": 0.004777260241251642, + "grad_norm": 30.875, + "learning_rate": 1.111111111111111e-08, + "logits/chosen": -1.9988330602645874, + "logits/rejected": -2.015357494354248, + "logps/chosen": -0.279768705368042, + "logps/rejected": -0.28206831216812134, + "loss": 0.39691096544265747, + "loss/core": 0.3805164694786072, + "loss/preference_sft": 0.279768705368042, + "loss/reference_anchor": 0.19061610102653503, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.4163806438446045, + "rewards/margins": 1.4569675922393799, + "rewards/rejected": 0.9594132304191589, + "step": 5 + }, + { + "drift/chosen_abs": 0.16912893950939178, + "drift/rejected_abs": 0.16102473437786102, + "epoch": 0.009554520482503284, + "grad_norm": 716.0, + "learning_rate": 2.5e-08, + "logits/chosen": -2.344573497772217, + "logits/rejected": -2.3923327922821045, + "logps/chosen": -0.28925657272338867, + "logps/rejected": -0.28922492265701294, + "loss": 1.0646731853485107, + "loss/core": 1.0241014957427979, + "loss/preference_sft": 0.28925657272338867, + "loss/reference_anchor": 0.5120292901992798, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6911067962646484, + "rewards/margins": 0.08549212664365768, + "rewards/rejected": 1.6056146621704102, + "step": 10 + }, + { + "drift/chosen_abs": 0.21334102749824524, + "drift/rejected_abs": 0.11430090665817261, + "epoch": 0.014331780723754926, + "grad_norm": 100.0, + "learning_rate": 3.888888888888889e-08, + "logits/chosen": -2.2332091331481934, + "logits/rejected": -2.1960010528564453, + "logps/chosen": -0.27760133147239685, + "logps/rejected": -0.28394412994384766, + "loss": 0.37293222546577454, + "loss/core": 0.35742494463920593, + "loss/preference_sft": 0.27760133147239685, + "loss/reference_anchor": 0.17900365591049194, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.132291316986084, + "rewards/margins": 0.991050124168396, + "rewards/rejected": 1.1412410736083984, + "step": 15 + }, + { + "drift/chosen_abs": 0.19138672947883606, + "drift/rejected_abs": 0.08749718964099884, + "epoch": 0.01910904096500657, + "grad_norm": 22.5, + "learning_rate": 5.2777777777777776e-08, + "logits/chosen": -2.1781129837036133, + "logits/rejected": -2.219083309173584, + "logps/chosen": -0.30154842138290405, + "logps/rejected": -0.2870495915412903, + "loss": 0.45054665207862854, + "loss/core": 0.4320584833621979, + "loss/preference_sft": 0.30154842138290405, + "loss/reference_anchor": 0.216354101896286, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.9138669967651367, + "rewards/margins": 1.0391050577163696, + "rewards/rejected": 0.8747620582580566, + "step": 20 + }, + { + "drift/chosen_abs": 0.38581329584121704, + "drift/rejected_abs": 0.1625959426164627, + "epoch": 0.02388630120625821, + "grad_norm": 26.5, + "learning_rate": 6.666666666666667e-08, + "logits/chosen": -2.0747623443603516, + "logits/rejected": -2.1344943046569824, + "logps/chosen": -0.26991984248161316, + "logps/rejected": -0.2599589228630066, + "loss": 1.819545030593872, + "loss/core": 1.7517907619476318, + "loss/preference_sft": 0.26991984248161316, + "loss/reference_anchor": 0.8763998746871948, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.8559670448303223, + "rewards/margins": 2.2310361862182617, + "rewards/rejected": 1.6249310970306396, + "step": 25 + }, + { + "drift/chosen_abs": 0.22763633728027344, + "drift/rejected_abs": 0.11158541589975357, + "epoch": 0.028663561447509853, + "grad_norm": 252.0, + "learning_rate": 8.055555555555555e-08, + "logits/chosen": -2.349606513977051, + "logits/rejected": -2.331936836242676, + "logps/chosen": -0.3003299832344055, + "logps/rejected": -0.32463088631629944, + "loss": 0.6905651092529297, + "loss/core": 0.6633950471878052, + "loss/preference_sft": 0.3003299832344055, + "loss/reference_anchor": 0.3322353661060333, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.275994300842285, + "rewards/margins": 1.2529335021972656, + "rewards/rejected": 1.0230610370635986, + "step": 30 + }, + { + "drift/chosen_abs": 0.26552093029022217, + "drift/rejected_abs": 0.11600001156330109, + "epoch": 0.033440821688761495, + "grad_norm": 2.28125, + "learning_rate": 9.444444444444444e-08, + "logits/chosen": -2.203125476837158, + "logits/rejected": -2.184936046600342, + "logps/chosen": -0.26805487275123596, + "logps/rejected": -0.2653547525405884, + "loss": 0.8213445544242859, + "loss/core": 0.7896862626075745, + "loss/preference_sft": 0.26805487275123596, + "loss/reference_anchor": 0.3953050374984741, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.6552093029022217, + "rewards/margins": 1.496461272239685, + "rewards/rejected": 1.1587477922439575, + "step": 35 + }, + { + "drift/chosen_abs": 0.44771280884742737, + "drift/rejected_abs": 0.16425152122974396, + "epoch": 0.03821808193001314, + "grad_norm": 81.0, + "learning_rate": 1.0833333333333334e-07, + "logits/chosen": -2.0018858909606934, + "logits/rejected": -2.061411142349243, + "logps/chosen": -0.2927972674369812, + "logps/rejected": -0.2809644341468811, + "loss": 2.044156789779663, + "loss/core": 1.9680871963500977, + "loss/preference_sft": 0.2927972674369812, + "loss/reference_anchor": 0.9849796295166016, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.476373672485352, + "rewards/margins": 2.8645739555358887, + "rewards/rejected": 1.6117998361587524, + "step": 40 + }, + { + "drift/chosen_abs": 0.21002939343452454, + "drift/rejected_abs": 0.08615503460168839, + "epoch": 0.04299534217126478, + "grad_norm": 8.3125, + "learning_rate": 1.2222222222222222e-07, + "logits/chosen": -2.2009475231170654, + "logits/rejected": -2.3619346618652344, + "logps/chosen": -0.2883842885494232, + "logps/rejected": -0.2879543900489807, + "loss": 0.2927166819572449, + "loss/core": 0.28003185987472534, + "loss/preference_sft": 0.2883842885494232, + "loss/reference_anchor": 0.14029237627983093, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.099714756011963, + "rewards/margins": 1.253118634223938, + "rewards/rejected": 0.8465961217880249, + "step": 45 + }, + { + "drift/chosen_abs": 0.28804001212120056, + "drift/rejected_abs": 0.14722736179828644, + "epoch": 0.04777260241251642, + "grad_norm": 75.5, + "learning_rate": 1.3611111111111108e-07, + "logits/chosen": -2.1644270420074463, + "logits/rejected": -2.1614725589752197, + "logps/chosen": -0.2816537022590637, + "logps/rejected": -0.2829938530921936, + "loss": 0.7515085339546204, + "loss/core": 0.7222825884819031, + "loss/preference_sft": 0.2816537022590637, + "loss/reference_anchor": 0.36151474714279175, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.8799638748168945, + "rewards/margins": 1.4102028608322144, + "rewards/rejected": 1.4697611331939697, + "step": 50 + }, + { + "drift/chosen_abs": 0.2258037030696869, + "drift/rejected_abs": 0.20605143904685974, + "epoch": 0.05254986265376806, + "grad_norm": 0.8984375, + "learning_rate": 1.5e-07, + "logits/chosen": -2.1963486671447754, + "logits/rejected": -2.184718608856201, + "logps/chosen": -0.28904852271080017, + "logps/rejected": -0.28756090998649597, + "loss": 1.5850727558135986, + "loss/core": 1.5256582498550415, + "loss/preference_sft": 0.28904852271080017, + "loss/reference_anchor": 0.7632866501808167, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.2579128742218018, + "rewards/margins": 0.2070862054824829, + "rewards/rejected": 2.0508265495300293, + "step": 55 + }, + { + "drift/chosen_abs": 0.21603329479694366, + "drift/rejected_abs": 0.1505412757396698, + "epoch": 0.057327122895019705, + "grad_norm": 512.0, + "learning_rate": 1.6388888888888888e-07, + "logits/chosen": -2.091665744781494, + "logits/rejected": -2.120565176010132, + "logps/chosen": -0.2742917537689209, + "logps/rejected": -0.2904832065105438, + "loss": 0.3291396200656891, + "loss/core": 0.3152475953102112, + "loss/preference_sft": 0.2742917537689209, + "loss/reference_anchor": 0.1577974259853363, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.157923698425293, + "rewards/margins": 0.668967068195343, + "rewards/rejected": 1.4889566898345947, + "step": 60 + }, + { + "drift/chosen_abs": 0.29638388752937317, + "drift/rejected_abs": 0.11892390251159668, + "epoch": 0.06210438313627135, + "grad_norm": 7.875, + "learning_rate": 1.7777777777777776e-07, + "logits/chosen": -1.9620473384857178, + "logits/rejected": -2.0643868446350098, + "logps/chosen": -0.3099573254585266, + "logps/rejected": -0.3055954873561859, + "loss": 1.0155407190322876, + "loss/core": 0.9765542149543762, + "loss/preference_sft": 0.3099573254585266, + "loss/reference_anchor": 0.4888220429420471, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.962282419204712, + "rewards/margins": 1.783328652381897, + "rewards/rejected": 1.178953766822815, + "step": 65 + }, + { + "drift/chosen_abs": 0.358979195356369, + "drift/rejected_abs": 0.22608236968517303, + "epoch": 0.06688164337752299, + "grad_norm": 56.25, + "learning_rate": 1.9166666666666668e-07, + "logits/chosen": -1.9711816310882568, + "logits/rejected": -1.9564964771270752, + "logps/chosen": -0.38483089208602905, + "logps/rejected": -0.2978837192058563, + "loss": 1.9197027683258057, + "loss/core": 1.84750235080719, + "loss/preference_sft": 0.38483089208602905, + "loss/reference_anchor": 0.9241858720779419, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.5887832641601562, + "rewards/margins": 1.4143999814987183, + "rewards/rejected": 2.1743829250335693, + "step": 70 + }, + { + "drift/chosen_abs": 0.2635686993598938, + "drift/rejected_abs": 0.15887705981731415, + "epoch": 0.07165890361877464, + "grad_norm": 8.25, + "learning_rate": 2.0555555555555553e-07, + "logits/chosen": -2.086611747741699, + "logits/rejected": -2.0510897636413574, + "logps/chosen": -0.3122991919517517, + "logps/rejected": -0.28983455896377563, + "loss": 0.7980114817619324, + "loss/core": 0.766870379447937, + "loss/preference_sft": 0.3122991919517517, + "loss/reference_anchor": 0.3839830458164215, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.6338839530944824, + "rewards/margins": 1.0476503372192383, + "rewards/rejected": 1.586233377456665, + "step": 75 + }, + { + "drift/chosen_abs": 0.32444751262664795, + "drift/rejected_abs": 0.12715654075145721, + "epoch": 0.07643616386002627, + "grad_norm": 4.15625, + "learning_rate": 2.1944444444444442e-07, + "logits/chosen": -2.069356918334961, + "logits/rejected": -2.190796375274658, + "logps/chosen": -0.26029911637306213, + "logps/rejected": -0.27222079038619995, + "loss": 0.951573371887207, + "loss/core": 0.9152650833129883, + "loss/preference_sft": 0.26029911637306213, + "loss/reference_anchor": 0.4580825865268707, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.244004726409912, + "rewards/margins": 1.9740562438964844, + "rewards/rejected": 1.2699484825134277, + "step": 80 + }, + { + "drift/chosen_abs": 0.35097089409828186, + "drift/rejected_abs": 0.13202176988124847, + "epoch": 0.08121342410127792, + "grad_norm": 6.0, + "learning_rate": 2.3333333333333333e-07, + "logits/chosen": -2.186922073364258, + "logits/rejected": -2.2361488342285156, + "logps/chosen": -0.30639296770095825, + "logps/rejected": -0.2788568139076233, + "loss": 1.790135383605957, + "loss/core": 1.7231495380401611, + "loss/preference_sft": 0.30639296770095825, + "loss/reference_anchor": 0.8625059127807617, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.509377956390381, + "rewards/margins": 2.1900832653045654, + "rewards/rejected": 1.3192951679229736, + "step": 85 + }, + { + "drift/chosen_abs": 0.17585346102714539, + "drift/rejected_abs": 0.08460784703493118, + "epoch": 0.08599068434252956, + "grad_norm": 1.0, + "learning_rate": 2.4722222222222224e-07, + "logits/chosen": -2.42102313041687, + "logits/rejected": -2.4086685180664062, + "logps/chosen": -0.2809022068977356, + "logps/rejected": -0.29607102274894714, + "loss": 0.32354456186294556, + "loss/core": 0.30980855226516724, + "loss/preference_sft": 0.2809022068977356, + "loss/reference_anchor": 0.1550559550523758, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.7583656311035156, + "rewards/margins": 0.9138404726982117, + "rewards/rejected": 0.844525158405304, + "step": 90 + }, + { + "drift/chosen_abs": 0.320295125246048, + "drift/rejected_abs": 0.21105077862739563, + "epoch": 0.09076794458378121, + "grad_norm": 67.0, + "learning_rate": 2.6111111111111113e-07, + "logits/chosen": -2.0537590980529785, + "logits/rejected": -2.1096103191375732, + "logps/chosen": -0.26810017228126526, + "logps/rejected": -0.2783377170562744, + "loss": 0.9392117261886597, + "loss/core": 0.903303325176239, + "loss/preference_sft": 0.26810017228126526, + "loss/reference_anchor": 0.45196953415870667, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.2027251720428467, + "rewards/margins": 1.0923994779586792, + "rewards/rejected": 2.110325574874878, + "step": 95 + }, + { + "drift/chosen_abs": 0.24583487212657928, + "drift/rejected_abs": 0.12835685908794403, + "epoch": 0.09554520482503284, + "grad_norm": 10.8125, + "learning_rate": 2.75e-07, + "logits/chosen": -2.234044075012207, + "logits/rejected": -2.2795557975769043, + "logps/chosen": -0.2577240467071533, + "logps/rejected": -0.24702732264995575, + "loss": 0.37607091665267944, + "loss/core": 0.3605777323246002, + "loss/preference_sft": 0.2577240467071533, + "loss/reference_anchor": 0.18080314993858337, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.4573445320129395, + "rewards/margins": 1.1746383905410767, + "rewards/rejected": 1.282706379890442, + "step": 100 + }, + { + "drift/chosen_abs": 0.19956639409065247, + "drift/rejected_abs": 0.08023403584957123, + "epoch": 0.10032246506628449, + "grad_norm": 302.0, + "learning_rate": 2.8888888888888885e-07, + "logits/chosen": -2.3368284702301025, + "logits/rejected": -2.421706438064575, + "logps/chosen": -0.2843855023384094, + "logps/rejected": -0.27989473938941956, + "loss": 0.41286808252334595, + "loss/core": 0.39585790038108826, + "loss/preference_sft": 0.2843855023384094, + "loss/reference_anchor": 0.19836413860321045, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.9953877925872803, + "rewards/margins": 1.1983610391616821, + "rewards/rejected": 0.7970272302627563, + "step": 105 + }, + { + "drift/chosen_abs": 0.44724979996681213, + "drift/rejected_abs": 0.18898524343967438, + "epoch": 0.10509972530753613, + "grad_norm": 14.5625, + "learning_rate": 3.0277777777777773e-07, + "logits/chosen": -1.9066753387451172, + "logits/rejected": -1.8599035739898682, + "logps/chosen": -0.33843153715133667, + "logps/rejected": -0.2979224920272827, + "loss": 1.8243690729141235, + "loss/core": 1.7559093236923218, + "loss/preference_sft": 0.33843153715133667, + "loss/reference_anchor": 0.8789511919021606, + "rewards/accuracies": 0.875, + "rewards/chosen": 4.470961570739746, + "rewards/margins": 2.5842795372009277, + "rewards/rejected": 1.8866825103759766, + "step": 110 + }, + { + "drift/chosen_abs": 0.17774026095867157, + "drift/rejected_abs": 0.14382174611091614, + "epoch": 0.10987698554878778, + "grad_norm": 11.9375, + "learning_rate": 3.166666666666666e-07, + "logits/chosen": -2.086132764816284, + "logits/rejected": -2.030756711959839, + "logps/chosen": -0.3001967668533325, + "logps/rejected": -0.2998013198375702, + "loss": 0.43933549523353577, + "loss/core": 0.4212765693664551, + "loss/preference_sft": 0.3001967668533325, + "loss/reference_anchor": 0.21076564490795135, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.777402639389038, + "rewards/margins": 0.34036320447921753, + "rewards/rejected": 1.4370393753051758, + "step": 115 + }, + { + "drift/chosen_abs": 0.18121027946472168, + "drift/rejected_abs": 0.11058615148067474, + "epoch": 0.11465424579003941, + "grad_norm": 180.0, + "learning_rate": 3.3055555555555556e-07, + "logits/chosen": -2.0374550819396973, + "logits/rejected": -2.0756847858428955, + "logps/chosen": -0.301984578371048, + "logps/rejected": -0.29793334007263184, + "loss": 0.1780211180448532, + "loss/core": 0.16938777267932892, + "loss/preference_sft": 0.301984578371048, + "loss/reference_anchor": 0.08491319417953491, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.8116956949234009, + "rewards/margins": 0.7075945734977722, + "rewards/rejected": 1.1041011810302734, + "step": 120 + }, + { + "drift/chosen_abs": 0.2069893628358841, + "drift/rejected_abs": 0.10128186643123627, + "epoch": 0.11943150603129106, + "grad_norm": 6.09375, + "learning_rate": 3.4444444444444444e-07, + "logits/chosen": -2.108165979385376, + "logits/rejected": -2.139435291290283, + "logps/chosen": -0.29883089661598206, + "logps/rejected": -0.3071492612361908, + "loss": 0.2796749472618103, + "loss/core": 0.2673766613006592, + "loss/preference_sft": 0.29883089661598206, + "loss/reference_anchor": 0.1340940147638321, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.067173480987549, + "rewards/margins": 1.0553032159805298, + "rewards/rejected": 1.0118701457977295, + "step": 125 + }, + { + "drift/chosen_abs": 0.18067234754562378, + "drift/rejected_abs": 0.08859051764011383, + "epoch": 0.1242087662725427, + "grad_norm": 11.25, + "learning_rate": 3.583333333333333e-07, + "logits/chosen": -1.9984080791473389, + "logits/rejected": -2.0914056301116943, + "logps/chosen": -0.28005415201187134, + "logps/rejected": -0.27907031774520874, + "loss": 0.15530189871788025, + "loss/core": 0.14763937890529633, + "loss/preference_sft": 0.28005415201187134, + "loss/reference_anchor": 0.0741613507270813, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.806658387184143, + "rewards/margins": 0.9225584268569946, + "rewards/rejected": 0.8841001391410828, + "step": 130 + }, + { + "drift/chosen_abs": 0.33673185110092163, + "drift/rejected_abs": 0.1371738612651825, + "epoch": 0.12898602651379434, + "grad_norm": 20.5, + "learning_rate": 3.722222222222222e-07, + "logits/chosen": -2.0906131267547607, + "logits/rejected": -2.16442608833313, + "logps/chosen": -0.3028051257133484, + "logps/rejected": -0.29675745964050293, + "loss": 0.9384252429008484, + "loss/core": 0.9022802114486694, + "loss/preference_sft": 0.3028051257133484, + "loss/reference_anchor": 0.4516521990299225, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.367079973220825, + "rewards/margins": 2.038417339324951, + "rewards/rejected": 1.3286627531051636, + "step": 135 + }, + { + "drift/chosen_abs": 0.1627669632434845, + "drift/rejected_abs": 0.0710747018456459, + "epoch": 0.13376328675504598, + "grad_norm": 12.1875, + "learning_rate": 3.861111111111111e-07, + "logits/chosen": -2.4494571685791016, + "logits/rejected": -2.5083508491516113, + "logps/chosen": -0.2825891971588135, + "logps/rejected": -0.2898746430873871, + "loss": 0.12142524868249893, + "loss/core": 0.11497412621974945, + "loss/preference_sft": 0.2825891971588135, + "loss/reference_anchor": 0.057755958288908005, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.6273698806762695, + "rewards/margins": 0.9166396856307983, + "rewards/rejected": 0.7107301950454712, + "step": 140 + }, + { + "drift/chosen_abs": 0.36183470487594604, + "drift/rejected_abs": 0.17905983328819275, + "epoch": 0.13854054699629761, + "grad_norm": 8.125, + "learning_rate": 4e-07, + "logits/chosen": -1.7783467769622803, + "logits/rejected": -1.8096199035644531, + "logps/chosen": -0.3000045418739319, + "logps/rejected": -0.30179673433303833, + "loss": 2.165755271911621, + "loss/core": 2.0852818489074707, + "loss/preference_sft": 0.3000045418739319, + "loss/reference_anchor": 1.0429770946502686, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.617523193359375, + "rewards/margins": 1.832786202430725, + "rewards/rejected": 1.78473699092865, + "step": 145 + }, + { + "drift/chosen_abs": 0.36336830258369446, + "drift/rejected_abs": 0.14750418066978455, + "epoch": 0.14331780723754928, + "grad_norm": 121.0, + "learning_rate": 4.1388888888888887e-07, + "logits/chosen": -2.2309694290161133, + "logits/rejected": -2.280487298965454, + "logps/chosen": -0.3181116282939911, + "logps/rejected": -0.3068601191043854, + "loss": 1.7123243808746338, + "loss/core": 1.6480745077133179, + "loss/preference_sft": 0.3181116282939911, + "loss/reference_anchor": 0.8248558044433594, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.6315560340881348, + "rewards/margins": 2.1605687141418457, + "rewards/rejected": 1.4709874391555786, + "step": 150 + }, + { + "drift/chosen_abs": 0.2860521078109741, + "drift/rejected_abs": 0.08609304577112198, + "epoch": 0.1480950674788009, + "grad_norm": 5.125, + "learning_rate": 4.2777777777777775e-07, + "logits/chosen": -2.0970890522003174, + "logits/rejected": -2.1750330924987793, + "logps/chosen": -0.28367698192596436, + "logps/rejected": -0.29151204228401184, + "loss": 0.7529366612434387, + "loss/core": 0.7236311435699463, + "loss/preference_sft": 0.28367698192596436, + "loss/reference_anchor": 0.36237236857414246, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.858731746673584, + "rewards/margins": 2.024120330810547, + "rewards/rejected": 0.8346115946769714, + "step": 155 + }, + { + "drift/chosen_abs": 0.31623023748397827, + "drift/rejected_abs": 0.13978847861289978, + "epoch": 0.15287232772005255, + "grad_norm": 282.0, + "learning_rate": 4.4166666666666664e-07, + "logits/chosen": -2.0535616874694824, + "logits/rejected": -2.027621030807495, + "logps/chosen": -0.30931371450424194, + "logps/rejected": -0.3156592547893524, + "loss": 1.4550539255142212, + "loss/core": 1.4001611471176147, + "loss/preference_sft": 0.30931371450424194, + "loss/reference_anchor": 0.7009710073471069, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.1623027324676514, + "rewards/margins": 1.764468789100647, + "rewards/rejected": 1.397834062576294, + "step": 160 + }, + { + "drift/chosen_abs": 0.16005036234855652, + "drift/rejected_abs": 0.07912009209394455, + "epoch": 0.15764958796130418, + "grad_norm": 5.1875, + "learning_rate": 4.555555555555555e-07, + "logits/chosen": -2.2203352451324463, + "logits/rejected": -2.1501989364624023, + "logps/chosen": -0.31128016114234924, + "logps/rejected": -0.3056836724281311, + "loss": 0.18936026096343994, + "loss/core": 0.18025028705596924, + "loss/preference_sft": 0.31128016114234924, + "loss/reference_anchor": 0.09033866971731186, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.60050368309021, + "rewards/margins": 0.8113292455673218, + "rewards/rejected": 0.7891744375228882, + "step": 165 + }, + { + "drift/chosen_abs": 0.26069825887680054, + "drift/rejected_abs": 0.2761599123477936, + "epoch": 0.16242684820255585, + "grad_norm": 378.0, + "learning_rate": 4.694444444444444e-07, + "logits/chosen": -2.186049461364746, + "logits/rejected": -2.1308929920196533, + "logps/chosen": -0.273814857006073, + "logps/rejected": -0.2607230842113495, + "loss": 1.9536501169204712, + "loss/core": 1.8810440301895142, + "loss/preference_sft": 0.273814857006073, + "loss/reference_anchor": 0.9407007098197937, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.6061038970947266, + "rewards/margins": -0.15422509610652924, + "rewards/rejected": 2.760329008102417, + "step": 170 + }, + { + "drift/chosen_abs": 0.25758832693099976, + "drift/rejected_abs": 0.12776905298233032, + "epoch": 0.16720410844380748, + "grad_norm": 406.0, + "learning_rate": 4.833333333333333e-07, + "logits/chosen": -2.1730058193206787, + "logits/rejected": -2.212725877761841, + "logps/chosen": -0.2877016067504883, + "logps/rejected": -0.30032774806022644, + "loss": 1.1656818389892578, + "loss/core": 1.1214388608932495, + "loss/preference_sft": 0.2877016067504883, + "loss/reference_anchor": 0.5611380338668823, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.5753438472747803, + "rewards/margins": 1.3018853664398193, + "rewards/rejected": 1.273458480834961, + "step": 175 + }, + { + "drift/chosen_abs": 0.3230380117893219, + "drift/rejected_abs": 0.150053471326828, + "epoch": 0.17198136868505912, + "grad_norm": 668.0, + "learning_rate": 4.972222222222222e-07, + "logits/chosen": -2.197150945663452, + "logits/rejected": -2.108093023300171, + "logps/chosen": -0.27259406447410583, + "logps/rejected": -0.2611132264137268, + "loss": 1.2022207975387573, + "loss/core": 1.1567691564559937, + "loss/preference_sft": 0.27259406447410583, + "loss/reference_anchor": 0.5787622332572937, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.2283949851989746, + "rewards/margins": 1.730065941810608, + "rewards/rejected": 1.4983290433883667, + "step": 180 + }, + { + "drift/chosen_abs": 0.10474558919668198, + "drift/rejected_abs": 0.06288503110408783, + "epoch": 0.17675862892631075, + "grad_norm": 3.453125, + "learning_rate": 4.999619237890978e-07, + "logits/chosen": -2.201859474182129, + "logits/rejected": -2.158825397491455, + "logps/chosen": -0.28844937682151794, + "logps/rejected": -0.2922508418560028, + "loss": 0.06655260920524597, + "loss/core": 0.06205412745475769, + "loss/preference_sft": 0.28844937682151794, + "loss/reference_anchor": 0.03113471530377865, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.0451735258102417, + "rewards/margins": 0.41988277435302734, + "rewards/rejected": 0.6252908706665039, + "step": 185 + }, + { + "drift/chosen_abs": 0.2833901643753052, + "drift/rejected_abs": 0.12482712417840958, + "epoch": 0.18153588916756241, + "grad_norm": 30.5, + "learning_rate": 4.998072590601808e-07, + "logits/chosen": -2.2913918495178223, + "logits/rejected": -2.2682085037231445, + "logps/chosen": -0.284027636051178, + "logps/rejected": -0.2837992310523987, + "loss": 1.1685082912445068, + "loss/core": 1.1242001056671143, + "loss/preference_sft": 0.284027636051178, + "loss/reference_anchor": 0.5623761415481567, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.8339016437530518, + "rewards/margins": 1.590225100517273, + "rewards/rejected": 1.2436766624450684, + "step": 190 + }, + { + "drift/chosen_abs": 0.26705488562583923, + "drift/rejected_abs": 0.0977572649717331, + "epoch": 0.18631314940881405, + "grad_norm": 23.25, + "learning_rate": 4.995336996054667e-07, + "logits/chosen": -1.982912302017212, + "logits/rejected": -2.0029821395874023, + "logps/chosen": -0.2908513844013214, + "logps/rejected": -0.2896444797515869, + "loss": 1.151555061340332, + "loss/core": 1.1077957153320312, + "loss/preference_sft": 0.2908513844013214, + "loss/reference_anchor": 0.5543731451034546, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.66994571685791, + "rewards/margins": 1.7028605937957764, + "rewards/rejected": 0.9670848846435547, + "step": 195 + }, + { + "drift/chosen_abs": 0.2615807056427002, + "drift/rejected_abs": 0.12196584045886993, + "epoch": 0.19109040965006568, + "grad_norm": 21.375, + "learning_rate": 4.991413756244404e-07, + "logits/chosen": -1.8908970355987549, + "logits/rejected": -1.9317600727081299, + "logps/chosen": -0.3235652446746826, + "logps/rejected": -0.316772997379303, + "loss": 0.5153834819793701, + "loss/core": 0.4943995475769043, + "loss/preference_sft": 0.3235652446746826, + "loss/reference_anchor": 0.2474292516708374, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.6145291328430176, + "rewards/margins": 1.3955014944076538, + "rewards/rejected": 1.2190272808074951, + "step": 200 + }, + { + "drift/chosen_abs": 0.3288207948207855, + "drift/rejected_abs": 0.2144632786512375, + "epoch": 0.19586766989131732, + "grad_norm": 10.6875, + "learning_rate": 4.986304738420683e-07, + "logits/chosen": -2.1127889156341553, + "logits/rejected": -2.164623975753784, + "logps/chosen": -0.25030291080474854, + "logps/rejected": -0.26794177293777466, + "loss": 1.3577306270599365, + "loss/core": 1.3068211078643799, + "loss/preference_sft": 0.25030291080474854, + "loss/reference_anchor": 0.6537634134292603, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.287494659423828, + "rewards/margins": 1.1692345142364502, + "rewards/rejected": 2.118259906768799, + "step": 205 + }, + { + "drift/chosen_abs": 0.2845247685909271, + "drift/rejected_abs": 0.14929194748401642, + "epoch": 0.20064493013256898, + "grad_norm": 28.625, + "learning_rate": 4.980012374199287e-07, + "logits/chosen": -2.107903003692627, + "logits/rejected": -2.131675958633423, + "logps/chosen": -0.3113544285297394, + "logps/rejected": -0.3212708830833435, + "loss": 1.2182084321975708, + "loss/core": 1.1718981266021729, + "loss/preference_sft": 0.3113544285297394, + "loss/reference_anchor": 0.5863357782363892, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.8425536155700684, + "rewards/margins": 1.352899193763733, + "rewards/rejected": 1.489654779434204, + "step": 210 + }, + { + "drift/chosen_abs": 0.31894436478614807, + "drift/rejected_abs": 0.17038756608963013, + "epoch": 0.20542219037382062, + "grad_norm": 7.25, + "learning_rate": 4.972539658404792e-07, + "logits/chosen": -2.0803780555725098, + "logits/rejected": -2.071502208709717, + "logps/chosen": -0.27592846751213074, + "logps/rejected": -0.28498896956443787, + "loss": 1.6364405155181885, + "loss/core": 1.5752842426300049, + "loss/preference_sft": 0.27592846751213074, + "loss/reference_anchor": 0.78782057762146, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.186903953552246, + "rewards/margins": 1.491759181022644, + "rewards/rejected": 1.6951450109481812, + "step": 215 + }, + { + "drift/chosen_abs": 0.1886764019727707, + "drift/rejected_abs": 0.10738885402679443, + "epoch": 0.21019945061507225, + "grad_norm": 34.25, + "learning_rate": 4.963890147645194e-07, + "logits/chosen": -2.0927789211273193, + "logits/rejected": -2.0071330070495605, + "logps/chosen": -0.3026772439479828, + "logps/rejected": -0.3015129268169403, + "loss": 0.2290460765361786, + "loss/core": 0.21856117248535156, + "loss/preference_sft": 0.3026772439479828, + "loss/reference_anchor": 0.10953140258789062, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.8838809728622437, + "rewards/margins": 0.8116692304611206, + "rewards/rejected": 1.072211503982544, + "step": 220 + }, + { + "drift/chosen_abs": 0.3618369996547699, + "drift/rejected_abs": 0.1408586949110031, + "epoch": 0.2149767108563239, + "grad_norm": 8.875, + "learning_rate": 4.95406795861916e-07, + "logits/chosen": -2.1790313720703125, + "logits/rejected": -2.1682028770446777, + "logps/chosen": -0.2909870743751526, + "logps/rejected": -0.2756298780441284, + "loss": 0.8731632232666016, + "loss/core": 0.8394603729248047, + "loss/preference_sft": 0.2909870743751526, + "loss/reference_anchor": 0.4202722907066345, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.6181743144989014, + "rewards/margins": 2.2104671001434326, + "rewards/rejected": 1.4077072143554688, + "step": 225 + }, + { + "drift/chosen_abs": 0.18520915508270264, + "drift/rejected_abs": 0.07412123680114746, + "epoch": 0.21975397109757555, + "grad_norm": 74.0, + "learning_rate": 4.943077766156697e-07, + "logits/chosen": -2.2688801288604736, + "logits/rejected": -2.276559829711914, + "logps/chosen": -0.2821432948112488, + "logps/rejected": -0.30950599908828735, + "loss": 0.19297373294830322, + "loss/core": 0.18392691016197205, + "loss/preference_sft": 0.2821432948112488, + "loss/reference_anchor": 0.09240987151861191, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8502082824707031, + "rewards/margins": 1.1134107112884521, + "rewards/rejected": 0.7367974519729614, + "step": 230 + }, + { + "drift/chosen_abs": 0.1905185729265213, + "drift/rejected_abs": 0.13251249492168427, + "epoch": 0.2245312313388272, + "grad_norm": 23.625, + "learning_rate": 4.930924800994191e-07, + "logits/chosen": -2.005171298980713, + "logits/rejected": -2.0407776832580566, + "logps/chosen": -0.29280489683151245, + "logps/rejected": -0.28741219639778137, + "loss": 0.4691334366798401, + "loss/core": 0.45003873109817505, + "loss/preference_sft": 0.29280489683151245, + "loss/reference_anchor": 0.2253153771162033, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.9031035900115967, + "rewards/margins": 0.58293217420578, + "rewards/rejected": 1.3201711177825928, + "step": 235 + }, + { + "drift/chosen_abs": 0.17637206614017487, + "drift/rejected_abs": 0.10210146754980087, + "epoch": 0.22930849158007882, + "grad_norm": 9.8125, + "learning_rate": 4.917614847284858e-07, + "logits/chosen": -2.019641399383545, + "logits/rejected": -2.0390303134918213, + "logps/chosen": -0.294040709733963, + "logps/rejected": -0.2948092818260193, + "loss": 0.14991113543510437, + "loss/core": 0.14234814047813416, + "loss/preference_sft": 0.294040709733963, + "loss/reference_anchor": 0.0714358538389206, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.762891411781311, + "rewards/margins": 0.7431442141532898, + "rewards/rejected": 1.019747257232666, + "step": 240 + }, + { + "drift/chosen_abs": 0.3610271215438843, + "drift/rejected_abs": 0.11232628673315048, + "epoch": 0.23408575182133046, + "grad_norm": 304.0, + "learning_rate": 4.903154239845797e-07, + "logits/chosen": -2.057809829711914, + "logits/rejected": -2.0461342334747314, + "logps/chosen": -0.2769336998462677, + "logps/rejected": -0.29468828439712524, + "loss": 1.2440202236175537, + "loss/core": 1.1970118284225464, + "loss/preference_sft": 0.2769336998462677, + "loss/reference_anchor": 0.5990844964981079, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.610271453857422, + "rewards/margins": 2.4941940307617188, + "rewards/rejected": 1.1160774230957031, + "step": 245 + }, + { + "drift/chosen_abs": 0.2345919907093048, + "drift/rejected_abs": 0.10711641609668732, + "epoch": 0.23886301206258212, + "grad_norm": 21.625, + "learning_rate": 4.887549861142967e-07, + "logits/chosen": -1.89980149269104, + "logits/rejected": -1.978960633277893, + "logps/chosen": -0.2994912564754486, + "logps/rejected": -0.29508280754089355, + "loss": 0.3056420385837555, + "loss/core": 0.2923932373523712, + "loss/preference_sft": 0.2994912564754486, + "loss/reference_anchor": 0.14670124650001526, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.342444896697998, + "rewards/margins": 1.2790577411651611, + "rewards/rejected": 1.0633875131607056, + "step": 250 + }, + { + "drift/chosen_abs": 0.244302898645401, + "drift/rejected_abs": 0.1597694456577301, + "epoch": 0.24364027230383375, + "grad_norm": 13.75, + "learning_rate": 4.870809138015498e-07, + "logits/chosen": -2.2113137245178223, + "logits/rejected": -2.206145763397217, + "logps/chosen": -0.2858472466468811, + "logps/rejected": -0.28811052441596985, + "loss": 0.6415420174598694, + "loss/core": 0.6162611246109009, + "loss/preference_sft": 0.2858472466468811, + "loss/reference_anchor": 0.3084937036037445, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.442288398742676, + "rewards/margins": 0.8456929922103882, + "rewards/rejected": 1.5965955257415771, + "step": 255 + }, + { + "drift/chosen_abs": 0.23896794021129608, + "drift/rejected_abs": 0.15801241993904114, + "epoch": 0.2484175325450854, + "grad_norm": 0.8046875, + "learning_rate": 4.852940038140927e-07, + "logits/chosen": -2.1472392082214355, + "logits/rejected": -2.219313859939575, + "logps/chosen": -0.2888643741607666, + "logps/rejected": -0.27107033133506775, + "loss": 0.5575072765350342, + "loss/core": 0.5352469682693481, + "loss/preference_sft": 0.2888643741607666, + "loss/reference_anchor": 0.2679183781147003, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.3884191513061523, + "rewards/margins": 0.8117639422416687, + "rewards/rejected": 1.5766551494598389, + "step": 260 + }, + { + "drift/chosen_abs": 0.1222555860877037, + "drift/rejected_abs": 0.11597619950771332, + "epoch": 0.25319479278633705, + "grad_norm": 1.203125, + "learning_rate": 4.833951066243004e-07, + "logits/chosen": -2.0287933349609375, + "logits/rejected": -2.036017656326294, + "logps/chosen": -0.3003501892089844, + "logps/rejected": -0.3035464286804199, + "loss": 0.44168415665626526, + "loss/core": 0.42354241013526917, + "loss/preference_sft": 0.3003501892089844, + "loss/reference_anchor": 0.21185505390167236, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.2217209339141846, + "rewards/margins": 0.06751598417758942, + "rewards/rejected": 1.154205083847046, + "step": 265 + }, + { + "drift/chosen_abs": 0.3274751305580139, + "drift/rejected_abs": 0.20972244441509247, + "epoch": 0.2579720530275887, + "grad_norm": 13.0, + "learning_rate": 4.813851260043915e-07, + "logits/chosen": -1.9322869777679443, + "logits/rejected": -2.0220537185668945, + "logps/chosen": -0.2892109155654907, + "logps/rejected": -0.2937650978565216, + "loss": 1.1844274997711182, + "loss/core": 1.1395008563995361, + "loss/preference_sft": 0.2892109155654907, + "loss/reference_anchor": 0.5701011419296265, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.274138927459717, + "rewards/margins": 1.1782522201538086, + "rewards/rejected": 2.0958871841430664, + "step": 270 + }, + { + "drift/chosen_abs": 0.23182424902915955, + "drift/rejected_abs": 0.12205634266138077, + "epoch": 0.2627493132688403, + "grad_norm": 35.0, + "learning_rate": 4.79265018596281e-07, + "logits/chosen": -2.081444501876831, + "logits/rejected": -2.0351366996765137, + "logps/chosen": -0.2973952293395996, + "logps/rejected": -0.30073636770248413, + "loss": 0.4742198586463928, + "loss/core": 0.45490771532058716, + "loss/preference_sft": 0.2973952293395996, + "loss/reference_anchor": 0.22775478661060333, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.3158555030822754, + "rewards/margins": 1.159383773803711, + "rewards/rejected": 1.1564717292785645, + "step": 275 + }, + { + "drift/chosen_abs": 0.20490439236164093, + "drift/rejected_abs": 0.08539918065071106, + "epoch": 0.26752657351009196, + "grad_norm": 4.1875, + "learning_rate": 4.770357934562704e-07, + "logits/chosen": -2.0511467456817627, + "logits/rejected": -1.9675155878067017, + "logps/chosen": -0.2882925868034363, + "logps/rejected": -0.3074631094932556, + "loss": 0.22447428107261658, + "loss/core": 0.21425719559192657, + "loss/preference_sft": 0.2882925868034363, + "loss/reference_anchor": 0.1073986068367958, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.044360876083374, + "rewards/margins": 1.2838475704193115, + "rewards/rejected": 0.7605134844779968, + "step": 280 + }, + { + "drift/chosen_abs": 0.2618315815925598, + "drift/rejected_abs": 0.1607695072889328, + "epoch": 0.2723038337513436, + "grad_norm": 245.0, + "learning_rate": 4.746985115747917e-07, + "logits/chosen": -2.0746307373046875, + "logits/rejected": -2.0701193809509277, + "logps/chosen": -0.2960381507873535, + "logps/rejected": -0.2952577471733093, + "loss": 0.5513629913330078, + "loss/core": 0.529272198677063, + "loss/preference_sft": 0.2960381507873535, + "loss/reference_anchor": 0.2649388313293457, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.616506814956665, + "rewards/margins": 1.0125617980957031, + "rewards/rejected": 1.6039451360702515, + "step": 285 + }, + { + "drift/chosen_abs": 0.14695926010608673, + "drift/rejected_abs": 0.14160466194152832, + "epoch": 0.27708109399259523, + "grad_norm": 676.0, + "learning_rate": 4.722542853714341e-07, + "logits/chosen": -2.154064655303955, + "logits/rejected": -2.165696144104004, + "logps/chosen": -0.28789079189300537, + "logps/rejected": -0.29165560007095337, + "loss": 0.3904039263725281, + "loss/core": 0.3742069602012634, + "loss/preference_sft": 0.28789079189300537, + "loss/reference_anchor": 0.18717019259929657, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.4682142734527588, + "rewards/margins": 0.05419105291366577, + "rewards/rejected": 1.4140231609344482, + "step": 290 + }, + { + "drift/chosen_abs": 0.13077983260154724, + "drift/rejected_abs": 0.06749530881643295, + "epoch": 0.28185835423384686, + "grad_norm": 8.625, + "learning_rate": 4.697042781654913e-07, + "logits/chosen": -2.5517144203186035, + "logits/rejected": -2.51605486869812, + "logps/chosen": -0.24481797218322754, + "logps/rejected": -0.2569727301597595, + "loss": 0.06375555694103241, + "loss/core": 0.05966717004776001, + "loss/preference_sft": 0.24481797218322754, + "loss/reference_anchor": 0.03002994880080223, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.307100534439087, + "rewards/margins": 0.6367514133453369, + "rewards/rejected": 0.6703491806983948, + "step": 295 + }, + { + "drift/chosen_abs": 0.3725462555885315, + "drift/rejected_abs": 0.10222160816192627, + "epoch": 0.28663561447509855, + "grad_norm": 105.5, + "learning_rate": 4.670497036222856e-07, + "logits/chosen": -1.96623957157135, + "logits/rejected": -1.934470772743225, + "logps/chosen": -0.3124147355556488, + "logps/rejected": -0.32228654623031616, + "loss": 1.024113416671753, + "loss/core": 0.9847720861434937, + "loss/preference_sft": 0.3124147355556488, + "loss/reference_anchor": 0.49330902099609375, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.7237000465393066, + "rewards/margins": 2.717276096343994, + "rewards/rejected": 1.006424069404602, + "step": 300 + }, + { + "drift/chosen_abs": 0.21607303619384766, + "drift/rejected_abs": 0.08962901681661606, + "epoch": 0.2914128747163502, + "grad_norm": 13.0, + "learning_rate": 4.642918251755281e-07, + "logits/chosen": -2.1090660095214844, + "logits/rejected": -2.171133279800415, + "logps/chosen": -0.2694811224937439, + "logps/rejected": -0.26096317172050476, + "loss": 0.23768694698810577, + "loss/core": 0.2271147072315216, + "loss/preference_sft": 0.2694811224937439, + "loss/reference_anchor": 0.11401481926441193, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.1587884426116943, + "rewards/margins": 1.2635397911071777, + "rewards/rejected": 0.895248532295227, + "step": 305 + }, + { + "drift/chosen_abs": 0.2609344720840454, + "drift/rejected_abs": 0.2091202735900879, + "epoch": 0.2961901349576018, + "grad_norm": 83.0, + "learning_rate": 4.614319554259933e-07, + "logits/chosen": -2.1155569553375244, + "logits/rejected": -2.0249547958374023, + "logps/chosen": -0.2733913064002991, + "logps/rejected": -0.2829728126525879, + "loss": 0.948021411895752, + "loss/core": 0.9117604494094849, + "loss/preference_sft": 0.2733913064002991, + "loss/reference_anchor": 0.45614108443260193, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.6087353229522705, + "rewards/margins": 0.6165834665298462, + "rewards/rejected": 1.9921518564224243, + "step": 310 + }, + { + "drift/chosen_abs": 0.2914541959762573, + "drift/rejected_abs": 0.13409768044948578, + "epoch": 0.30096739519885346, + "grad_norm": 5.875, + "learning_rate": 4.58471455516792e-07, + "logits/chosen": -1.945861577987671, + "logits/rejected": -2.051717519760132, + "logps/chosen": -0.26913851499557495, + "logps/rejected": -0.27637916803359985, + "loss": 0.48234397172927856, + "loss/core": 0.46292680501937866, + "loss/preference_sft": 0.26913851499557495, + "loss/reference_anchor": 0.23198194801807404, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.9132609367370605, + "rewards/margins": 1.5744664669036865, + "rewards/rejected": 1.3387947082519531, + "step": 315 + }, + { + "drift/chosen_abs": 0.15800070762634277, + "drift/rejected_abs": 0.12147839367389679, + "epoch": 0.3057446554401051, + "grad_norm": 3.5625, + "learning_rate": 4.5541173448554095e-07, + "logits/chosen": -1.986737847328186, + "logits/rejected": -2.0025219917297363, + "logps/chosen": -0.29054489731788635, + "logps/rejected": -0.29405659437179565, + "loss": 0.2033577710390091, + "loss/core": 0.19389116764068604, + "loss/preference_sft": 0.29054489731788635, + "loss/reference_anchor": 0.09716695547103882, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.5768026113510132, + "rewards/margins": 0.3625461161136627, + "rewards/rejected": 1.2142564058303833, + "step": 320 + }, + { + "drift/chosen_abs": 0.37172552943229675, + "drift/rejected_abs": 0.15326622128486633, + "epoch": 0.31052191568135673, + "grad_norm": 548.0, + "learning_rate": 4.5225424859373684e-07, + "logits/chosen": -2.2413594722747803, + "logits/rejected": -2.1880881786346436, + "logps/chosen": -0.28455644845962524, + "logps/rejected": -0.2839685380458832, + "loss": 1.9195263385772705, + "loss/core": 1.848060965538025, + "loss/preference_sft": 0.28455644845962524, + "loss/reference_anchor": 0.9244130849838257, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.716318130493164, + "rewards/margins": 2.1848549842834473, + "rewards/rejected": 1.5314635038375854, + "step": 325 + }, + { + "drift/chosen_abs": 0.20601582527160645, + "drift/rejected_abs": 0.11289147287607193, + "epoch": 0.31529917592260837, + "grad_norm": 2.765625, + "learning_rate": 4.4900050063365547e-07, + "logits/chosen": -2.2116036415100098, + "logits/rejected": -2.1998820304870605, + "logps/chosen": -0.27572888135910034, + "logps/rejected": -0.2708529829978943, + "loss": 0.31742313504219055, + "loss/core": 0.30393102765083313, + "loss/preference_sft": 0.27572888135910034, + "loss/reference_anchor": 0.15232183039188385, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.0594685077667236, + "rewards/margins": 0.9315606951713562, + "rewards/rejected": 1.1279077529907227, + "step": 330 + }, + { + "drift/chosen_abs": 0.21230271458625793, + "drift/rejected_abs": 0.08433900028467178, + "epoch": 0.32007643616386, + "grad_norm": 320.0, + "learning_rate": 4.4565203921310344e-07, + "logits/chosen": -2.03387188911438, + "logits/rejected": -2.0167250633239746, + "logps/chosen": -0.3023034930229187, + "logps/rejected": -0.3277498781681061, + "loss": 0.46993932127952576, + "loss/core": 0.45074281096458435, + "loss/preference_sft": 0.3023034930229187, + "loss/reference_anchor": 0.22572243213653564, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.119868755340576, + "rewards/margins": 1.523357629776001, + "rewards/rejected": 0.5965113043785095, + "step": 335 + }, + { + "drift/chosen_abs": 0.20402033627033234, + "drift/rejected_abs": 0.08628232777118683, + "epoch": 0.3248536964051117, + "grad_norm": 154.0, + "learning_rate": 4.422104580183649e-07, + "logits/chosen": -2.082024335861206, + "logits/rejected": -2.2043795585632324, + "logps/chosen": -0.28011229634284973, + "logps/rejected": -0.28280124068260193, + "loss": 0.32376912236213684, + "loss/core": 0.3100174069404602, + "loss/preference_sft": 0.28011229634284973, + "loss/reference_anchor": 0.15534545481204987, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.039736032485962, + "rewards/margins": 1.180772066116333, + "rewards/rejected": 0.8589638471603394, + "step": 340 + }, + { + "drift/chosen_abs": 0.16444484889507294, + "drift/rejected_abs": 0.08357802778482437, + "epoch": 0.3296309566463633, + "grad_norm": 8.875, + "learning_rate": 4.3867739505569303e-07, + "logits/chosen": -2.301851987838745, + "logits/rejected": -2.362597942352295, + "logps/chosen": -0.2823006212711334, + "logps/rejected": -0.2937981188297272, + "loss": 0.2071170061826706, + "loss/core": 0.19756881892681122, + "loss/preference_sft": 0.2823006212711334, + "loss/reference_anchor": 0.09907897561788559, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.6424602270126343, + "rewards/margins": 0.8107796907424927, + "rewards/rejected": 0.8316806554794312, + "step": 345 + }, + { + "drift/chosen_abs": 0.19968092441558838, + "drift/rejected_abs": 0.09401963651180267, + "epoch": 0.33440821688761496, + "grad_norm": 7.9375, + "learning_rate": 4.35054531871708e-07, + "logits/chosen": -2.045243978500366, + "logits/rejected": -2.024528980255127, + "logps/chosen": -0.26065462827682495, + "logps/rejected": -0.2573546767234802, + "loss": 0.21272404491901398, + "loss/core": 0.2031305581331253, + "loss/preference_sft": 0.26065462827682495, + "loss/reference_anchor": 0.10184787213802338, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.995410680770874, + "rewards/margins": 1.057207703590393, + "rewards/rejected": 0.9382032155990601, + "step": 350 + }, + { + "drift/chosen_abs": 0.1851430982351303, + "drift/rejected_abs": 0.1371186524629593, + "epoch": 0.3391854771288666, + "grad_norm": 9.6875, + "learning_rate": 4.3134359275307185e-07, + "logits/chosen": -2.0893471240997314, + "logits/rejected": -2.2154877185821533, + "logps/chosen": -0.2807328701019287, + "logps/rejected": -0.29229921102523804, + "loss": 0.4610310196876526, + "loss/core": 0.4423205256462097, + "loss/preference_sft": 0.2807328701019287, + "loss/reference_anchor": 0.22139939665794373, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.850562334060669, + "rewards/margins": 0.49644041061401367, + "rewards/rejected": 1.3541219234466553, + "step": 355 + }, + { + "drift/chosen_abs": 0.2644307017326355, + "drift/rejected_abs": 0.12655285000801086, + "epoch": 0.34396273737011823, + "grad_norm": 140.0, + "learning_rate": 4.2754634390582133e-07, + "logits/chosen": -2.113999605178833, + "logits/rejected": -2.1762969493865967, + "logps/chosen": -0.2791525721549988, + "logps/rejected": -0.28297150135040283, + "loss": 0.6517452001571655, + "loss/core": 0.626146674156189, + "loss/preference_sft": 0.2791525721549988, + "loss/reference_anchor": 0.31339868903160095, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.643383026123047, + "rewards/margins": 1.3792022466659546, + "rewards/rejected": 1.2641807794570923, + "step": 360 + }, + { + "drift/chosen_abs": 0.28687232732772827, + "drift/rejected_abs": 0.07955290377140045, + "epoch": 0.34873999761136987, + "grad_norm": 470.0, + "learning_rate": 4.236645926147493e-07, + "logits/chosen": -2.053063154220581, + "logits/rejected": -2.159447193145752, + "logps/chosen": -0.2858540415763855, + "logps/rejected": -0.28544557094573975, + "loss": 1.0082862377166748, + "loss/core": 0.9697242975234985, + "loss/preference_sft": 0.2858540415763855, + "loss/reference_anchor": 0.4855729043483734, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.865591049194336, + "rewards/margins": 2.0717544555664062, + "rewards/rejected": 0.793836772441864, + "step": 365 + }, + { + "drift/chosen_abs": 0.16405682265758514, + "drift/rejected_abs": 0.1052694097161293, + "epoch": 0.3535172578526215, + "grad_norm": 22.375, + "learning_rate": 4.1970018638323547e-07, + "logits/chosen": -2.2507824897766113, + "logits/rejected": -2.2962427139282227, + "logps/chosen": -0.28528791666030884, + "logps/rejected": -0.28648024797439575, + "loss": 0.18379460275173187, + "loss/core": 0.17507164180278778, + "loss/preference_sft": 0.28528791666030884, + "loss/reference_anchor": 0.08777736127376556, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.639823317527771, + "rewards/margins": 0.5898867249488831, + "rewards/rejected": 1.0499365329742432, + "step": 370 + }, + { + "drift/chosen_abs": 0.30081304907798767, + "drift/rejected_abs": 0.22256799042224884, + "epoch": 0.35829451809387314, + "grad_norm": 8.8125, + "learning_rate": 4.1565501205393436e-07, + "logits/chosen": -2.0259885787963867, + "logits/rejected": -2.004533290863037, + "logps/chosen": -0.2593756318092346, + "logps/rejected": -0.2595575451850891, + "loss": 0.8363986015319824, + "loss/core": 0.8042640686035156, + "loss/preference_sft": 0.2593756318092346, + "loss/reference_anchor": 0.4025222659111023, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.007532835006714, + "rewards/margins": 0.8016997575759888, + "rewards/rejected": 2.2058329582214355, + "step": 375 + }, + { + "drift/chosen_abs": 0.2039259374141693, + "drift/rejected_abs": 0.16159279644489288, + "epoch": 0.36307177833512483, + "grad_norm": 17.125, + "learning_rate": 4.1153099491074093e-07, + "logits/chosen": -2.1002986431121826, + "logits/rejected": -2.1101698875427246, + "logps/chosen": -0.28150156140327454, + "logps/rejected": -0.29337161779403687, + "loss": 0.7919434905052185, + "loss/core": 0.7612730264663696, + "loss/preference_sft": 0.28150156140327454, + "loss/reference_anchor": 0.3807896077632904, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.036198854446411, + "rewards/margins": 0.47472089529037476, + "rewards/rejected": 1.5614778995513916, + "step": 380 + }, + { + "drift/chosen_abs": 0.21656015515327454, + "drift/rejected_abs": 0.1512974351644516, + "epoch": 0.36784903857637646, + "grad_norm": 30.0, + "learning_rate": 4.0733009776245937e-07, + "logits/chosen": -2.0933432579040527, + "logits/rejected": -2.113175630569458, + "logps/chosen": -0.27162864804267883, + "logps/rejected": -0.2761984169483185, + "loss": 0.38956135511398315, + "loss/core": 0.37349286675453186, + "loss/preference_sft": 0.27162864804267883, + "loss/reference_anchor": 0.18708345293998718, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.1639719009399414, + "rewards/margins": 0.6549761295318604, + "rewards/rejected": 1.5089956521987915, + "step": 385 + }, + { + "drift/chosen_abs": 0.17612993717193604, + "drift/rejected_abs": 0.08776899427175522, + "epoch": 0.3726262988176281, + "grad_norm": 2.015625, + "learning_rate": 4.0305432000861226e-07, + "logits/chosen": -1.8524726629257202, + "logits/rejected": -1.9306892156600952, + "logps/chosen": -0.3076761066913605, + "logps/rejected": -0.3044106662273407, + "loss": 0.21172335743904114, + "loss/core": 0.20182469487190247, + "loss/preference_sft": 0.3076761066913605, + "loss/reference_anchor": 0.10121461004018784, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.7569738626480103, + "rewards/margins": 0.8802940249443054, + "rewards/rejected": 0.8766800165176392, + "step": 390 + }, + { + "drift/chosen_abs": 0.16845259070396423, + "drift/rejected_abs": 0.07968206703662872, + "epoch": 0.37740355905887973, + "grad_norm": 18.375, + "learning_rate": 3.9870569668783533e-07, + "logits/chosen": -2.2020130157470703, + "logits/rejected": -2.3044018745422363, + "logps/chosen": -0.2947576642036438, + "logps/rejected": -0.2907804548740387, + "loss": 0.1688641607761383, + "loss/core": 0.1606094241142273, + "loss/preference_sft": 0.2947576642036438, + "loss/reference_anchor": 0.08058713376522064, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 1.6845258474349976, + "rewards/margins": 0.8892566561698914, + "rewards/rejected": 0.7952693104743958, + "step": 395 + }, + { + "drift/chosen_abs": 0.2936556935310364, + "drift/rejected_abs": 0.08415323495864868, + "epoch": 0.38218081930013137, + "grad_norm": 53.25, + "learning_rate": 3.942862975093084e-07, + "logits/chosen": -2.0824859142303467, + "logits/rejected": -2.1172521114349365, + "logps/chosen": -0.2918018698692322, + "logps/rejected": -0.2934037148952484, + "loss": 1.0916287899017334, + "loss/core": 1.0500308275222778, + "loss/preference_sft": 0.2918018698692322, + "loss/reference_anchor": 0.5254592299461365, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.9353275299072266, + "rewards/margins": 2.0966556072235107, + "rewards/rejected": 0.8386720418930054, + "step": 400 + }, + { + "drift/chosen_abs": 0.23455221951007843, + "drift/rejected_abs": 0.12242819368839264, + "epoch": 0.386958079541383, + "grad_norm": 6.0, + "learning_rate": 3.8979822586768666e-07, + "logits/chosen": -2.113771677017212, + "logits/rejected": -2.168362617492676, + "logps/chosen": -0.3119937777519226, + "logps/rejected": -0.30502599477767944, + "loss": 0.7244704961776733, + "loss/core": 0.69599449634552, + "loss/preference_sft": 0.3119937777519226, + "loss/reference_anchor": 0.3484812080860138, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.345109224319458, + "rewards/margins": 1.12223219871521, + "rewards/rejected": 1.222877025604248, + "step": 405 + }, + { + "drift/chosen_abs": 0.20881287753582, + "drift/rejected_abs": 0.11809121072292328, + "epoch": 0.39173533978263464, + "grad_norm": 21.875, + "learning_rate": 3.8524361784199847e-07, + "logits/chosen": -2.228954315185547, + "logits/rejected": -2.2588367462158203, + "logps/chosen": -0.2771432101726532, + "logps/rejected": -0.28896427154541016, + "loss": 0.2878721058368683, + "loss/core": 0.27545636892318726, + "loss/preference_sft": 0.2771432101726532, + "loss/reference_anchor": 0.13782820105552673, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.0866684913635254, + "rewards/margins": 0.9072427749633789, + "rewards/rejected": 1.179425597190857, + "step": 410 + }, + { + "drift/chosen_abs": 0.09865269809961319, + "drift/rejected_abs": 0.0373404435813427, + "epoch": 0.3965126000238863, + "grad_norm": 13.25, + "learning_rate": 3.806246411789872e-07, + "logits/chosen": -2.0978753566741943, + "logits/rejected": -2.176331043243408, + "logps/chosen": -0.33135882019996643, + "logps/rejected": -0.32153475284576416, + "loss": 0.06347070634365082, + "loss/core": 0.0587652213871479, + "loss/preference_sft": 0.33135882019996643, + "loss/reference_anchor": 0.029603928327560425, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 0.9844989776611328, + "rewards/margins": 0.61436527967453, + "rewards/rejected": 0.37013378739356995, + "step": 415 + }, + { + "drift/chosen_abs": 0.20410975813865662, + "drift/rejected_abs": 0.07133093476295471, + "epoch": 0.40128986026513797, + "grad_norm": 92.5, + "learning_rate": 3.7594349426138154e-07, + "logits/chosen": -2.3295199871063232, + "logits/rejected": -2.2967400550842285, + "logps/chosen": -0.28380584716796875, + "logps/rejected": -0.2998279929161072, + "loss": 0.35089415311813354, + "loss/core": 0.3361276388168335, + "loss/preference_sft": 0.28380584716796875, + "loss/reference_anchor": 0.16850566864013672, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.040860176086426, + "rewards/margins": 1.3300426006317139, + "rewards/rejected": 0.7108174562454224, + "step": 420 + }, + { + "drift/chosen_abs": 0.22340837121009827, + "drift/rejected_abs": 0.16072794795036316, + "epoch": 0.4060671205063896, + "grad_norm": 186.0, + "learning_rate": 3.712024050615843e-07, + "logits/chosen": -1.9938865900039673, + "logits/rejected": -2.0516488552093506, + "logps/chosen": -0.2920078635215759, + "logps/rejected": -0.2692197859287262, + "loss": 0.8027600049972534, + "loss/core": 0.7715979814529419, + "loss/preference_sft": 0.2920078635215759, + "loss/reference_anchor": 0.38629359006881714, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.2311575412750244, + "rewards/margins": 0.626264214515686, + "rewards/rejected": 1.6048933267593384, + "step": 425 + }, + { + "drift/chosen_abs": 0.4396797716617584, + "drift/rejected_abs": 0.1412547528743744, + "epoch": 0.41084438074764124, + "grad_norm": 696.0, + "learning_rate": 3.664036300812778e-07, + "logits/chosen": -2.0478415489196777, + "logits/rejected": -2.0716514587402344, + "logps/chosen": -0.2803536057472229, + "logps/rejected": -0.26729705929756165, + "loss": 1.897403359413147, + "loss/core": 1.8267196416854858, + "loss/preference_sft": 0.2803536057472229, + "loss/reference_anchor": 0.914415717124939, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.394786834716797, + "rewards/margins": 2.983668804168701, + "rewards/rejected": 1.4111182689666748, + "step": 430 + }, + { + "drift/chosen_abs": 0.1620672643184662, + "drift/rejected_abs": 0.10048636049032211, + "epoch": 0.41562164098889287, + "grad_norm": 8.6875, + "learning_rate": 3.615494532774522e-07, + "logits/chosen": -2.3244786262512207, + "logits/rejected": -2.30962872505188, + "logps/chosen": -0.27878594398498535, + "logps/rejected": -0.2934947609901428, + "loss": 0.2484576255083084, + "loss/core": 0.23745962977409363, + "loss/preference_sft": 0.27878594398498535, + "loss/reference_anchor": 0.11876034736633301, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.6202609539031982, + "rewards/margins": 0.6176348328590393, + "rewards/rejected": 1.0026261806488037, + "step": 435 + }, + { + "drift/chosen_abs": 0.29970285296440125, + "drift/rejected_abs": 0.13486061990261078, + "epoch": 0.4203989012301445, + "grad_norm": 490.0, + "learning_rate": 3.5664218497536456e-07, + "logits/chosen": -2.0376389026641846, + "logits/rejected": -2.108769416809082, + "logps/chosen": -0.304705411195755, + "logps/rejected": -0.3040980398654938, + "loss": 1.0282645225524902, + "loss/core": 0.9888750314712524, + "loss/preference_sft": 0.304705411195755, + "loss/reference_anchor": 0.4947218894958496, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.992257595062256, + "rewards/margins": 1.6467006206512451, + "rewards/rejected": 1.3455569744110107, + "step": 440 + }, + { + "drift/chosen_abs": 0.2948855757713318, + "drift/rejected_abs": 0.1586662232875824, + "epoch": 0.42517616147139614, + "grad_norm": 40.0, + "learning_rate": 3.516841607689501e-07, + "logits/chosen": -1.9140384197235107, + "logits/rejected": -1.8901231288909912, + "logps/chosen": -0.29691654443740845, + "logps/rejected": -0.2970215678215027, + "loss": 0.5381649732589722, + "loss/core": 0.5165340900421143, + "loss/preference_sft": 0.29691654443740845, + "loss/reference_anchor": 0.2587199807167053, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.9474780559539795, + "rewards/margins": 1.3833874464035034, + "rewards/rejected": 1.5640907287597656, + "step": 445 + }, + { + "drift/chosen_abs": 0.3455376923084259, + "drift/rejected_abs": 0.15802158415317535, + "epoch": 0.4299534217126478, + "grad_norm": 426.0, + "learning_rate": 3.4667774040920515e-07, + "logits/chosen": -1.9539282321929932, + "logits/rejected": -1.9721944332122803, + "logps/chosen": -0.29860302805900574, + "logps/rejected": -0.29259243607521057, + "loss": 0.9283429980278015, + "loss/core": 0.8925935626029968, + "loss/preference_sft": 0.29860302805900574, + "loss/reference_anchor": 0.4467983841896057, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.454946994781494, + "rewards/margins": 1.8800350427627563, + "rewards/rejected": 1.5749118328094482, + "step": 450 + }, + { + "drift/chosen_abs": 0.3471214175224304, + "drift/rejected_abs": 0.14864817261695862, + "epoch": 0.4347306819538994, + "grad_norm": 200.0, + "learning_rate": 3.416253066810743e-07, + "logits/chosen": -1.918105125427246, + "logits/rejected": -2.1081409454345703, + "logps/chosen": -0.3223501741886139, + "logps/rejected": -0.31352201104164124, + "loss": 1.5264394283294678, + "loss/core": 1.4688888788223267, + "loss/preference_sft": 0.3223501741886139, + "loss/reference_anchor": 0.7351065278053284, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.4693026542663574, + "rewards/margins": 1.9863710403442383, + "rewards/rejected": 1.4829317331314087, + "step": 455 + }, + { + "drift/chosen_abs": 0.36486533284187317, + "drift/rejected_abs": 0.1869690865278244, + "epoch": 0.4395079421951511, + "grad_norm": 33.75, + "learning_rate": 3.3652926426937325e-07, + "logits/chosen": -2.0386669635772705, + "logits/rejected": -2.027118444442749, + "logps/chosen": -0.29761844873428345, + "logps/rejected": -0.3065989017486572, + "loss": 1.2015959024429321, + "loss/core": 1.155981421470642, + "loss/preference_sft": 0.29761844873428345, + "loss/reference_anchor": 0.5784320831298828, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.648583173751831, + "rewards/margins": 1.7795528173446655, + "rewards/rejected": 1.869030237197876, + "step": 460 + }, + { + "drift/chosen_abs": 0.2300829440355301, + "drift/rejected_abs": 0.09263820201158524, + "epoch": 0.44428520243640274, + "grad_norm": 11.0, + "learning_rate": 3.3139203861428914e-07, + "logits/chosen": -1.8710740804672241, + "logits/rejected": -1.9699445962905884, + "logps/chosen": -0.29734542965888977, + "logps/rejected": -0.29221880435943604, + "loss": 0.24437861144542694, + "loss/core": 0.23336219787597656, + "loss/preference_sft": 0.29734542965888977, + "loss/reference_anchor": 0.1171511560678482, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.299163818359375, + "rewards/margins": 1.3750882148742676, + "rewards/rejected": 0.9240756034851074, + "step": 465 + }, + { + "drift/chosen_abs": 0.297715425491333, + "drift/rejected_abs": 0.12405421584844589, + "epoch": 0.4490624626776544, + "grad_norm": 1064.0, + "learning_rate": 3.262160747570027e-07, + "logits/chosen": -2.1081130504608154, + "logits/rejected": -2.1759676933288574, + "logps/chosen": -0.282444030046463, + "logps/rejected": -0.26066750288009644, + "loss": 1.1672271490097046, + "loss/core": 1.122953176498413, + "loss/preference_sft": 0.282444030046463, + "loss/reference_anchor": 0.5620765686035156, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.976147174835205, + "rewards/margins": 1.8132495880126953, + "rewards/rejected": 1.1628977060317993, + "step": 470 + }, + { + "drift/chosen_abs": 0.1827501803636551, + "drift/rejected_abs": 0.14713791012763977, + "epoch": 0.453839722918906, + "grad_norm": 82.5, + "learning_rate": 3.210038361759807e-07, + "logits/chosen": -2.0727429389953613, + "logits/rejected": -2.012450933456421, + "logps/chosen": -0.2958831191062927, + "logps/rejected": -0.32066217064857483, + "loss": 0.4937218129634857, + "loss/core": 0.47373858094215393, + "loss/preference_sft": 0.2958831191062927, + "loss/reference_anchor": 0.23685435950756073, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8267005681991577, + "rewards/margins": 0.35839390754699707, + "rewards/rejected": 1.4683068990707397, + "step": 475 + }, + { + "drift/chosen_abs": 0.23704016208648682, + "drift/rejected_abs": 0.10950712114572525, + "epoch": 0.45861698316015764, + "grad_norm": 6.28125, + "learning_rate": 3.1575780361449364e-07, + "logits/chosen": -2.2403244972229004, + "logits/rejected": -2.296858310699463, + "logps/chosen": -0.2744103670120239, + "logps/rejected": -0.27569180727005005, + "loss": 0.5795036554336548, + "loss/core": 0.5565477013587952, + "loss/preference_sft": 0.2744103670120239, + "loss/reference_anchor": 0.2786380648612976, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.36838960647583, + "rewards/margins": 1.3759324550628662, + "rewards/rejected": 0.9924572110176086, + "step": 480 + }, + { + "drift/chosen_abs": 0.17343619465827942, + "drift/rejected_abs": 0.08826018869876862, + "epoch": 0.4633942434014093, + "grad_norm": 13.4375, + "learning_rate": 3.104804738999169e-07, + "logits/chosen": -2.1836743354797363, + "logits/rejected": -2.1913857460021973, + "logps/chosen": -0.2924710214138031, + "logps/rejected": -0.2964377999305725, + "loss": 0.16332216560840607, + "loss/core": 0.15528026223182678, + "loss/preference_sft": 0.2924710214138031, + "loss/reference_anchor": 0.07797853648662567, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7339932918548584, + "rewards/margins": 0.8536950349807739, + "rewards/rejected": 0.8802981376647949, + "step": 485 + }, + { + "drift/chosen_abs": 0.2068719118833542, + "drift/rejected_abs": 0.1048976331949234, + "epoch": 0.4681715036426609, + "grad_norm": 21.25, + "learning_rate": 3.0517435875537536e-07, + "logits/chosen": -2.1321020126342773, + "logits/rejected": -2.1710128784179688, + "logps/chosen": -0.3080752491950989, + "logps/rejected": -0.2949379086494446, + "loss": 0.30973607301712036, + "loss/core": 0.2963031232357025, + "loss/preference_sft": 0.3080752491950989, + "loss/reference_anchor": 0.1482984721660614, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.0682833194732666, + "rewards/margins": 1.0205515623092651, + "rewards/rejected": 1.047731637954712, + "step": 490 + }, + { + "drift/chosen_abs": 0.26098620891571045, + "drift/rejected_abs": 0.0885407105088234, + "epoch": 0.47294876388391255, + "grad_norm": 20.875, + "learning_rate": 2.998419836042993e-07, + "logits/chosen": -2.2028145790100098, + "logits/rejected": -2.297614097595215, + "logps/chosen": -0.29816746711730957, + "logps/rejected": -0.2916865348815918, + "loss": 0.7447795271873474, + "loss/core": 0.7156817317008972, + "loss/preference_sft": 0.29816746711730957, + "loss/reference_anchor": 0.3581535518169403, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.6079015731811523, + "rewards/margins": 1.724554419517517, + "rewards/rejected": 0.8833469152450562, + "step": 495 + }, + { + "drift/chosen_abs": 0.41630882024765015, + "drift/rejected_abs": 0.20278768241405487, + "epoch": 0.47772602412516424, + "grad_norm": 123.0, + "learning_rate": 2.9448588636846043e-07, + "logits/chosen": -1.9985599517822266, + "logits/rejected": -1.949119210243225, + "logps/chosen": -0.28114375472068787, + "logps/rejected": -0.30273741483688354, + "loss": 2.383561134338379, + "loss/core": 2.295335292816162, + "loss/preference_sft": 0.28114375472068787, + "loss/reference_anchor": 1.1482231616973877, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.163088798522949, + "rewards/margins": 2.138087511062622, + "rewards/rejected": 2.025001049041748, + "step": 500 + }, + { + "drift/chosen_abs": 0.3645007014274597, + "drift/rejected_abs": 0.18596860766410828, + "epoch": 0.4825032843664159, + "grad_norm": 354.0, + "learning_rate": 2.8910861626005773e-07, + "logits/chosen": -2.129087209701538, + "logits/rejected": -2.1622016429901123, + "logps/chosen": -0.28229135274887085, + "logps/rejected": -0.28652527928352356, + "loss": 1.6089633703231812, + "loss/core": 1.548729658126831, + "loss/preference_sft": 0.28229135274887085, + "loss/reference_anchor": 0.7748891115188599, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.643630266189575, + "rewards/margins": 1.8779332637786865, + "rewards/rejected": 1.7656971216201782, + "step": 505 + }, + { + "drift/chosen_abs": 0.20183968544006348, + "drift/rejected_abs": 0.0795665830373764, + "epoch": 0.4872805446076675, + "grad_norm": 0.28515625, + "learning_rate": 2.8371273256843074e-07, + "logits/chosen": -2.1265504360198975, + "logits/rejected": -2.259223699569702, + "logps/chosen": -0.31546053290367126, + "logps/rejected": -0.29201287031173706, + "loss": 0.4467145502567291, + "loss/core": 0.42825937271118164, + "loss/preference_sft": 0.31546053290367126, + "loss/reference_anchor": 0.21452267467975616, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.015305995941162, + "rewards/margins": 1.221923589706421, + "rewards/rejected": 0.793382465839386, + "step": 510 + }, + { + "drift/chosen_abs": 0.25470539927482605, + "drift/rejected_abs": 0.16728895902633667, + "epoch": 0.49205780484891914, + "grad_norm": 38.75, + "learning_rate": 2.783008034419767e-07, + "logits/chosen": -2.011662721633911, + "logits/rejected": -2.014399528503418, + "logps/chosen": -0.2826010584831238, + "logps/rejected": -0.306089848279953, + "loss": 0.4587891697883606, + "loss/core": 0.44014644622802734, + "loss/preference_sft": 0.2826010584831238, + "loss/reference_anchor": 0.22030992805957794, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.545462131500244, + "rewards/margins": 0.8881236910820007, + "rewards/rejected": 1.6573383808135986, + "step": 515 + }, + { + "drift/chosen_abs": 0.3071853816509247, + "drift/rejected_abs": 0.09579263627529144, + "epoch": 0.4968350650901708, + "grad_norm": 55.5, + "learning_rate": 2.7287540466585064e-07, + "logits/chosen": -2.0407145023345947, + "logits/rejected": -2.208981990814209, + "logps/chosen": -0.32281801104545593, + "logps/rejected": -0.3359902501106262, + "loss": 0.8434049487113953, + "loss/core": 0.8105605244636536, + "loss/preference_sft": 0.32281801104545593, + "loss/reference_anchor": 0.40564513206481934, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.066298007965088, + "rewards/margins": 2.121892213821411, + "rewards/rejected": 0.9444060325622559, + "step": 520 + }, + { + "drift/chosen_abs": 0.41961222887039185, + "drift/rejected_abs": 0.15004144608974457, + "epoch": 0.5016123253314224, + "grad_norm": 1408.0, + "learning_rate": 2.674391184360313e-07, + "logits/chosen": -2.1737782955169678, + "logits/rejected": -2.141087293624878, + "logps/chosen": -0.31184646487236023, + "logps/rejected": -0.29689449071884155, + "loss": 1.9573285579681396, + "loss/core": 1.8842939138412476, + "loss/preference_sft": 0.31184646487236023, + "loss/reference_anchor": 0.9426106214523315, + "rewards/accuracies": 0.875, + "rewards/chosen": 4.192916393280029, + "rewards/margins": 2.6963582038879395, + "rewards/rejected": 1.4965574741363525, + "step": 525 + }, + { + "drift/chosen_abs": 0.3052569031715393, + "drift/rejected_abs": 0.1496833860874176, + "epoch": 0.5063895855726741, + "grad_norm": 0.41796875, + "learning_rate": 2.6199453213033593e-07, + "logits/chosen": -2.0180935859680176, + "logits/rejected": -1.9813159704208374, + "logps/chosen": -0.28599947690963745, + "logps/rejected": -0.28876030445098877, + "loss": 0.8264201283454895, + "loss/core": 0.7944486141204834, + "loss/preference_sft": 0.28599947690963745, + "loss/reference_anchor": 0.3976878523826599, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.0507431030273438, + "rewards/margins": 1.5630565881729126, + "rewards/rejected": 1.4876863956451416, + "step": 530 + }, + { + "drift/chosen_abs": 0.31068387627601624, + "drift/rejected_abs": 0.10734076797962189, + "epoch": 0.5111668458139257, + "grad_norm": 580.0, + "learning_rate": 2.565442370769683e-07, + "logits/chosen": -2.1644198894500732, + "logits/rejected": -2.085977554321289, + "logps/chosen": -0.2844741642475128, + "logps/rejected": -0.27978286147117615, + "loss": 0.9638272523880005, + "loss/core": 0.9268702268600464, + "loss/preference_sft": 0.2844741642475128, + "loss/reference_anchor": 0.46431416273117065, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.104759931564331, + "rewards/margins": 2.0328688621520996, + "rewards/rejected": 1.0718910694122314, + "step": 535 + }, + { + "drift/chosen_abs": 0.15837408602237701, + "drift/rejected_abs": 0.15098315477371216, + "epoch": 0.5159441060551774, + "grad_norm": 1.5, + "learning_rate": 2.510908273211866e-07, + "logits/chosen": -2.086716413497925, + "logits/rejected": -2.065261125564575, + "logps/chosen": -0.27920594811439514, + "logps/rejected": -0.2877506613731384, + "loss": 0.33345627784729004, + "loss/core": 0.31936904788017273, + "loss/preference_sft": 0.27920594811439514, + "loss/reference_anchor": 0.15990903973579407, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.5837407112121582, + "rewards/margins": 0.07735572755336761, + "rewards/rejected": 1.5063849687576294, + "step": 540 + }, + { + "drift/chosen_abs": 0.2585676312446594, + "drift/rejected_abs": 0.08764716237783432, + "epoch": 0.520721366296429, + "grad_norm": 1264.0, + "learning_rate": 2.456368983906791e-07, + "logits/chosen": -2.305999994277954, + "logits/rejected": -2.1507370471954346, + "logps/chosen": -0.28886252641677856, + "logps/rejected": -0.2935750484466553, + "loss": 0.9189807772636414, + "loss/core": 0.8836311101913452, + "loss/preference_sft": 0.28886252641677856, + "loss/reference_anchor": 0.44244223833084106, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.5849061012268066, + "rewards/margins": 1.7120176553726196, + "rewards/rejected": 0.8728883862495422, + "step": 545 + }, + { + "drift/chosen_abs": 0.27887991070747375, + "drift/rejected_abs": 0.1610538214445114, + "epoch": 0.5254986265376806, + "grad_norm": 12.75, + "learning_rate": 2.401850460602329e-07, + "logits/chosen": -1.8745006322860718, + "logits/rejected": -1.8935530185699463, + "logps/chosen": -0.2741144299507141, + "logps/rejected": -0.2991863191127777, + "loss": 0.8370600938796997, + "loss/core": 0.8047995567321777, + "loss/preference_sft": 0.2741144299507141, + "loss/reference_anchor": 0.40272873640060425, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.7885258197784424, + "rewards/margins": 1.1791884899139404, + "rewards/rejected": 1.6093372106552124, + "step": 550 + }, + { + "drift/chosen_abs": 0.3264530301094055, + "drift/rejected_abs": 0.09586937725543976, + "epoch": 0.5302758867789323, + "grad_norm": 5.8125, + "learning_rate": 2.3473786511628575e-07, + "logits/chosen": -2.0708940029144287, + "logits/rejected": -2.1273093223571777, + "logps/chosen": -0.3148099184036255, + "logps/rejected": -0.2786844074726105, + "loss": 0.81494140625, + "loss/core": 0.7831686735153198, + "loss/preference_sft": 0.3148099184036255, + "loss/reference_anchor": 0.39215487241744995, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.2636561393737793, + "rewards/margins": 2.305521011352539, + "rewards/rejected": 0.9581351280212402, + "step": 555 + }, + { + "drift/chosen_abs": 0.28125500679016113, + "drift/rejected_abs": 0.15266069769859314, + "epoch": 0.5350531470201839, + "grad_norm": 50.5, + "learning_rate": 2.2929794812194894e-07, + "logits/chosen": -2.1583635807037354, + "logits/rejected": -2.1458077430725098, + "logps/chosen": -0.27610182762145996, + "logps/rejected": -0.28124934434890747, + "loss": 0.7136462926864624, + "loss/core": 0.6858313083648682, + "loss/preference_sft": 0.27610182762145996, + "loss/reference_anchor": 0.3432568311691284, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.8115084171295166, + "rewards/margins": 1.2856230735778809, + "rewards/rejected": 1.5258853435516357, + "step": 560 + }, + { + "drift/chosen_abs": 0.27818697690963745, + "drift/rejected_abs": 0.16565275192260742, + "epoch": 0.5398304072614356, + "grad_norm": 19.5, + "learning_rate": 2.2386788418308665e-07, + "logits/chosen": -2.056506395339966, + "logits/rejected": -2.082536220550537, + "logps/chosen": -0.28751203417778015, + "logps/rejected": -0.3056444227695465, + "loss": 1.35868239402771, + "loss/core": 1.3074853420257568, + "loss/preference_sft": 0.28751203417778015, + "loss/reference_anchor": 0.6538757085800171, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.7808547019958496, + "rewards/margins": 1.124733805656433, + "rewards/rejected": 1.6561208963394165, + "step": 565 + }, + { + "drift/chosen_abs": 0.3337910771369934, + "drift/rejected_abs": 0.16797366738319397, + "epoch": 0.5446076675026872, + "grad_norm": 71.5, + "learning_rate": 2.184502577160426e-07, + "logits/chosen": -2.0548603534698486, + "logits/rejected": -2.1045546531677246, + "logps/chosen": -0.2894991338253021, + "logps/rejected": -0.2895090579986572, + "loss": 0.9317213892936707, + "loss/core": 0.8959233164787292, + "loss/preference_sft": 0.2894991338253021, + "loss/reference_anchor": 0.4483569264411926, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.335692882537842, + "rewards/margins": 1.657538652420044, + "rewards/rejected": 1.6781543493270874, + "step": 570 + }, + { + "drift/chosen_abs": 0.18387334048748016, + "drift/rejected_abs": 0.06442725658416748, + "epoch": 0.5493849277439389, + "grad_norm": 8.9375, + "learning_rate": 2.1304764721759732e-07, + "logits/chosen": -2.1352288722991943, + "logits/rejected": -2.1651110649108887, + "logps/chosen": -0.30129939317703247, + "logps/rejected": -0.32308152318000793, + "loss": 0.345123291015625, + "loss/core": 0.3304422199726105, + "loss/preference_sft": 0.30129939317703247, + "loss/reference_anchor": 0.16561798751354218, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.8363609313964844, + "rewards/margins": 1.274545669555664, + "rewards/rejected": 0.5618152022361755, + "step": 575 + }, + { + "drift/chosen_abs": 0.14127376675605774, + "drift/rejected_abs": 0.0939093828201294, + "epoch": 0.5541621879851905, + "grad_norm": 0.49609375, + "learning_rate": 2.0766262403774385e-07, + "logits/chosen": -2.4066150188446045, + "logits/rejected": -2.462902545928955, + "logps/chosen": -0.26729652285575867, + "logps/rejected": -0.2831682562828064, + "loss": 0.17010077834129333, + "loss/core": 0.1620091199874878, + "loss/preference_sft": 0.26729652285575867, + "loss/reference_anchor": 0.08115912973880768, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.4122838973999023, + "rewards/margins": 0.4755134582519531, + "rewards/rejected": 0.9367703199386597, + "step": 580 + }, + { + "drift/chosen_abs": 0.22129257023334503, + "drift/rejected_abs": 0.13631665706634521, + "epoch": 0.5589394482264421, + "grad_norm": 35.25, + "learning_rate": 2.0229775115586378e-07, + "logits/chosen": -2.1155853271484375, + "logits/rejected": -2.0802512168884277, + "logps/chosen": -0.2712664008140564, + "logps/rejected": -0.28381767868995667, + "loss": 1.4533600807189941, + "loss/core": 1.3988597393035889, + "loss/preference_sft": 0.2712664008140564, + "loss/reference_anchor": 0.6995447874069214, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.212214469909668, + "rewards/margins": 0.8542079925537109, + "rewards/rejected": 1.358006477355957, + "step": 585 + }, + { + "drift/chosen_abs": 0.3607145845890045, + "drift/rejected_abs": 0.10202290117740631, + "epoch": 0.5637167084676937, + "grad_norm": 4.625, + "learning_rate": 1.9695558196088844e-07, + "logits/chosen": -2.002814292907715, + "logits/rejected": -2.099381923675537, + "logps/chosen": -0.28898441791534424, + "logps/rejected": -0.28851771354675293, + "loss": 0.9255863428115845, + "loss/core": 0.8899755477905273, + "loss/preference_sft": 0.28898441791534424, + "loss/reference_anchor": 0.44591063261032104, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.606508731842041, + "rewards/margins": 2.5896670818328857, + "rewards/rejected": 1.0168412923812866, + "step": 590 + }, + { + "drift/chosen_abs": 0.1707986295223236, + "drift/rejected_abs": 0.08907818049192429, + "epoch": 0.5684939687089454, + "grad_norm": 4.28125, + "learning_rate": 1.9163865903602372e-07, + "logits/chosen": -2.0221307277679443, + "logits/rejected": -2.0168774127960205, + "logps/chosen": -0.2755976915359497, + "logps/rejected": -0.27291139960289, + "loss": 0.11963596194982529, + "loss/core": 0.1133008748292923, + "loss/preference_sft": 0.2755976915359497, + "loss/reference_anchor": 0.05690792202949524, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.7069168090820312, + "rewards/margins": 0.8279293775558472, + "rewards/rejected": 0.8789873123168945, + "step": 595 + }, + { + "drift/chosen_abs": 0.18752028048038483, + "drift/rejected_abs": 0.10666258633136749, + "epoch": 0.5732712289501971, + "grad_norm": 24.875, + "learning_rate": 1.8634951294861806e-07, + "logits/chosen": -2.253175735473633, + "logits/rejected": -2.2427711486816406, + "logps/chosen": -0.2663193345069885, + "logps/rejected": -0.28532105684280396, + "loss": 0.31239908933639526, + "loss/core": 0.2991681396961212, + "loss/preference_sft": 0.2663193345069885, + "loss/reference_anchor": 0.14978043735027313, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.8731563091278076, + "rewards/margins": 0.82030189037323, + "rewards/rejected": 1.052854299545288, + "step": 600 + }, + { + "drift/chosen_abs": 0.41676053404808044, + "drift/rejected_abs": 0.11940282583236694, + "epoch": 0.5780484891914487, + "grad_norm": 133.0, + "learning_rate": 1.810906610457502e-07, + "logits/chosen": -2.0973143577575684, + "logits/rejected": -2.171078681945801, + "logps/chosen": -0.2819843590259552, + "logps/rejected": -0.27706506848335266, + "loss": 1.816592812538147, + "loss/core": 1.7488460540771484, + "loss/preference_sft": 0.2819843590259552, + "loss/reference_anchor": 0.875086784362793, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.165308952331543, + "rewards/margins": 2.9730935096740723, + "rewards/rejected": 1.1922153234481812, + "step": 605 + }, + { + "drift/chosen_abs": 0.4254422187805176, + "drift/rejected_abs": 0.19570092856884003, + "epoch": 0.5828257494327004, + "grad_norm": 106.0, + "learning_rate": 1.7586460625610726e-07, + "logits/chosen": -2.0943334102630615, + "logits/rejected": -2.1102890968322754, + "logps/chosen": -0.2782813310623169, + "logps/rejected": -0.27373403310775757, + "loss": 1.9183744192123413, + "loss/core": 1.8469626903533936, + "loss/preference_sft": 0.2782813310623169, + "loss/reference_anchor": 0.9243267178535461, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 4.251742362976074, + "rewards/margins": 2.3000569343566895, + "rewards/rejected": 1.9516862630844116, + "step": 610 + }, + { + "drift/chosen_abs": 0.27416449785232544, + "drift/rejected_abs": 0.14728380739688873, + "epoch": 0.587603009673952, + "grad_norm": 13.5, + "learning_rate": 1.70673835898727e-07, + "logits/chosen": -1.9468109607696533, + "logits/rejected": -2.0125274658203125, + "logps/chosen": -0.3140425682067871, + "logps/rejected": -0.30548495054244995, + "loss": 0.6002452373504639, + "loss/core": 0.5762513279914856, + "loss/preference_sft": 0.3140425682067871, + "loss/reference_anchor": 0.2885145843029022, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.7396042346954346, + "rewards/margins": 1.2685816287994385, + "rewards/rejected": 1.471022367477417, + "step": 615 + }, + { + "drift/chosen_abs": 0.16883981227874756, + "drift/rejected_abs": 0.14558133482933044, + "epoch": 0.5923802699152036, + "grad_norm": 7.15625, + "learning_rate": 1.6552082049916824e-07, + "logits/chosen": -2.119967460632324, + "logits/rejected": -2.1456103324890137, + "logps/chosen": -0.2863292992115021, + "logps/rejected": -0.29261788725852966, + "loss": 0.40366506576538086, + "loss/core": 0.38699546456336975, + "loss/preference_sft": 0.2863292992115021, + "loss/reference_anchor": 0.1936284750699997, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.6874778270721436, + "rewards/margins": 0.236982062458992, + "rewards/rejected": 1.450495719909668, + "step": 620 + }, + { + "drift/chosen_abs": 0.388235867023468, + "drift/rejected_abs": 0.17607903480529785, + "epoch": 0.5971575301564552, + "grad_norm": 1256.0, + "learning_rate": 1.6040801261367493e-07, + "logits/chosen": -1.9117774963378906, + "logits/rejected": -1.9400612115859985, + "logps/chosen": -0.3054983913898468, + "logps/rejected": -0.3448588252067566, + "loss": 2.1722350120544434, + "loss/core": 2.0914366245269775, + "loss/preference_sft": 0.3054983913898468, + "loss/reference_anchor": 1.0467677116394043, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.8796839714050293, + "rewards/margins": 2.4553399085998535, + "rewards/rejected": 1.4243443012237549, + "step": 625 + }, + { + "drift/chosen_abs": 0.29388323426246643, + "drift/rejected_abs": 0.12398219108581543, + "epoch": 0.6019347903977069, + "grad_norm": 204.0, + "learning_rate": 1.5533784566189175e-07, + "logits/chosen": -1.9094196557998657, + "logits/rejected": -2.0244593620300293, + "logps/chosen": -0.27296558022499084, + "logps/rejected": -0.2840172350406647, + "loss": 0.9247919321060181, + "loss/core": 0.8893438577651978, + "loss/preference_sft": 0.27296558022499084, + "loss/reference_anchor": 0.4453449249267578, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.938032388687134, + "rewards/margins": 1.7004245519638062, + "rewards/rejected": 1.237607479095459, + "step": 630 + }, + { + "drift/chosen_abs": 0.23004408180713654, + "drift/rejected_abs": 0.10162544250488281, + "epoch": 0.6067120506389586, + "grad_norm": 300.0, + "learning_rate": 1.5031273276868845e-07, + "logits/chosen": -1.9615974426269531, + "logits/rejected": -2.0472800731658936, + "logps/chosen": -0.3058452904224396, + "logps/rejected": -0.3000328540802002, + "loss": 0.45604729652404785, + "loss/core": 0.43732786178588867, + "loss/preference_sft": 0.3058452904224396, + "loss/reference_anchor": 0.21900811791419983, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.2964746952056885, + "rewards/margins": 1.2814719676971436, + "rewards/rejected": 1.015002727508545, + "step": 635 + }, + { + "drift/chosen_abs": 0.19580507278442383, + "drift/rejected_abs": 0.07966174185276031, + "epoch": 0.6114893108802102, + "grad_norm": 2.625, + "learning_rate": 1.4533506561564305e-07, + "logits/chosen": -2.188485622406006, + "logits/rejected": -2.2632412910461426, + "logps/chosen": -0.30155614018440247, + "logps/rejected": -0.2989227771759033, + "loss": 0.4078275263309479, + "loss/core": 0.39089325070381165, + "loss/preference_sft": 0.30155614018440247, + "loss/reference_anchor": 0.19563566148281097, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.9569017887115479, + "rewards/margins": 1.161104440689087, + "rewards/rejected": 0.7957972288131714, + "step": 640 + }, + { + "drift/chosen_abs": 0.1673978865146637, + "drift/rejected_abs": 0.11365523189306259, + "epoch": 0.6162665711214619, + "grad_norm": 15.8125, + "learning_rate": 1.404072133027306e-07, + "logits/chosen": -2.133847713470459, + "logits/rejected": -2.1120846271514893, + "logps/chosen": -0.2908037602901459, + "logps/rejected": -0.30515560507774353, + "loss": 0.2214515656232834, + "loss/core": 0.21132436394691467, + "loss/preference_sft": 0.2908037602901459, + "loss/reference_anchor": 0.10594894737005234, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.6722071170806885, + "rewards/margins": 0.5402682423591614, + "rewards/rejected": 1.1319388151168823, + "step": 645 + }, + { + "drift/chosen_abs": 0.13293756544589996, + "drift/rejected_abs": 0.08515778928995132, + "epoch": 0.6210438313627135, + "grad_norm": 4.21875, + "learning_rate": 1.3553152122076078e-07, + "logits/chosen": -2.2003543376922607, + "logits/rejected": -2.1520040035247803, + "logps/chosen": -0.2900927662849426, + "logps/rejected": -0.30046719312667847, + "loss": 0.15672235190868378, + "loss/core": 0.14894674718379974, + "loss/preference_sft": 0.2900927662849426, + "loss/reference_anchor": 0.07466545701026917, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.327453851699829, + "rewards/margins": 0.5011469125747681, + "rewards/rejected": 0.8263068199157715, + "step": 650 + }, + { + "drift/chosen_abs": 0.20016181468963623, + "drift/rejected_abs": 0.09695510566234589, + "epoch": 0.6258210916039652, + "grad_norm": 56.0, + "learning_rate": 1.3071030993509787e-07, + "logits/chosen": -2.0374953746795654, + "logits/rejected": -2.099519729614258, + "logps/chosen": -0.2827564477920532, + "logps/rejected": -0.29163193702697754, + "loss": 0.2658204436302185, + "loss/core": 0.2541511654853821, + "loss/preference_sft": 0.2827564477920532, + "loss/reference_anchor": 0.12731416523456573, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9995691776275635, + "rewards/margins": 1.030291199684143, + "rewards/rejected": 0.9692778587341309, + "step": 655 + }, + { + "drift/chosen_abs": 0.27798160910606384, + "drift/rejected_abs": 0.13286152482032776, + "epoch": 0.6305983518452167, + "grad_norm": 2.375, + "learning_rate": 1.2594587408119804e-07, + "logits/chosen": -2.0071966648101807, + "logits/rejected": -2.002094268798828, + "logps/chosen": -0.293853223323822, + "logps/rejected": -0.3049863874912262, + "loss": 1.3727947473526, + "loss/core": 1.321033000946045, + "loss/preference_sft": 0.293853223323822, + "loss/reference_anchor": 0.6607749462127686, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.7776036262512207, + "rewards/margins": 1.4595942497253418, + "rewards/rejected": 1.3180092573165894, + "step": 660 + }, + { + "drift/chosen_abs": 0.23454730212688446, + "drift/rejected_abs": 0.14776912331581116, + "epoch": 0.6353756120864684, + "grad_norm": 141.0, + "learning_rate": 1.2124048127248644e-07, + "logits/chosen": -2.1691813468933105, + "logits/rejected": -2.143655776977539, + "logps/chosen": -0.28466880321502686, + "logps/rejected": -0.2818228602409363, + "loss": 0.5208796858787537, + "loss/core": 0.4999764859676361, + "loss/preference_sft": 0.28466880321502686, + "loss/reference_anchor": 0.2502433955669403, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.3442444801330566, + "rewards/margins": 0.8682305216789246, + "rewards/rejected": 1.4760141372680664, + "step": 665 + }, + { + "drift/chosen_abs": 0.364292711019516, + "drift/rejected_abs": 0.1280832588672638, + "epoch": 0.64015287232772, + "grad_norm": 1.484375, + "learning_rate": 1.1659637102109712e-07, + "logits/chosen": -1.9568021297454834, + "logits/rejected": -2.003387689590454, + "logps/chosen": -0.2745230197906494, + "logps/rejected": -0.31976574659347534, + "loss": 1.0210597515106201, + "loss/core": 0.982128918170929, + "loss/preference_sft": 0.2745230197906494, + "loss/reference_anchor": 0.49162501096725464, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.6429271697998047, + "rewards/margins": 2.3816723823547363, + "rewards/rejected": 1.2612544298171997, + "step": 670 + }, + { + "drift/chosen_abs": 0.1619919240474701, + "drift/rejected_abs": 0.08836764097213745, + "epoch": 0.6449301325689717, + "grad_norm": 14.5625, + "learning_rate": 1.1201575367198546e-07, + "logits/chosen": -2.012540340423584, + "logits/rejected": -2.0545716285705566, + "logps/chosen": -0.2888466715812683, + "logps/rejected": -0.28877875208854675, + "loss": 0.13238023221492767, + "loss/core": 0.12548786401748657, + "loss/preference_sft": 0.2888466715812683, + "loss/reference_anchor": 0.06301363557577133, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6180137395858765, + "rewards/margins": 0.7427892088890076, + "rewards/rejected": 0.8752244114875793, + "step": 675 + }, + { + "drift/chosen_abs": 0.21459881961345673, + "drift/rejected_abs": 0.1355762928724289, + "epoch": 0.6497073928102234, + "grad_norm": 1.7734375, + "learning_rate": 1.0750080935092423e-07, + "logits/chosen": -2.1347737312316895, + "logits/rejected": -2.0533447265625, + "logps/chosen": -0.2823336720466614, + "logps/rejected": -0.2946282923221588, + "loss": 0.43657293915748596, + "loss/core": 0.41873565316200256, + "loss/preference_sft": 0.2823336720466614, + "loss/reference_anchor": 0.20959632098674774, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.1431565284729004, + "rewards/margins": 0.7881289720535278, + "rewards/rejected": 1.3550279140472412, + "step": 680 + }, + { + "drift/chosen_abs": 0.2882576882839203, + "drift/rejected_abs": 0.15576216578483582, + "epoch": 0.654484653051475, + "grad_norm": 4.375, + "learning_rate": 1.0305368692688174e-07, + "logits/chosen": -2.0634636878967285, + "logits/rejected": -2.049027919769287, + "logps/chosen": -0.25410422682762146, + "logps/rejected": -0.2750704884529114, + "loss": 1.0024207830429077, + "loss/core": 0.9643166661262512, + "loss/preference_sft": 0.25410422682762146, + "loss/reference_anchor": 0.4826439917087555, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.8811697959899902, + "rewards/margins": 1.326778769493103, + "rewards/rejected": 1.5543910264968872, + "step": 685 + }, + { + "drift/chosen_abs": 0.17661243677139282, + "drift/rejected_abs": 0.1041385754942894, + "epoch": 0.6592619132927267, + "grad_norm": 3.359375, + "learning_rate": 9.867650298927643e-08, + "logits/chosen": -1.8653026819229126, + "logits/rejected": -1.8865511417388916, + "logps/chosen": -0.3086770176887512, + "logps/rejected": -0.3341335952281952, + "loss": 0.19030842185020447, + "loss/core": 0.18117818236351013, + "loss/preference_sft": 0.3086770176887512, + "loss/reference_anchor": 0.0908689871430397, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7632665634155273, + "rewards/margins": 0.7324223518371582, + "rewards/rejected": 1.0308443307876587, + "step": 690 + }, + { + "drift/chosen_abs": 0.19695012271404266, + "drift/rejected_abs": 0.06876302510499954, + "epoch": 0.6640391735339782, + "grad_norm": 10.125, + "learning_rate": 9.437134084059515e-08, + "logits/chosen": -2.2494120597839355, + "logits/rejected": -2.259532928466797, + "logps/chosen": -0.3065325617790222, + "logps/rejected": -0.29353365302085876, + "loss": 0.39579930901527405, + "loss/core": 0.3792479932308197, + "loss/preference_sft": 0.3065325617790222, + "loss/reference_anchor": 0.19003014266490936, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9680402278900146, + "rewards/margins": 1.29190993309021, + "rewards/rejected": 0.6761302947998047, + "step": 695 + }, + { + "drift/chosen_abs": 0.2228858470916748, + "drift/rejected_abs": 0.15424631536006927, + "epoch": 0.6688164337752299, + "grad_norm": 470.0, + "learning_rate": 9.014024950485383e-08, + "logits/chosen": -2.0826706886291504, + "logits/rejected": -2.187513828277588, + "logps/chosen": -0.2971203923225403, + "logps/rejected": -0.3260975480079651, + "loss": 0.5006673336029053, + "loss/core": 0.48041480779647827, + "loss/preference_sft": 0.2971203923225403, + "loss/reference_anchor": 0.24032163619995117, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.227379083633423, + "rewards/margins": 0.7068957090377808, + "rewards/rejected": 1.5204832553863525, + "step": 700 + }, + { + "drift/chosen_abs": 0.33960968255996704, + "drift/rejected_abs": 0.15266656875610352, + "epoch": 0.6735936940164815, + "grad_norm": 95.0, + "learning_rate": 8.598524275237321e-08, + "logits/chosen": -2.175156593322754, + "logits/rejected": -2.1317992210388184, + "logps/chosen": -0.28126639127731323, + "logps/rejected": -0.2680434584617615, + "loss": 0.8337176442146301, + "loss/core": 0.8014980554580688, + "loss/preference_sft": 0.28126639127731323, + "loss/reference_anchor": 0.4014674723148346, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.3955001831054688, + "rewards/margins": 1.8699357509613037, + "rewards/rejected": 1.525564432144165, + "step": 705 + }, + { + "drift/chosen_abs": 0.3576631546020508, + "drift/rejected_abs": 0.18644385039806366, + "epoch": 0.6783709542577332, + "grad_norm": 440.0, + "learning_rate": 8.190829814133293e-08, + "logits/chosen": -1.863407850265503, + "logits/rejected": -1.8509485721588135, + "logps/chosen": -0.27954185009002686, + "logps/rejected": -0.3012255132198334, + "loss": 1.3718466758728027, + "loss/core": 1.320197582244873, + "loss/preference_sft": 0.27954185009002686, + "loss/reference_anchor": 0.6606996655464172, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.5734474658966064, + "rewards/margins": 1.7104486227035522, + "rewards/rejected": 1.8629987239837646, + "step": 710 + }, + { + "drift/chosen_abs": 0.18702775239944458, + "drift/rejected_abs": 0.08271317183971405, + "epoch": 0.6831482144989849, + "grad_norm": 3.921875, + "learning_rate": 7.791135607656146e-08, + "logits/chosen": -2.228705644607544, + "logits/rejected": -2.209040403366089, + "logps/chosen": -0.2836971879005432, + "logps/rejected": -0.3425886332988739, + "loss": 0.5329797267913818, + "loss/core": 0.5116499066352844, + "loss/preference_sft": 0.2836971879005432, + "loss/reference_anchor": 0.25602635741233826, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.8697589635849, + "rewards/margins": 1.2295645475387573, + "rewards/rejected": 0.6401944160461426, + "step": 715 + }, + { + "drift/chosen_abs": 0.20850583910942078, + "drift/rejected_abs": 0.1485956758260727, + "epoch": 0.6879254747402365, + "grad_norm": 24.875, + "learning_rate": 7.399631888600796e-08, + "logits/chosen": -1.9758691787719727, + "logits/rejected": -1.9829914569854736, + "logps/chosen": -0.29822248220443726, + "logps/rejected": -0.30190637707710266, + "loss": 0.5085002183914185, + "loss/core": 0.487942636013031, + "loss/preference_sft": 0.29822248220443726, + "loss/reference_anchor": 0.24427810311317444, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.083251953125, + "rewards/margins": 0.5999056100845337, + "rewards/rejected": 1.4833463430404663, + "step": 720 + }, + { + "drift/chosen_abs": 0.15803976356983185, + "drift/rejected_abs": 0.07249420881271362, + "epoch": 0.6927027349814882, + "grad_norm": 4.03125, + "learning_rate": 7.016504991533726e-08, + "logits/chosen": -2.101017475128174, + "logits/rejected": -2.215353488922119, + "logps/chosen": -0.28623729944229126, + "logps/rejected": -0.2781991958618164, + "loss": 0.18330907821655273, + "loss/core": 0.17459434270858765, + "loss/preference_sft": 0.28623729944229126, + "loss/reference_anchor": 0.08757317811250687, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.5793583393096924, + "rewards/margins": 0.8571645021438599, + "rewards/rejected": 0.7221937775611877, + "step": 725 + }, + { + "drift/chosen_abs": 0.10135958343744278, + "drift/rejected_abs": 0.08568556606769562, + "epoch": 0.6974799952227397, + "grad_norm": 422.0, + "learning_rate": 6.641937264107867e-08, + "logits/chosen": -2.1984219551086426, + "logits/rejected": -2.248124599456787, + "logps/chosen": -0.287166953086853, + "logps/rejected": -0.2960854470729828, + "loss": 0.1953887790441513, + "loss/core": 0.18624430894851685, + "loss/preference_sft": 0.287166953086853, + "loss/reference_anchor": 0.09321005642414093, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.012143850326538, + "rewards/margins": 0.15792995691299438, + "rewards/rejected": 0.8542140126228333, + "step": 730 + }, + { + "drift/chosen_abs": 0.14169880747795105, + "drift/rejected_abs": 0.06580595672130585, + "epoch": 0.7022572554639914, + "grad_norm": 57.5, + "learning_rate": 6.276106980274944e-08, + "logits/chosen": -2.031146764755249, + "logits/rejected": -2.07769513130188, + "logps/chosen": -0.3201434314250946, + "logps/rejected": -0.3183977007865906, + "loss": 0.15187807381153107, + "loss/core": 0.14405521750450134, + "loss/preference_sft": 0.3201434314250946, + "loss/reference_anchor": 0.07229013741016388, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.4163552522659302, + "rewards/margins": 0.759195864200592, + "rewards/rejected": 0.6571593880653381, + "step": 735 + }, + { + "drift/chosen_abs": 0.25509342551231384, + "drift/rejected_abs": 0.1779322326183319, + "epoch": 0.707034515705243, + "grad_norm": 5.75, + "learning_rate": 5.919188255436777e-08, + "logits/chosen": -2.076237440109253, + "logits/rejected": -2.094517230987549, + "logps/chosen": -0.31439438462257385, + "logps/rejected": -0.3211822211742401, + "loss": 1.5399339199066162, + "loss/core": 1.4819676876068115, + "loss/preference_sft": 0.31439438462257385, + "loss/reference_anchor": 0.7414413094520569, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.5507071018218994, + "rewards/margins": 0.7717518210411072, + "rewards/rejected": 1.7789552211761475, + "step": 740 + }, + { + "drift/chosen_abs": 0.31124961376190186, + "drift/rejected_abs": 0.15094509720802307, + "epoch": 0.7118117759464947, + "grad_norm": 402.0, + "learning_rate": 5.571350963575727e-08, + "logits/chosen": -2.030458450317383, + "logits/rejected": -2.0747735500335693, + "logps/chosen": -0.2915259897708893, + "logps/rejected": -0.2831938862800598, + "loss": 0.6287981271743774, + "loss/core": 0.6039372682571411, + "loss/preference_sft": 0.2915259897708893, + "loss/reference_anchor": 0.30232542753219604, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.110504150390625, + "rewards/margins": 1.6071808338165283, + "rewards/rejected": 1.5033233165740967, + "step": 745 + }, + { + "drift/chosen_abs": 0.11992615461349487, + "drift/rejected_abs": 0.12524381279945374, + "epoch": 0.7165890361877463, + "grad_norm": 274.0, + "learning_rate": 5.232760656403923e-08, + "logits/chosen": -2.166827917098999, + "logits/rejected": -2.1589953899383545, + "logps/chosen": -0.3125108480453491, + "logps/rejected": -0.304274320602417, + "loss": 0.5836885571479797, + "loss/core": 0.5603240728378296, + "loss/preference_sft": 0.3125108480453491, + "loss/reference_anchor": 0.2802754044532776, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.197912335395813, + "rewards/margins": -0.051997799426317215, + "rewards/rejected": 1.2499101161956787, + "step": 750 + }, + { + "drift/chosen_abs": 0.23778387904167175, + "drift/rejected_abs": 0.16927723586559296, + "epoch": 0.721366296428998, + "grad_norm": 18.75, + "learning_rate": 4.9035784845695675e-08, + "logits/chosen": -1.9069569110870361, + "logits/rejected": -1.817491888999939, + "logps/chosen": -0.30986255407333374, + "logps/rejected": -0.29673609137535095, + "loss": 0.40122777223587036, + "loss/core": 0.3844679892063141, + "loss/preference_sft": 0.30986255407333374, + "loss/reference_anchor": 0.19247810542583466, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.373359203338623, + "rewards/margins": 0.687952995300293, + "rewards/rejected": 1.6854064464569092, + "step": 755 + }, + { + "drift/chosen_abs": 0.19871307909488678, + "drift/rejected_abs": 0.08636997640132904, + "epoch": 0.7261435566702497, + "grad_norm": 22.0, + "learning_rate": 4.583961120958027e-08, + "logits/chosen": -2.157874345779419, + "logits/rejected": -2.1768975257873535, + "logps/chosen": -0.3034326434135437, + "logps/rejected": -0.2945040166378021, + "loss": 0.28818535804748535, + "loss/core": 0.27553677558898926, + "loss/preference_sft": 0.3034326434135437, + "loss/reference_anchor": 0.13830482959747314, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9834944009780884, + "rewards/margins": 1.1288237571716309, + "rewards/rejected": 0.8546707034111023, + "step": 760 + }, + { + "drift/chosen_abs": 0.17785514891147614, + "drift/rejected_abs": 0.10489670932292938, + "epoch": 0.7309208169115012, + "grad_norm": 44.75, + "learning_rate": 4.2740606861239594e-08, + "logits/chosen": -2.0543203353881836, + "logits/rejected": -2.1251354217529297, + "logps/chosen": -0.2776801586151123, + "logps/rejected": -0.2736127972602844, + "loss": 0.24507012963294983, + "loss/core": 0.2341887503862381, + "loss/preference_sft": 0.2776801586151123, + "loss/reference_anchor": 0.11731725931167603, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7780964374542236, + "rewards/margins": 0.7318509817123413, + "rewards/rejected": 1.0462454557418823, + "step": 765 + }, + { + "drift/chosen_abs": 0.3685626983642578, + "drift/rejected_abs": 0.23492638766765594, + "epoch": 0.7356980771527529, + "grad_norm": 52.25, + "learning_rate": 3.974024675890189e-08, + "logits/chosen": -1.9574251174926758, + "logits/rejected": -1.9248749017715454, + "logps/chosen": -0.3148086667060852, + "logps/rejected": -0.28525421023368835, + "loss": 2.2472522258758545, + "loss/core": 2.1636857986450195, + "loss/preference_sft": 0.3148086667060852, + "loss/reference_anchor": 1.0827428102493286, + "rewards/accuracies": 0.8125, + "rewards/chosen": 3.68304181098938, + "rewards/margins": 1.3362334966659546, + "rewards/rejected": 2.346808433532715, + "step": 770 + }, + { + "drift/chosen_abs": 0.15117646753787994, + "drift/rejected_abs": 0.09319499135017395, + "epoch": 0.7404753373940045, + "grad_norm": 2.078125, + "learning_rate": 3.683995891147695e-08, + "logits/chosen": -2.073338031768799, + "logits/rejected": -2.103886127471924, + "logps/chosen": -0.30775031447410583, + "logps/rejected": -0.3014872670173645, + "loss": 0.1496042162179947, + "loss/core": 0.14195644855499268, + "loss/preference_sft": 0.30775031447410583, + "loss/reference_anchor": 0.07119497656822205, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.5114030838012695, + "rewards/margins": 0.5813368558883667, + "rewards/rejected": 0.9300659894943237, + "step": 775 + }, + { + "drift/chosen_abs": 0.18838293850421906, + "drift/rejected_abs": 0.12301009893417358, + "epoch": 0.7452525976352562, + "grad_norm": 19.75, + "learning_rate": 3.404112369890108e-08, + "logits/chosen": -2.15893816947937, + "logits/rejected": -2.1729979515075684, + "logps/chosen": -0.30296415090560913, + "logps/rejected": -0.307880163192749, + "loss": 0.23419561982154846, + "loss/core": 0.2235209047794342, + "loss/preference_sft": 0.30296415090560913, + "loss/reference_anchor": 0.11203303188085556, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.8827606439590454, + "rewards/margins": 0.6561026573181152, + "rewards/rejected": 1.2266579866409302, + "step": 780 + }, + { + "drift/chosen_abs": 0.34316182136535645, + "drift/rejected_abs": 0.13256040215492249, + "epoch": 0.7500298578765078, + "grad_norm": 1.765625, + "learning_rate": 3.134507321515106e-08, + "logits/chosen": -1.9713062047958374, + "logits/rejected": -1.9291038513183594, + "logps/chosen": -0.31109505891799927, + "logps/rejected": -0.2964383065700531, + "loss": 1.2833325862884521, + "loss/core": 1.2346382141113281, + "loss/preference_sft": 0.31109505891799927, + "loss/reference_anchor": 0.6181516647338867, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.4299960136413574, + "rewards/margins": 2.1115283966064453, + "rewards/rejected": 1.318467617034912, + "step": 785 + }, + { + "drift/chosen_abs": 0.2622746527194977, + "drift/rejected_abs": 0.18666431307792664, + "epoch": 0.7548071181177595, + "grad_norm": 576.0, + "learning_rate": 2.875309063423956e-08, + "logits/chosen": -2.1486878395080566, + "logits/rejected": -1.9767792224884033, + "logps/chosen": -0.28585386276245117, + "logps/rejected": -0.3247356414794922, + "loss": 1.1903468370437622, + "loss/core": 1.1452568769454956, + "loss/preference_sft": 0.28585386276245117, + "loss/reference_anchor": 0.5726125240325928, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.621675968170166, + "rewards/margins": 0.7564936876296997, + "rewards/rejected": 1.8651822805404663, + "step": 790 + }, + { + "drift/chosen_abs": 0.2373015433549881, + "drift/rejected_abs": 0.11792800575494766, + "epoch": 0.7595843783590112, + "grad_norm": 25.0, + "learning_rate": 2.626640959949375e-08, + "logits/chosen": -1.9540714025497437, + "logits/rejected": -1.9722576141357422, + "logps/chosen": -0.30823540687561035, + "logps/rejected": -0.2971736788749695, + "loss": 1.095926284790039, + "loss/core": 1.0540673732757568, + "loss/preference_sft": 0.30823540687561035, + "loss/reference_anchor": 0.5272958278656006, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.3723232746124268, + "rewards/margins": 1.1968507766723633, + "rewards/rejected": 1.175472378730774, + "step": 795 + }, + { + "drift/chosen_abs": 0.22911421954631805, + "drift/rejected_abs": 0.11774910986423492, + "epoch": 0.7643616386002627, + "grad_norm": 4.5, + "learning_rate": 2.388621363640797e-08, + "logits/chosen": -2.1360487937927246, + "logits/rejected": -2.111950159072876, + "logps/chosen": -0.2900012731552124, + "logps/rejected": -0.287829726934433, + "loss": 0.3200365900993347, + "loss/core": 0.30634862184524536, + "loss/preference_sft": 0.2900012731552124, + "loss/reference_anchor": 0.15350642800331116, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.289325475692749, + "rewards/margins": 1.1134859323501587, + "rewards/rejected": 1.1758396625518799, + "step": 800 + }, + { + "drift/chosen_abs": 0.22425393760204315, + "drift/rejected_abs": 0.19786129891872406, + "epoch": 0.7691388988415144, + "grad_norm": 9.3125, + "learning_rate": 2.1613635589349756e-08, + "logits/chosen": -2.0049588680267334, + "logits/rejected": -1.9930200576782227, + "logps/chosen": -0.27656620740890503, + "logps/rejected": -0.271397203207016, + "loss": 0.5283772349357605, + "loss/core": 0.5072607398033142, + "loss/preference_sft": 0.27656620740890503, + "loss/reference_anchor": 0.25389814376831055, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.2400403022766113, + "rewards/margins": 0.2621103823184967, + "rewards/rejected": 1.9779300689697266, + "step": 805 + }, + { + "drift/chosen_abs": 0.1524631530046463, + "drift/rejected_abs": 0.14172717928886414, + "epoch": 0.773916159082766, + "grad_norm": 25.5, + "learning_rate": 1.944975708238708e-08, + "logits/chosen": -2.2034547328948975, + "logits/rejected": -2.108600616455078, + "logps/chosen": -0.30901938676834106, + "logps/rejected": -0.32294124364852905, + "loss": 0.2564198076725006, + "loss/core": 0.2449137419462204, + "loss/preference_sft": 0.30901938676834106, + "loss/reference_anchor": 0.12251242250204086, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.523074746131897, + "rewards/margins": 0.10768821090459824, + "rewards/rejected": 1.4153865575790405, + "step": 810 + }, + { + "drift/chosen_abs": 0.2113420069217682, + "drift/rejected_abs": 0.10915307700634003, + "epoch": 0.7786934193240177, + "grad_norm": 42.25, + "learning_rate": 1.7395608004493884e-08, + "logits/chosen": -2.0760385990142822, + "logits/rejected": -2.164649486541748, + "logps/chosen": -0.28481438755989075, + "logps/rejected": -0.28481075167655945, + "loss": 0.25434860587120056, + "loss/core": 0.24307891726493835, + "loss/preference_sft": 0.28481438755989075, + "loss/reference_anchor": 0.12178117036819458, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.1131508350372314, + "rewards/margins": 1.0244156122207642, + "rewards/rejected": 1.0887352228164673, + "step": 815 + }, + { + "drift/chosen_abs": 0.19611340761184692, + "drift/rejected_abs": 0.07267270982265472, + "epoch": 0.7834706795652693, + "grad_norm": 3.203125, + "learning_rate": 1.5452166019378987e-08, + "logits/chosen": -2.0779826641082764, + "logits/rejected": -2.111990213394165, + "logps/chosen": -0.28833597898483276, + "logps/rejected": -0.3186201751232147, + "loss": 0.25229331851005554, + "loss/core": 0.24105815589427948, + "loss/preference_sft": 0.28833597898483276, + "loss/reference_anchor": 0.12096866220235825, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.9596529006958008, + "rewards/margins": 1.2877837419509888, + "rewards/rejected": 0.6718690991401672, + "step": 820 + }, + { + "drift/chosen_abs": 0.24504788219928741, + "drift/rejected_abs": 0.12149325758218765, + "epoch": 0.788247939806521, + "grad_norm": 61.0, + "learning_rate": 1.3620356100170788e-08, + "logits/chosen": -2.120765209197998, + "logits/rejected": -2.2250657081604004, + "logps/chosen": -0.3034924864768982, + "logps/rejected": -0.2849355936050415, + "loss": 0.31910449266433716, + "loss/core": 0.3053540587425232, + "loss/preference_sft": 0.3034924864768982, + "loss/reference_anchor": 0.15299084782600403, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.4493212699890137, + "rewards/margins": 1.2364730834960938, + "rewards/rejected": 1.2128479480743408, + "step": 825 + }, + { + "drift/chosen_abs": 0.1710740029811859, + "drift/rejected_abs": 0.16245757043361664, + "epoch": 0.7930252000477725, + "grad_norm": 8.75, + "learning_rate": 1.190105008918041e-08, + "logits/chosen": -2.1558587551116943, + "logits/rejected": -2.1049137115478516, + "logps/chosen": -0.280143678188324, + "logps/rejected": -0.3109564781188965, + "loss": 0.34464460611343384, + "loss/core": 0.3301551938056946, + "loss/preference_sft": 0.280143678188324, + "loss/reference_anchor": 0.16517803072929382, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.7102434635162354, + "rewards/margins": 0.08717556297779083, + "rewards/rejected": 1.623067855834961, + "step": 830 + }, + { + "drift/chosen_abs": 0.23977330327033997, + "drift/rejected_abs": 0.11179669201374054, + "epoch": 0.7978024602890242, + "grad_norm": 160.0, + "learning_rate": 1.0295066282951737e-08, + "logits/chosen": -2.0725464820861816, + "logits/rejected": -2.0728602409362793, + "logps/chosen": -0.2556314766407013, + "logps/rejected": -0.3013136684894562, + "loss": 0.39849942922592163, + "loss/core": 0.38221579790115356, + "loss/preference_sft": 0.2556314766407013, + "loss/reference_anchor": 0.19155162572860718, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.395071268081665, + "rewards/margins": 1.566588282585144, + "rewards/rejected": 0.8284828066825867, + "step": 835 + }, + { + "drift/chosen_abs": 0.280472993850708, + "drift/rejected_abs": 0.10291789472103119, + "epoch": 0.8025797205302759, + "grad_norm": 2592.0, + "learning_rate": 8.803169042796765e-09, + "logits/chosen": -2.0131630897521973, + "logits/rejected": -2.0630502700805664, + "logps/chosen": -0.32252293825149536, + "logps/rejected": -0.30228742957115173, + "loss": 0.9987700581550598, + "loss/core": 0.9603092074394226, + "loss/preference_sft": 0.32252293825149536, + "loss/reference_anchor": 0.48055869340896606, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.8026461601257324, + "rewards/margins": 1.7743022441864014, + "rewards/rejected": 1.0283435583114624, + "step": 840 + }, + { + "drift/chosen_abs": 0.14095182716846466, + "drift/rejected_abs": 0.06469657272100449, + "epoch": 0.8073569807715275, + "grad_norm": 3.53125, + "learning_rate": 7.4260684310008815e-09, + "logits/chosen": -2.151334285736084, + "logits/rejected": -2.22589111328125, + "logps/chosen": -0.29466524720191956, + "logps/rejected": -0.3147200047969818, + "loss": 0.1673881560564041, + "loss/core": 0.15918193757534027, + "loss/preference_sft": 0.29466524720191956, + "loss/reference_anchor": 0.07995006442070007, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.4074944257736206, + "rewards/margins": 0.9872743487358093, + "rewards/rejected": 0.4202199876308441, + "step": 845 + }, + { + "drift/chosen_abs": 0.15515586733818054, + "drift/rejected_abs": 0.09092449396848679, + "epoch": 0.8121342410127792, + "grad_norm": 36.0, + "learning_rate": 6.164419872871835e-09, + "logits/chosen": -2.2680633068084717, + "logits/rejected": -2.3154819011688232, + "logps/chosen": -0.25293999910354614, + "logps/rejected": -0.24826927483081818, + "loss": 0.1021793931722641, + "loss/core": 0.09663856029510498, + "loss/preference_sft": 0.25293999910354614, + "loss/reference_anchor": 0.0485837496817112, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.5486667156219482, + "rewards/margins": 0.6556525826454163, + "rewards/rejected": 0.8930143117904663, + "step": 850 + }, + { + "drift/chosen_abs": 0.2674712538719177, + "drift/rejected_abs": 0.12466195970773697, + "epoch": 0.8169115012540308, + "grad_norm": 59.75, + "learning_rate": 5.018823844792602e-09, + "logits/chosen": -1.9834197759628296, + "logits/rejected": -2.0199997425079346, + "logps/chosen": -0.30243033170700073, + "logps/rejected": -0.29364144802093506, + "loss": 0.8800665736198425, + "loss/core": 0.8460347056388855, + "loss/preference_sft": 0.30243033170700073, + "loss/reference_anchor": 0.4235146939754486, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.672776937484741, + "rewards/margins": 1.4277340173721313, + "rewards/rejected": 1.2450430393218994, + "step": 855 + }, + { + "drift/chosen_abs": 0.4073793292045593, + "drift/rejected_abs": 0.19243915379047394, + "epoch": 0.8216887614952825, + "grad_norm": 2.71875, + "learning_rate": 3.989825588427281e-09, + "logits/chosen": -1.9286167621612549, + "logits/rejected": -2.017611026763916, + "logps/chosen": -0.3008403480052948, + "logps/rejected": -0.3007083833217621, + "loss": 1.669266939163208, + "loss/core": 1.606706976890564, + "loss/preference_sft": 0.3008403480052948, + "loss/reference_anchor": 0.8040491342544556, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.073380470275879, + "rewards/margins": 2.1619973182678223, + "rewards/rejected": 1.9113833904266357, + "step": 860 + }, + { + "drift/chosen_abs": 0.24328342080116272, + "drift/rejected_abs": 0.155743807554245, + "epoch": 0.826466021736534, + "grad_norm": 59.5, + "learning_rate": 3.077914851215585e-09, + "logits/chosen": -1.938284158706665, + "logits/rejected": -2.010418176651001, + "logps/chosen": -0.2928522229194641, + "logps/rejected": -0.29682302474975586, + "loss": 0.6092508435249329, + "loss/core": 0.5850942730903625, + "loss/preference_sft": 0.2928522229194641, + "loss/reference_anchor": 0.292802631855011, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.432499647140503, + "rewards/margins": 0.8761860132217407, + "rewards/rejected": 1.5563137531280518, + "step": 865 + }, + { + "drift/chosen_abs": 0.32113441824913025, + "drift/rejected_abs": 0.09325097501277924, + "epoch": 0.8312432819777857, + "grad_norm": 13.8125, + "learning_rate": 2.2835256532794387e-09, + "logits/chosen": -2.088789939880371, + "logits/rejected": -2.097416877746582, + "logps/chosen": -0.30860528349876404, + "logps/rejected": -0.2980922758579254, + "loss": 1.1249186992645264, + "loss/core": 1.0819661617279053, + "loss/preference_sft": 0.30860528349876404, + "loss/reference_anchor": 0.5418386459350586, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.210113525390625, + "rewards/margins": 2.283482074737549, + "rewards/rejected": 0.9266314506530762, + "step": 870 + }, + { + "drift/chosen_abs": 0.1851302534341812, + "drift/rejected_abs": 0.101983942091465, + "epoch": 0.8360205422190374, + "grad_norm": 7.34375, + "learning_rate": 1.6070360808531359e-09, + "logits/chosen": -2.1583588123321533, + "logits/rejected": -2.2066867351531982, + "logps/chosen": -0.30905434489250183, + "logps/rejected": -0.3321419954299927, + "loss": 0.7497228980064392, + "loss/core": 0.7203631401062012, + "loss/preference_sft": 0.30905434489250183, + "loss/reference_anchor": 0.36055710911750793, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8509142398834229, + "rewards/margins": 0.8404647707939148, + "rewards/rejected": 1.0104494094848633, + "step": 875 + }, + { + "drift/chosen_abs": 0.24564549326896667, + "drift/rejected_abs": 0.18484672904014587, + "epoch": 0.840797802460289, + "grad_norm": 56.5, + "learning_rate": 1.0487681063345854e-09, + "logits/chosen": -2.097510814666748, + "logits/rejected": -2.0806679725646973, + "logps/chosen": -0.2908017039299011, + "logps/rejected": -0.3314078450202942, + "loss": 0.9822714924812317, + "loss/core": 0.9446454048156738, + "loss/preference_sft": 0.2908017039299011, + "loss/reference_anchor": 0.4726010262966156, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.4564552307128906, + "rewards/margins": 0.6901001930236816, + "rewards/rejected": 1.7663547992706299, + "step": 880 + }, + { + "drift/chosen_abs": 0.2716746926307678, + "drift/rejected_abs": 0.11291627585887909, + "epoch": 0.8455750627015407, + "grad_norm": 24.875, + "learning_rate": 6.089874350439505e-10, + "logits/chosen": -2.0179717540740967, + "logits/rejected": -2.096179485321045, + "logps/chosen": -0.2834438979625702, + "logps/rejected": -0.27716511487960815, + "loss": 0.4409695565700531, + "loss/core": 0.42294350266456604, + "loss/preference_sft": 0.2834438979625702, + "loss/reference_anchor": 0.21200327575206757, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.7161340713500977, + "rewards/margins": 1.5943241119384766, + "rewards/rejected": 1.1218100786209106, + "step": 885 + }, + { + "drift/chosen_abs": 0.2433289736509323, + "drift/rejected_abs": 0.14475123584270477, + "epoch": 0.8503523229427923, + "grad_norm": 342.0, + "learning_rate": 2.8790337876233305e-10, + "logits/chosen": -1.9500147104263306, + "logits/rejected": -2.0721776485443115, + "logps/chosen": -0.30229297280311584, + "logps/rejected": -0.2738516330718994, + "loss": 0.661720335483551, + "loss/core": 0.6356015205383301, + "loss/preference_sft": 0.30229297280311584, + "loss/reference_anchor": 0.3180215358734131, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.4331092834472656, + "rewards/margins": 0.9898265600204468, + "rewards/rejected": 1.443282961845398, + "step": 890 + }, + { + "drift/chosen_abs": 0.22940556704998016, + "drift/rejected_abs": 0.1513952761888504, + "epoch": 0.855129583184044, + "grad_norm": 50.75, + "learning_rate": 8.566875611068503e-11, + "logits/chosen": -2.022388458251953, + "logits/rejected": -2.0541129112243652, + "logps/chosen": -0.27332523465156555, + "logps/rejected": -0.29117733240127563, + "loss": 0.33594077825546265, + "loss/core": 0.3218122124671936, + "loss/preference_sft": 0.27332523465156555, + "loss/reference_anchor": 0.1610485315322876, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.292048454284668, + "rewards/margins": 0.7850906252861023, + "rewards/rejected": 1.506957769393921, + "step": 895 + }, + { + "drift/chosen_abs": 0.3108707070350647, + "drift/rejected_abs": 0.08576279878616333, + "epoch": 0.8599068434252956, + "grad_norm": 18.5, + "learning_rate": 2.3798198163782478e-12, + "logits/chosen": -2.0334689617156982, + "logits/rejected": -2.154085636138916, + "logps/chosen": -0.26980048418045044, + "logps/rejected": -0.26923516392707825, + "loss": 1.0657157897949219, + "loss/core": 1.0252106189727783, + "loss/preference_sft": 0.26980048418045044, + "loss/reference_anchor": 0.5130892395973206, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 3.1086440086364746, + "rewards/margins": 2.26877498626709, + "rewards/rejected": 0.8398691415786743, + "step": 900 + }, + { + "epoch": 0.8599068434252956, + "step": 900, + "total_flos": 0.0, + "train_loss": 0.7444233691361215, + "train_runtime": 7251.7494, + "train_samples_per_second": 1.986, + "train_steps_per_second": 0.124 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..19f83a5 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:193596bfc183508db2b3df40b7f9967e31127fc2bd47f5d4ee8dc0dea32ab9d3 +size 6993 diff --git a/training_status.json b/training_status.json new file mode 100644 index 0000000..41cf43c --- /dev/null +++ b/training_status.json @@ -0,0 +1,20 @@ +{ + "status": "completed", + "candidate_id": "sppo_b", + "method": "sppo_avg", + "seed": 42, + "gpu": 3, + "pid": 762647, + "config": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/fair_demo_20260715/sweep/candidates/sppo_b/config.yaml", + "dataset": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/avg", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "a2bd72efe9d9", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/a2bd72efe9d9", + "started_at": "2026-07-15T13:38:53+09:00", + "returncode": 0, + "model_complete": true, + "finite_results": true, + "measured_step": null, + "completed_at": "2026-07-15T15:40:43+09:00" +}