commit a0bc76759a014fee8116a39704076b3273912834 Author: ModelHub XC Date: Sat Jul 18 11:48:10 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: promotion/ronpo-qwen3-8b-fair-inpo-avg-s42 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..7d988ee --- /dev/null +++ b/README.md @@ -0,0 +1,12 @@ +--- +base_model: Qwen/Qwen3-8B +library_name: transformers +tags: +- preference-optimization +- ronpo +- qwen3 +--- + +# Qwen3-8B fair-demo checkpoint: inpo_avg + +Validation-selected candidate: `inpo_b`. Exact evaluator, selection, sweep, and training metadata are stored under `experiment/`. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..6ed9b89 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 24.184869588216145, + "train_runtime": 7109.0777, + "train_samples": 16746, + "train_samples_per_second": 2.026, + "train_steps_per_second": 0.127 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..e375ed5 --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +model_name_or_path: /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218 +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/avg: 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: inpo +eta: 0.1 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.075 +preference_sft_weight: 0.0075 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 5.0e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.2 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 42 +gradient_accumulation_steps: 16 +gradient_checkpointing: true +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 1 +per_device_eval_batch_size: 1 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/fair_demo_20260715/sweep/candidates/inpo_b +run_name: qwen3-8b-fair-demo-inpo_b diff --git a/experiment/evaluator_lock.json b/experiment/evaluator_lock.json new file mode 100644 index 0000000..b406c9a --- /dev/null +++ b/experiment/evaluator_lock.json @@ -0,0 +1,85 @@ +{ + "status": "LOCKED_BEFORE_ANY_NEW_METHOD_RANKING", + "locked_at": "2026-07-15T20:20:29+09:00", + "objective_signals": [ + "skywork", + "armo_safety", + "length_conciseness" + ], + "objective_semantics": [ + "helpfulness", + "safety", + "conciseness" + ], + "primary": { + "name": "open_weight_panel_mean_prompt_worst_vs_base", + "definition": "For each prompt and objective, average win=1/tie=0.5/loss=0 over two A/B positions and two judges; take the minimum objective, then mean over prompts.", + "judges": [ + { + "model": "openai/gpt-oss-120b", + "revision": "b5c939de8f754692c1647ca79fbf85e8c1e70f8a" + }, + { + "model": "Qwen/Qwen3-32B", + "revision": "9216db5781bf21249d130ec9da846c4624c16137" + } + ], + "decode": { + "temperature": 0.0, + "top_p": 1.0, + "max_new_tokens": 512, + "seed": 42 + }, + "position_swap": true, + "validation_selection_rule": "Highest eligible mean prompt-level worst panel score within each method; an exact numeric tie is broken by candidate_id lexical order." + }, + "secondary": { + "normalization": "For each locked signal, divide each evaluation prompt's candidate-minus-base raw delta by the population SD of the diagnostic base and matched-control scores; take the prompt-level minimum across objectives, then the mean. No per-prompt min-max.", + "diagnostic_control_scale": { + "skywork": 22.807301785782975, + "armo_safety": 0.3098083353203591, + "length_conciseness": 1.5891969646897472 + }, + "reward_signal_provenance": { + "skywork": { + "model": "Skywork/Skywork-Reward-V2-Llama-3.1-8B", + "revision": "cba2f842f3f1af2f1b2f0d35e794d789976390c5", + "semantics": "helpfulness" + }, + "armo_safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "revision": "eb2676d20da2f2d41082289d23c59b9f7427f955", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "length_conciseness": { + "deterministic": "-log1p(response_word_count)", + "tokenization": "Python str.split whitespace words" + } + }, + "report_raw_paired_deltas": true + }, + "bootstrap": { + "paired_prompt_resamples": 2000, + "seed": 42, + "interval": "percentile_95" + }, + "power": { + "target_absolute_effect": 0.05, + "paired_sd": 0.13831629563357775, + "required_prompts_80pct_power": 61, + "planned_fresh_test_prompts": 1024, + "alpha_two_sided": 0.05, + "power": 0.8 + }, + "fresh_test_source": { + "dataset": "HuggingFaceH4/ultrachat_200k", + "revision": "8049631c405ae6576f93f445c6b8166f76f5505a", + "split": "test_sft" + }, + "diagnostic_summary_sha256": "e365ed814926ac5c17777e0fc65d0299a45b81bd390542889bbd55b5453d4ffe", + "judge_diagnostic_lock_sha256": "fce2d97806053cabdf3d358806f4baa80df7f89d71d8c1c298883327d4e02635", + "prereg_sha256": "71cbd1ac17c3e807f2ece1cb0a82374bbe74edb654c2845b97c3222ed680f0d0", + "method_ranking_computed": false, + "spent_sealed_split_touched": false +} diff --git a/experiment/selection_lock.json b/experiment/selection_lock.json new file mode 100644 index 0000000..bae8b9c --- /dev/null +++ b/experiment/selection_lock.json @@ -0,0 +1,99 @@ +{ + "status": "VALIDATION_SELECTION_LOCKED_BEFORE_FRESH_TEST", + "locked_at": "2026-07-15T20:37:11+09:00", + "selection_metric": "open_weight_panel_mean_prompt_worst_vs_base", + "tie_break": "candidate_id lexical order", + "selected_by_method": { + "dpo": { + "candidate_id": "dpo_b", + "validation_primary": 0.40625, + "validation_primary_ci95": [ + 0.3837890625, + 0.427734375 + ], + "eligible_candidates": [ + "dpo_a", + "dpo_b" + ] + }, + "ht_mnpo_helpfulness": { + "candidate_id": "ht_help_b", + "validation_primary": 0.4072265625, + "validation_primary_ci95": [ + 0.3828125, + 0.4296875 + ], + "eligible_candidates": [ + "ht_help_b" + ] + }, + "ht_mnpo_safety": { + "candidate_id": "ht_safety_a", + "validation_primary": 0.4150390625, + "validation_primary_ci95": [ + 0.3925537109375, + 0.4384765625 + ], + "eligible_candidates": [ + "ht_safety_a", + "ht_safety_b" + ] + }, + "inpo_avg": { + "candidate_id": "inpo_b", + "validation_primary": 0.40625, + "validation_primary_ci95": [ + 0.3818359375, + 0.427734375 + ], + "eligible_candidates": [ + "inpo_b" + ] + }, + "ronpo_full_expect": { + "candidate_id": "ronpo_full_a", + "validation_primary": 0.4033203125, + "validation_primary_ci95": [ + 0.37890625, + 0.42580566406249987 + ], + "eligible_candidates": [ + "ronpo_full_a", + "ronpo_full_b" + ] + }, + "ronpo_k_only": { + "candidate_id": "ronpo_top_a", + "validation_primary": 0.412109375, + "validation_primary_ci95": [ + 0.3876953125, + 0.43557128906249987 + ], + "eligible_candidates": [ + "ronpo_top_a" + ] + }, + "sppo_avg": { + "candidate_id": "sppo_b", + "validation_primary": 0.4111328125, + "validation_primary_ci95": [ + 0.3866943359375, + 0.4345703125 + ], + "eligible_candidates": [ + "sppo_b" + ] + } + }, + "selected_ronpo_overall": "ronpo_top_a", + "failed_methods": [ + "ht_mnpo_conciseness", + "ipo", + "simpo" + ], + "panel_summary_sha256": "8850f23e16cadf56a99e3d562d224ce5337ab36fa588d8a843382853e6ad17bd", + "evaluator_lock_sha256": "ab7b12c84932ed57acfec7edbcbc1ff1df5e9acd203869b1458343adf6e5b2c8", + "grid_sha256": "abfd982ef10c3bc71364752241150286ebf06be5e97f087596adf80e55d459e0", + "fresh_test_opened": false, + "spent_sealed_split_touched": false +} diff --git a/experiment/sweep_grid.json b/experiment/sweep_grid.json new file mode 100644 index 0000000..1ee759f --- /dev/null +++ b/experiment/sweep_grid.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "status": "frozen_before_training_and_validation_ranking", + "frozen_at_kst": "2026-07-15T09:01:45+09:00", + "seed": 42, + "common": { + "optimizer_steps": 900, + "effective_batch_size": 16, + "per_device_train_batch_size": 1, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "bf16": true, + "attn_implementation": "sdpa", + "cudnn_sdpa": false, + "lr_scheduler_type": "cosine", + "max_length": 2048, + "max_prompt_length": 1800, + "save_total_limit": 1, + "wandb_entity": "promotion-kim", + "wandb_project": "mnpo", + "wandb_group": "qwen3-8b-fair-demo-20260715" + }, + "budget_rule": "Exactly two configs per reported method; no extension after validation rankings are visible.", + "candidates": [ + {"id":"ronpo_full_a","method":"ronpo_full_expect","dataset":"ronpo_full_expect_kall","learning_rate":1e-7,"warmup_ratio":0.15,"ronpo_alpha":0.25,"ronpo_tau":0.10,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075,"expected_support_k":0}, + {"id":"ronpo_full_b","method":"ronpo_full_expect","dataset":"ronpo_full_expect_k6","learning_rate":2.5e-7,"warmup_ratio":0.20,"ronpo_alpha":0.35,"ronpo_tau":0.05,"reference_anchor_weight":0.10,"preference_sft_weight":0.01,"expected_support_k":6}, + {"id":"ronpo_top_a","method":"ronpo_k_only","dataset":"ronpo_k_only_k1","learning_rate":5e-8,"warmup_ratio":0.20,"ronpo_alpha":0.15,"ronpo_tau":0.10,"reference_anchor_weight":0.10,"preference_sft_weight":0.01,"expected_support_k":1}, + {"id":"ronpo_top_b","method":"ronpo_k_only","dataset":"ronpo_k_only_k2","learning_rate":1e-7,"warmup_ratio":0.15,"ronpo_alpha":0.25,"ronpo_tau":0.10,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075,"expected_support_k":2}, + {"id":"dpo_a","method":"dpo","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.10,"dpo_beta":0.05,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"dpo_b","method":"dpo","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.15,"dpo_beta":0.10,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ipo_a","method":"ipo","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.15,"dpo_beta":0.20,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ipo_b","method":"ipo","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.20,"dpo_beta":0.50,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075}, + {"id":"simpo_a","method":"simpo","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.10,"simpo_beta":1.0,"simpo_gamma":0.3,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"simpo_b","method":"simpo","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.15,"simpo_beta":2.0,"simpo_gamma":0.6,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"sppo_a","method":"sppo_avg","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.15,"eta":0.005,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"sppo_b","method":"sppo_avg","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.20,"eta":0.010,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075}, + {"id":"inpo_a","method":"inpo_avg","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.15,"eta":0.050,"ratio":0.3333,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"inpo_b","method":"inpo_avg","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.20,"eta":0.100,"ratio":0.3333,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075}, + {"id":"ht_help_a","method":"ht_mnpo_helpfulness","dataset":"ht_mnpo_helpfulness","learning_rate":2.5e-7,"warmup_ratio":0.10,"eta":0.005,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"ht_help_b","method":"ht_mnpo_helpfulness","dataset":"ht_mnpo_helpfulness","learning_rate":5e-7,"warmup_ratio":0.15,"eta":0.010,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ht_safety_a","method":"ht_mnpo_safety","dataset":"ht_mnpo_safety","learning_rate":2.5e-7,"warmup_ratio":0.10,"eta":0.005,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"ht_safety_b","method":"ht_mnpo_safety","dataset":"ht_mnpo_safety","learning_rate":5e-7,"warmup_ratio":0.15,"eta":0.010,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}, + {"id":"ht_concise_a","method":"ht_mnpo_conciseness","dataset":"ht_mnpo_conciseness","learning_rate":2.5e-7,"warmup_ratio":0.10,"eta":0.005,"reference_anchor_weight":0.02,"preference_sft_weight":0.002}, + {"id":"ht_concise_b","method":"ht_mnpo_conciseness","dataset":"ht_mnpo_conciseness","learning_rate":5e-7,"warmup_ratio":0.15,"eta":0.010,"reference_anchor_weight":0.05,"preference_sft_weight":0.005} + ] +} diff --git a/experiment/training_status.json b/experiment/training_status.json new file mode 100644 index 0000000..97926e3 --- /dev/null +++ b/experiment/training_status.json @@ -0,0 +1,20 @@ +{ + "status": "completed", + "candidate_id": "inpo_b", + "method": "inpo_avg", + "seed": 42, + "gpu": 0, + "pid": 1079779, + "config": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/fair_demo_20260715/sweep/candidates/inpo_b/config.yaml", + "dataset": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/avg", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "098086245ba4", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/098086245ba4", + "started_at": "2026-07-15T15:36:21+09:00", + "returncode": 0, + "model_complete": true, + "finite_results": true, + "measured_step": null, + "completed_at": "2026-07-15T17:35:44+09:00" +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..bf7795e --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13ecf1fe52b368464f8bba1caa7719d0c7a3e6b555c26ef3872918666fcd836c +size 16381517208 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..e2cb12d --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": true, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..6ed9b89 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 24.184869588216145, + "train_runtime": 7109.0777, + "train_samples": 16746, + "train_samples_per_second": 2.026, + "train_steps_per_second": 0.127 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..bb7bbc4 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,3643 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8599068434252956, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "drift/chosen_abs": 0.24268421530723572, + "drift/rejected_abs": 0.09689199179410934, + "epoch": 0.004777260241251642, + "grad_norm": 158.0, + "learning_rate": 1.111111111111111e-08, + "logits/chosen": -1.9987547397613525, + "logits/rejected": -2.0148520469665527, + "logps/chosen": -0.27876606583595276, + "logps/rejected": -0.2827395498752594, + "loss": 23.720317840576172, + "loss/core": 23.703845977783203, + "loss/preference_sft": 0.27876606583595276, + "loss/reference_anchor": 0.19172723591327667, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.4264075756073, + "rewards/margins": 1.473706603050232, + "rewards/rejected": 0.9527010917663574, + "step": 5 + }, + { + "drift/chosen_abs": 0.1691116988658905, + "drift/rejected_abs": 0.1621948778629303, + "epoch": 0.009554520482503284, + "grad_norm": 1336.0, + "learning_rate": 2.5e-08, + "logits/chosen": -2.345815658569336, + "logits/rejected": -2.392836093902588, + "logps/chosen": -0.28935980796813965, + "logps/rejected": -0.2881030738353729, + "loss": 25.067256927490234, + "loss/core": 25.026386260986328, + "loss/preference_sft": 0.28935980796813965, + "loss/reference_anchor": 0.5160242915153503, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.6900745630264282, + "rewards/margins": 0.07324112951755524, + "rewards/rejected": 1.6168334484100342, + "step": 10 + }, + { + "drift/chosen_abs": 0.21343913674354553, + "drift/rejected_abs": 0.11456535756587982, + "epoch": 0.014331780723754926, + "grad_norm": 118.0, + "learning_rate": 3.888888888888889e-08, + "logits/chosen": -2.2311110496520996, + "logits/rejected": -2.1946258544921875, + "logps/chosen": -0.277534544467926, + "logps/rejected": -0.28368276357650757, + "loss": 24.18115997314453, + "loss/core": 24.16556167602539, + "loss/preference_sft": 0.277534544467926, + "loss/reference_anchor": 0.1802430897951126, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.1329588890075684, + "rewards/margins": 0.9891046285629272, + "rewards/rejected": 1.1438543796539307, + "step": 15 + }, + { + "drift/chosen_abs": 0.19140727818012238, + "drift/rejected_abs": 0.08797380328178406, + "epoch": 0.01910904096500657, + "grad_norm": 104.5, + "learning_rate": 5.2777777777777776e-08, + "logits/chosen": -2.1776981353759766, + "logits/rejected": -2.2193965911865234, + "logps/chosen": -0.30152779817581177, + "logps/rejected": -0.2865992486476898, + "loss": 24.163585662841797, + "loss/core": 24.14507484436035, + "loss/preference_sft": 0.30152779817581177, + "loss/reference_anchor": 0.21664480865001678, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9140727519989014, + "rewards/margins": 1.0348074436187744, + "rewards/rejected": 0.8792654275894165, + "step": 20 + }, + { + "drift/chosen_abs": 0.3856329321861267, + "drift/rejected_abs": 0.16464610397815704, + "epoch": 0.02388630120625821, + "grad_norm": 86.5, + "learning_rate": 6.666666666666667e-08, + "logits/chosen": -2.0732712745666504, + "logits/rejected": -2.13415789604187, + "logps/chosen": -0.2700391411781311, + "logps/rejected": -0.2578909993171692, + "loss": 24.2296142578125, + "loss/core": 24.161888122558594, + "loss/preference_sft": 0.2700391411781311, + "loss/reference_anchor": 0.8760007619857788, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.8547744750976562, + "rewards/margins": 2.2091641426086426, + "rewards/rejected": 1.6456102132797241, + "step": 25 + }, + { + "drift/chosen_abs": 0.22750432789325714, + "drift/rejected_abs": 0.11023910343647003, + "epoch": 0.028663561447509853, + "grad_norm": 77.5, + "learning_rate": 8.055555555555555e-08, + "logits/chosen": -2.3493716716766357, + "logits/rejected": -2.3317930698394775, + "logps/chosen": -0.3004586100578308, + "logps/rejected": -0.3256753087043762, + "loss": 24.168228149414062, + "loss/core": 24.141149520874023, + "loss/preference_sft": 0.3004586100578308, + "loss/reference_anchor": 0.33102959394454956, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.2747082710266113, + "rewards/margins": 1.2620917558670044, + "rewards/rejected": 1.0126166343688965, + "step": 30 + }, + { + "drift/chosen_abs": 0.26583537459373474, + "drift/rejected_abs": 0.11596353352069855, + "epoch": 0.033440821688761495, + "grad_norm": 50.25, + "learning_rate": 9.444444444444444e-08, + "logits/chosen": -2.203460693359375, + "logits/rejected": -2.18483829498291, + "logps/chosen": -0.2677518129348755, + "logps/rejected": -0.26530081033706665, + "loss": 23.941822052001953, + "loss/core": 23.910053253173828, + "loss/preference_sft": 0.2677518129348755, + "loss/reference_anchor": 0.3968318998813629, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.6582398414611816, + "rewards/margins": 1.4989526271820068, + "rewards/rejected": 1.1592872142791748, + "step": 35 + }, + { + "drift/chosen_abs": 0.44815701246261597, + "drift/rejected_abs": 0.16203448176383972, + "epoch": 0.03821808193001314, + "grad_norm": 151.0, + "learning_rate": 1.0833333333333334e-07, + "logits/chosen": -2.002789258956909, + "logits/rejected": -2.062656879425049, + "logps/chosen": -0.2923213839530945, + "logps/rejected": -0.2835318148136139, + "loss": 23.72269058227539, + "loss/core": 23.646509170532227, + "loss/preference_sft": 0.2923213839530945, + "loss/reference_anchor": 0.9865428805351257, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.481132984161377, + "rewards/margins": 2.8950064182281494, + "rewards/rejected": 1.5861257314682007, + "step": 40 + }, + { + "drift/chosen_abs": 0.20982900261878967, + "drift/rejected_abs": 0.08536230772733688, + "epoch": 0.04299534217126478, + "grad_norm": 65.5, + "learning_rate": 1.2222222222222222e-07, + "logits/chosen": -2.199631452560425, + "logits/rejected": -2.363292694091797, + "logps/chosen": -0.2886757254600525, + "logps/rejected": -0.2883760929107666, + "loss": 23.874040603637695, + "loss/core": 23.861454010009766, + "loss/preference_sft": 0.2886757254600525, + "loss/reference_anchor": 0.13895340263843536, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.0968005657196045, + "rewards/margins": 1.254421353340149, + "rewards/rejected": 0.8423792123794556, + "step": 45 + }, + { + "drift/chosen_abs": 0.2884393632411957, + "drift/rejected_abs": 0.1451389044523239, + "epoch": 0.04777260241251642, + "grad_norm": 402.0, + "learning_rate": 1.3611111111111108e-07, + "logits/chosen": -2.166757106781006, + "logits/rejected": -2.1628246307373047, + "logps/chosen": -0.28129029273986816, + "logps/rejected": -0.2850467562675476, + "loss": 24.05858612060547, + "loss/core": 24.029706954956055, + "loss/preference_sft": 0.28129029273986816, + "loss/reference_anchor": 0.3569650650024414, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.8835978507995605, + "rewards/margins": 1.4343656301498413, + "rewards/rejected": 1.4492321014404297, + "step": 50 + }, + { + "drift/chosen_abs": 0.22632427513599396, + "drift/rejected_abs": 0.20729069411754608, + "epoch": 0.05254986265376806, + "grad_norm": 32.25, + "learning_rate": 1.5e-07, + "logits/chosen": -2.1963820457458496, + "logits/rejected": -2.1848561763763428, + "logps/chosen": -0.28860360383987427, + "logps/rejected": -0.28664013743400574, + "loss": 25.665725708007812, + "loss/core": 25.60601806640625, + "loss/preference_sft": 0.28860360383987427, + "loss/reference_anchor": 0.7672308087348938, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.262362003326416, + "rewards/margins": 0.20232732594013214, + "rewards/rejected": 2.0600345134735107, + "step": 55 + }, + { + "drift/chosen_abs": 0.21578149497509003, + "drift/rejected_abs": 0.1569838523864746, + "epoch": 0.057327122895019705, + "grad_norm": 1848.0, + "learning_rate": 1.6388888888888888e-07, + "logits/chosen": -2.0918967723846436, + "logits/rejected": -2.1219000816345215, + "logps/chosen": -0.27454960346221924, + "logps/rejected": -0.2835273742675781, + "loss": 24.56880760192871, + "loss/core": 24.553863525390625, + "loss/preference_sft": 0.27454960346221924, + "loss/reference_anchor": 0.1717853546142578, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.1553452014923096, + "rewards/margins": 0.5968301892280579, + "rewards/rejected": 1.5585150718688965, + "step": 60 + }, + { + "drift/chosen_abs": 0.29738813638687134, + "drift/rejected_abs": 0.11811880022287369, + "epoch": 0.06210438313627135, + "grad_norm": 66.0, + "learning_rate": 1.7777777777777776e-07, + "logits/chosen": -1.962087869644165, + "logits/rejected": -2.0643563270568848, + "logps/chosen": -0.30899226665496826, + "logps/rejected": -0.3064674437046051, + "loss": 23.558834075927734, + "loss/core": 23.519794464111328, + "loss/preference_sft": 0.30899226665496826, + "loss/reference_anchor": 0.48962312936782837, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.9719326496124268, + "rewards/margins": 1.8016983270645142, + "rewards/rejected": 1.170233964920044, + "step": 65 + }, + { + "drift/chosen_abs": 0.3609868586063385, + "drift/rejected_abs": 0.228200763463974, + "epoch": 0.06688164337752299, + "grad_norm": 229.0, + "learning_rate": 1.9166666666666668e-07, + "logits/chosen": -1.9723434448242188, + "logits/rejected": -1.9571367502212524, + "logps/chosen": -0.3828391134738922, + "logps/rejected": -0.297024130821228, + "loss": 24.988039016723633, + "loss/core": 24.914230346679688, + "loss/preference_sft": 0.3828391134738922, + "loss/reference_anchor": 0.9458276629447937, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.6087005138397217, + "rewards/margins": 1.4257218837738037, + "rewards/rejected": 2.182979106903076, + "step": 70 + }, + { + "drift/chosen_abs": 0.2607436776161194, + "drift/rejected_abs": 0.1599571406841278, + "epoch": 0.07165890361877464, + "grad_norm": 56.5, + "learning_rate": 2.0555555555555553e-07, + "logits/chosen": -2.0862269401550293, + "logits/rejected": -2.050337553024292, + "logps/chosen": -0.31514787673950195, + "logps/rejected": -0.28866785764694214, + "loss": 24.35919189453125, + "loss/core": 24.329198837280273, + "loss/preference_sft": 0.31514787673950195, + "loss/reference_anchor": 0.36842408776283264, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.6053974628448486, + "rewards/margins": 1.00749671459198, + "rewards/rejected": 1.5979008674621582, + "step": 75 + }, + { + "drift/chosen_abs": 0.3240804970264435, + "drift/rejected_abs": 0.12827441096305847, + "epoch": 0.07643616386002627, + "grad_norm": 55.5, + "learning_rate": 2.1944444444444442e-07, + "logits/chosen": -2.0698435306549072, + "logits/rejected": -2.1897106170654297, + "logps/chosen": -0.26063475012779236, + "logps/rejected": -0.2740657925605774, + "loss": 23.68960189819336, + "loss/core": 23.653255462646484, + "loss/preference_sft": 0.26063475012779236, + "loss/reference_anchor": 0.45853161811828613, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.2406482696533203, + "rewards/margins": 1.9891496896743774, + "rewards/rejected": 1.2514985799789429, + "step": 80 + }, + { + "drift/chosen_abs": 0.3498837351799011, + "drift/rejected_abs": 0.13262492418289185, + "epoch": 0.08121342410127792, + "grad_norm": 79.0, + "learning_rate": 2.3333333333333333e-07, + "logits/chosen": -2.189903497695923, + "logits/rejected": -2.2383172512054443, + "logps/chosen": -0.3075157105922699, + "logps/rejected": -0.2782510817050934, + "loss": 23.684247970581055, + "loss/core": 23.61785125732422, + "loss/preference_sft": 0.3075157105922699, + "loss/reference_anchor": 0.8545700311660767, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.498150587081909, + "rewards/margins": 2.172797918319702, + "rewards/rejected": 1.3253527879714966, + "step": 85 + }, + { + "drift/chosen_abs": 0.17537356913089752, + "drift/rejected_abs": 0.08442483097314835, + "epoch": 0.08599068434252956, + "grad_norm": 31.875, + "learning_rate": 2.4722222222222224e-07, + "logits/chosen": -2.4203712940216064, + "logits/rejected": -2.4085536003112793, + "logps/chosen": -0.28136521577835083, + "logps/rejected": -0.296181857585907, + "loss": 24.193899154663086, + "loss/core": 24.180316925048828, + "loss/preference_sft": 0.28136521577835083, + "loss/reference_anchor": 0.15298345685005188, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.7537355422973633, + "rewards/margins": 0.9103192090988159, + "rewards/rejected": 0.8434165716171265, + "step": 90 + }, + { + "drift/chosen_abs": 0.3204111158847809, + "drift/rejected_abs": 0.21080467104911804, + "epoch": 0.09076794458378121, + "grad_norm": 118.0, + "learning_rate": 2.6111111111111113e-07, + "logits/chosen": -2.0531086921691895, + "logits/rejected": -2.108947992324829, + "logps/chosen": -0.267969012260437, + "logps/rejected": -0.27856558561325073, + "loss": 24.32411766052246, + "loss/core": 24.288156509399414, + "loss/preference_sft": 0.267969012260437, + "loss/reference_anchor": 0.4526856541633606, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.2040367126464844, + "rewards/margins": 1.0959895849227905, + "rewards/rejected": 2.1080470085144043, + "step": 95 + }, + { + "drift/chosen_abs": 0.2465881109237671, + "drift/rejected_abs": 0.1286160796880722, + "epoch": 0.09554520482503284, + "grad_norm": 73.5, + "learning_rate": 2.75e-07, + "logits/chosen": -2.2330098152160645, + "logits/rejected": -2.2795863151550293, + "logps/chosen": -0.2569820284843445, + "logps/rejected": -0.2466917484998703, + "loss": 23.988306045532227, + "loss/core": 23.972742080688477, + "loss/preference_sft": 0.2569820284843445, + "loss/reference_anchor": 0.18182215094566345, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.4647648334503174, + "rewards/margins": 1.1787031888961792, + "rewards/rejected": 1.2860618829727173, + "step": 100 + }, + { + "drift/chosen_abs": 0.2001236379146576, + "drift/rejected_abs": 0.079637810587883, + "epoch": 0.10032246506628449, + "grad_norm": 50.25, + "learning_rate": 2.8888888888888885e-07, + "logits/chosen": -2.3382439613342285, + "logits/rejected": -2.424198865890503, + "logps/chosen": -0.2838006615638733, + "logps/rejected": -0.28019005060195923, + "loss": 24.08165168762207, + "loss/core": 24.064653396606445, + "loss/preference_sft": 0.2838006615638733, + "loss/reference_anchor": 0.1982627958059311, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.0012366771698, + "rewards/margins": 1.207162618637085, + "rewards/rejected": 0.7940739393234253, + "step": 105 + }, + { + "drift/chosen_abs": 0.4472831189632416, + "drift/rejected_abs": 0.18904665112495422, + "epoch": 0.10509972530753613, + "grad_norm": 71.0, + "learning_rate": 3.0277777777777773e-07, + "logits/chosen": -1.9077485799789429, + "logits/rejected": -1.8608009815216064, + "logps/chosen": -0.3383413255214691, + "logps/rejected": -0.2978266477584839, + "loss": 23.436580657958984, + "loss/core": 23.36850357055664, + "loss/preference_sft": 0.3383413255214691, + "loss/reference_anchor": 0.8738870620727539, + "rewards/accuracies": 0.875, + "rewards/chosen": 4.471864223480225, + "rewards/margins": 2.584223508834839, + "rewards/rejected": 1.8876409530639648, + "step": 110 + }, + { + "drift/chosen_abs": 0.17792600393295288, + "drift/rejected_abs": 0.14627966284751892, + "epoch": 0.10987698554878778, + "grad_norm": 334.0, + "learning_rate": 3.166666666666666e-07, + "logits/chosen": -2.088899850845337, + "logits/rejected": -2.032494068145752, + "logps/chosen": -0.30003082752227783, + "logps/rejected": -0.2972942888736725, + "loss": 24.94771957397461, + "loss/core": 24.928913116455078, + "loss/preference_sft": 0.30003082752227783, + "loss/reference_anchor": 0.22078463435173035, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.7790625095367432, + "rewards/margins": 0.31695252656936646, + "rewards/rejected": 1.4621098041534424, + "step": 115 + }, + { + "drift/chosen_abs": 0.18117384612560272, + "drift/rejected_abs": 0.11267610639333725, + "epoch": 0.11465424579003941, + "grad_norm": 768.0, + "learning_rate": 3.3055555555555556e-07, + "logits/chosen": -2.0376081466674805, + "logits/rejected": -2.0747933387756348, + "logps/chosen": -0.3019946813583374, + "logps/rejected": -0.29592204093933105, + "loss": 24.376644134521484, + "loss/core": 24.367816925048828, + "loss/preference_sft": 0.3019946813583374, + "loss/reference_anchor": 0.08752311021089554, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8115949630737305, + "rewards/margins": 0.6873809099197388, + "rewards/rejected": 1.1242139339447021, + "step": 120 + }, + { + "drift/chosen_abs": 0.20648498833179474, + "drift/rejected_abs": 0.10197492688894272, + "epoch": 0.11943150603129106, + "grad_norm": 50.75, + "learning_rate": 3.4444444444444444e-07, + "logits/chosen": -2.1082651615142822, + "logits/rejected": -2.139118194580078, + "logps/chosen": -0.29931074380874634, + "logps/rejected": -0.30651599168777466, + "loss": 24.086589813232422, + "loss/core": 24.07431983947754, + "loss/preference_sft": 0.29931074380874634, + "loss/reference_anchor": 0.13366781175136566, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.0623745918273926, + "rewards/margins": 1.0441718101501465, + "rewards/rejected": 1.018202543258667, + "step": 125 + }, + { + "drift/chosen_abs": 0.18083539605140686, + "drift/rejected_abs": 0.08944060653448105, + "epoch": 0.1242087662725427, + "grad_norm": 100.5, + "learning_rate": 3.583333333333333e-07, + "logits/chosen": -2.0009732246398926, + "logits/rejected": -2.0936858654022217, + "logps/chosen": -0.27988457679748535, + "logps/rejected": -0.278131902217865, + "loss": 24.129959106445312, + "loss/core": 24.122276306152344, + "loss/preference_sft": 0.27988457679748535, + "loss/reference_anchor": 0.07443337142467499, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.8083540201187134, + "rewards/margins": 0.9148699641227722, + "rewards/rejected": 0.8934842348098755, + "step": 130 + }, + { + "drift/chosen_abs": 0.33803224563598633, + "drift/rejected_abs": 0.13368059694766998, + "epoch": 0.12898602651379434, + "grad_norm": 134.0, + "learning_rate": 3.722222222222222e-07, + "logits/chosen": -2.0908658504486084, + "logits/rejected": -2.1650338172912598, + "logps/chosen": -0.3015690743923187, + "logps/rejected": -0.2998407483100891, + "loss": 23.31682777404785, + "loss/core": 23.28109359741211, + "loss/preference_sft": 0.3015690743923187, + "loss/reference_anchor": 0.446286141872406, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.3794403076171875, + "rewards/margins": 2.081610918045044, + "rewards/rejected": 1.2978298664093018, + "step": 135 + }, + { + "drift/chosen_abs": 0.16290757060050964, + "drift/rejected_abs": 0.07148541510105133, + "epoch": 0.13376328675504598, + "grad_norm": 60.25, + "learning_rate": 3.861111111111111e-07, + "logits/chosen": -2.4541397094726562, + "logits/rejected": -2.5143344402313232, + "logps/chosen": -0.28250986337661743, + "logps/rejected": -0.2895656228065491, + "loss": 24.135583877563477, + "loss/core": 24.1291446685791, + "loss/preference_sft": 0.28250986337661743, + "loss/reference_anchor": 0.05761883780360222, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.6281633377075195, + "rewards/margins": 0.9143427610397339, + "rewards/rejected": 0.7138205170631409, + "step": 140 + }, + { + "drift/chosen_abs": 0.3603426516056061, + "drift/rejected_abs": 0.17863936722278595, + "epoch": 0.13854054699629761, + "grad_norm": 96.0, + "learning_rate": 4e-07, + "logits/chosen": -1.77964186668396, + "logits/rejected": -1.8101189136505127, + "logps/chosen": -0.30144834518432617, + "logps/rejected": -0.30214372277259827, + "loss": 23.7529239654541, + "loss/core": 23.6727352142334, + "loss/preference_sft": 0.30144834518432617, + "loss/reference_anchor": 1.0389857292175293, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.6030845642089844, + "rewards/margins": 1.8218176364898682, + "rewards/rejected": 1.7812671661376953, + "step": 145 + }, + { + "drift/chosen_abs": 0.36440587043762207, + "drift/rejected_abs": 0.1463252753019333, + "epoch": 0.14331780723754928, + "grad_norm": 98.5, + "learning_rate": 4.1388888888888887e-07, + "logits/chosen": -2.2338650226593018, + "logits/rejected": -2.282841444015503, + "logps/chosen": -0.31720206141471863, + "logps/rejected": -0.3081175684928894, + "loss": 23.425756454467773, + "loss/core": 23.36142349243164, + "loss/preference_sft": 0.31720206141471863, + "loss/reference_anchor": 0.8260464668273926, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.640651226043701, + "rewards/margins": 2.182239055633545, + "rewards/rejected": 1.4584124088287354, + "step": 150 + }, + { + "drift/chosen_abs": 0.28685861825942993, + "drift/rejected_abs": 0.07811577618122101, + "epoch": 0.1480950674788009, + "grad_norm": 77.5, + "learning_rate": 4.2777777777777775e-07, + "logits/chosen": -2.0995328426361084, + "logits/rejected": -2.177826166152954, + "logps/chosen": -0.2826915383338928, + "logps/rejected": -0.29879477620124817, + "loss": 23.43790054321289, + "loss/core": 23.408916473388672, + "loss/preference_sft": 0.2826915383338928, + "loss/reference_anchor": 0.3581943213939667, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.8685860633850098, + "rewards/margins": 2.106802225112915, + "rewards/rejected": 0.7617841958999634, + "step": 155 + }, + { + "drift/chosen_abs": 0.31482672691345215, + "drift/rejected_abs": 0.1407286822795868, + "epoch": 0.15287232772005255, + "grad_norm": 200.0, + "learning_rate": 4.4166666666666664e-07, + "logits/chosen": -2.05705189704895, + "logits/rejected": -2.0315186977386475, + "logps/chosen": -0.31071725487709045, + "logps/rejected": -0.31473129987716675, + "loss": 24.094379425048828, + "loss/core": 24.039676666259766, + "loss/preference_sft": 0.31071725487709045, + "loss/reference_anchor": 0.698303759098053, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.1482670307159424, + "rewards/margins": 1.7411534786224365, + "rewards/rejected": 1.4071136713027954, + "step": 160 + }, + { + "drift/chosen_abs": 0.1602366864681244, + "drift/rejected_abs": 0.07925491034984589, + "epoch": 0.15764958796130418, + "grad_norm": 80.5, + "learning_rate": 4.555555555555555e-07, + "logits/chosen": -2.223989725112915, + "logits/rejected": -2.1556718349456787, + "logps/chosen": -0.3111506998538971, + "logps/rejected": -0.30567336082458496, + "loss": 24.250789642333984, + "loss/core": 24.241710662841797, + "loss/preference_sft": 0.3111506998538971, + "loss/reference_anchor": 0.08993285894393921, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.601798415184021, + "rewards/margins": 0.8125208020210266, + "rewards/rejected": 0.7892775535583496, + "step": 165 + }, + { + "drift/chosen_abs": 0.2597414255142212, + "drift/rejected_abs": 0.2784227728843689, + "epoch": 0.16242684820255585, + "grad_norm": 81.5, + "learning_rate": 4.694444444444444e-07, + "logits/chosen": -2.187836170196533, + "logits/rejected": -2.132575511932373, + "logps/chosen": -0.27488964796066284, + "logps/rejected": -0.2585277855396271, + "loss": 26.337051391601562, + "loss/core": 26.263935089111328, + "loss/preference_sft": 0.27488964796066284, + "loss/reference_anchor": 0.9473751783370972, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.59535551071167, + "rewards/margins": -0.186926007270813, + "rewards/rejected": 2.7822816371917725, + "step": 170 + }, + { + "drift/chosen_abs": 0.2583083212375641, + "drift/rejected_abs": 0.12779515981674194, + "epoch": 0.16720410844380748, + "grad_norm": 372.0, + "learning_rate": 4.833333333333333e-07, + "logits/chosen": -2.1770424842834473, + "logits/rejected": -2.216740608215332, + "logps/chosen": -0.2869277000427246, + "logps/rejected": -0.30118730664253235, + "loss": 23.88767433166504, + "loss/core": 23.843341827392578, + "loss/preference_sft": 0.2869277000427246, + "loss/reference_anchor": 0.5624145865440369, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 2.583082914352417, + "rewards/margins": 1.3182199001312256, + "rewards/rejected": 1.2648632526397705, + "step": 175 + }, + { + "drift/chosen_abs": 0.3239268660545349, + "drift/rejected_abs": 0.1492597758769989, + "epoch": 0.17198136868505912, + "grad_norm": 280.0, + "learning_rate": 4.972222222222222e-07, + "logits/chosen": -2.201796054840088, + "logits/rejected": -2.111999988555908, + "logps/chosen": -0.27172645926475525, + "logps/rejected": -0.2618543803691864, + "loss": 23.988094329833984, + "loss/core": 23.94233512878418, + "loss/preference_sft": 0.27172645926475525, + "loss/reference_anchor": 0.5829019546508789, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.2370712757110596, + "rewards/margins": 1.7461532354354858, + "rewards/rejected": 1.4909178018569946, + "step": 180 + }, + { + "drift/chosen_abs": 0.10453528165817261, + "drift/rejected_abs": 0.06307251751422882, + "epoch": 0.17675862892631075, + "grad_norm": 62.0, + "learning_rate": 4.999619237890978e-07, + "logits/chosen": -2.206681966781616, + "logits/rejected": -2.1635756492614746, + "logps/chosen": -0.2884823679924011, + "logps/rejected": -0.29190367460250854, + "loss": 24.60689926147461, + "loss/core": 24.602405548095703, + "loss/preference_sft": 0.2884823679924011, + "loss/reference_anchor": 0.031054329127073288, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.0448436737060547, + "rewards/margins": 0.41608095169067383, + "rewards/rejected": 0.6287627816200256, + "step": 185 + }, + { + "drift/chosen_abs": 0.28438663482666016, + "drift/rejected_abs": 0.12391813099384308, + "epoch": 0.18153588916756241, + "grad_norm": 125.0, + "learning_rate": 4.998072590601808e-07, + "logits/chosen": -2.2966725826263428, + "logits/rejected": -2.2743964195251465, + "logps/chosen": -0.28313106298446655, + "logps/rejected": -0.28445905447006226, + "loss": 23.94525718688965, + "loss/core": 23.900707244873047, + "loss/preference_sft": 0.28313106298446655, + "loss/reference_anchor": 0.5657359957695007, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.842867374420166, + "rewards/margins": 1.6057891845703125, + "rewards/rejected": 1.2370779514312744, + "step": 190 + }, + { + "drift/chosen_abs": 0.26715975999832153, + "drift/rejected_abs": 0.09772941470146179, + "epoch": 0.18631314940881405, + "grad_norm": 67.5, + "learning_rate": 4.995336996054667e-07, + "logits/chosen": -1.9860740900039673, + "logits/rejected": -2.006164073944092, + "logps/chosen": -0.29072457551956177, + "logps/rejected": -0.28935790061950684, + "loss": 24.14964485168457, + "loss/core": 24.105886459350586, + "loss/preference_sft": 0.29072457551956177, + "loss/reference_anchor": 0.5543695092201233, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.6712136268615723, + "rewards/margins": 1.701263427734375, + "rewards/rejected": 0.9699504971504211, + "step": 195 + }, + { + "drift/chosen_abs": 0.26089510321617126, + "drift/rejected_abs": 0.12243443727493286, + "epoch": 0.19109040965006568, + "grad_norm": 75.0, + "learning_rate": 4.991413756244404e-07, + "logits/chosen": -1.8967984914779663, + "logits/rejected": -1.9376294612884521, + "logps/chosen": -0.3242345452308655, + "logps/rejected": -0.3163239061832428, + "loss": 23.804271697998047, + "loss/core": 23.7833309173584, + "loss/preference_sft": 0.3242345452308655, + "loss/reference_anchor": 0.2467818558216095, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.6078362464904785, + "rewards/margins": 1.3843178749084473, + "rewards/rejected": 1.2235184907913208, + "step": 200 + }, + { + "drift/chosen_abs": 0.328901469707489, + "drift/rejected_abs": 0.2119167149066925, + "epoch": 0.19586766989131732, + "grad_norm": 50.75, + "learning_rate": 4.986304738420683e-07, + "logits/chosen": -2.1179518699645996, + "logits/rejected": -2.169663667678833, + "logps/chosen": -0.25019803643226624, + "logps/rejected": -0.2696530222892761, + "loss": 24.55345344543457, + "loss/core": 24.50376319885254, + "loss/preference_sft": 0.25019803643226624, + "loss/reference_anchor": 0.6374753713607788, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.288543224334717, + "rewards/margins": 1.1873959302902222, + "rewards/rejected": 2.101147413253784, + "step": 205 + }, + { + "drift/chosen_abs": 0.2828282415866852, + "drift/rejected_abs": 0.1485763043165207, + "epoch": 0.20064493013256898, + "grad_norm": 121.0, + "learning_rate": 4.980012374199287e-07, + "logits/chosen": -2.1135048866271973, + "logits/rejected": -2.1378347873687744, + "logps/chosen": -0.3129284381866455, + "logps/rejected": -0.32191547751426697, + "loss": 23.932619094848633, + "loss/core": 23.88772964477539, + "loss/preference_sft": 0.3129284381866455, + "loss/reference_anchor": 0.5672147870063782, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.8268136978149414, + "rewards/margins": 1.3436048030853271, + "rewards/rejected": 1.4832088947296143, + "step": 210 + }, + { + "drift/chosen_abs": 0.3194449245929718, + "drift/rejected_abs": 0.1719251424074173, + "epoch": 0.20542219037382062, + "grad_norm": 133.0, + "learning_rate": 4.972539658404792e-07, + "logits/chosen": -2.0891597270965576, + "logits/rejected": -2.0813040733337402, + "logps/chosen": -0.2754015326499939, + "logps/rejected": -0.2838408946990967, + "loss": 24.59800910949707, + "loss/core": 24.53702735900879, + "loss/preference_sft": 0.2754015326499939, + "loss/reference_anchor": 0.7855226397514343, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.1921732425689697, + "rewards/margins": 1.485547661781311, + "rewards/rejected": 1.7066256999969482, + "step": 215 + }, + { + "drift/chosen_abs": 0.18876484036445618, + "drift/rejected_abs": 0.10738082975149155, + "epoch": 0.21019945061507225, + "grad_norm": 138.0, + "learning_rate": 4.963890147645194e-07, + "logits/chosen": -2.1035685539245605, + "logits/rejected": -2.017023801803589, + "logps/chosen": -0.3024953007698059, + "logps/rejected": -0.30150723457336426, + "loss": 24.240825653076172, + "loss/core": 24.230350494384766, + "loss/preference_sft": 0.3024953007698059, + "loss/reference_anchor": 0.1094045639038086, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.8857002258300781, + "rewards/margins": 0.813431441783905, + "rewards/rejected": 1.0722688436508179, + "step": 220 + }, + { + "drift/chosen_abs": 0.36280855536460876, + "drift/rejected_abs": 0.14171245694160461, + "epoch": 0.2149767108563239, + "grad_norm": 68.0, + "learning_rate": 4.95406795861916e-07, + "logits/chosen": -2.189195156097412, + "logits/rejected": -2.1777374744415283, + "logps/chosen": -0.29006287455558777, + "logps/rejected": -0.2748886048793793, + "loss": 23.25436782836914, + "loss/core": 23.220571517944336, + "loss/preference_sft": 0.29006287455558777, + "loss/reference_anchor": 0.4215586185455322, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.6274166107177734, + "rewards/margins": 2.212296485900879, + "rewards/rejected": 1.4151195287704468, + "step": 225 + }, + { + "drift/chosen_abs": 0.18566292524337769, + "drift/rejected_abs": 0.07588974386453629, + "epoch": 0.21975397109757555, + "grad_norm": 828.0, + "learning_rate": 4.943077766156697e-07, + "logits/chosen": -2.2765140533447266, + "logits/rejected": -2.2845187187194824, + "logps/chosen": -0.2816498279571533, + "logps/rejected": -0.30764302611351013, + "loss": 24.015369415283203, + "loss/core": 24.00626564025879, + "loss/preference_sft": 0.2816498279571533, + "loss/reference_anchor": 0.09326620399951935, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.8551431894302368, + "rewards/margins": 1.0997159481048584, + "rewards/rejected": 0.7554270625114441, + "step": 230 + }, + { + "drift/chosen_abs": 0.19052401185035706, + "drift/rejected_abs": 0.13283082842826843, + "epoch": 0.2245312313388272, + "grad_norm": 75.0, + "learning_rate": 4.930924800994191e-07, + "logits/chosen": -2.0162127017974854, + "logits/rejected": -2.050363302230835, + "logps/chosen": -0.2928079068660736, + "logps/rejected": -0.28698280453681946, + "loss": 24.55010414123535, + "loss/core": 24.53103256225586, + "loss/preference_sft": 0.2928079068660736, + "loss/reference_anchor": 0.22502753138542175, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.9030735492706299, + "rewards/margins": 0.5786088705062866, + "rewards/rejected": 1.3244645595550537, + "step": 235 + }, + { + "drift/chosen_abs": 0.1768554449081421, + "drift/rejected_abs": 0.10264088213443756, + "epoch": 0.22930849158007882, + "grad_norm": 88.0, + "learning_rate": 4.917614847284858e-07, + "logits/chosen": -2.0287160873413086, + "logits/rejected": -2.0479371547698975, + "logps/chosen": -0.29354214668273926, + "logps/rejected": -0.29428401589393616, + "loss": 24.292566299438477, + "loss/core": 24.284984588623047, + "loss/preference_sft": 0.29354214668273926, + "loss/reference_anchor": 0.07169236242771149, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.767877221107483, + "rewards/margins": 0.7428776621818542, + "rewards/rejected": 1.0249996185302734, + "step": 240 + }, + { + "drift/chosen_abs": 0.35989120602607727, + "drift/rejected_abs": 0.1157422810792923, + "epoch": 0.23408575182133046, + "grad_norm": 1056.0, + "learning_rate": 4.903154239845797e-07, + "logits/chosen": -2.0682170391082764, + "logits/rejected": -2.0564630031585693, + "logps/chosen": -0.2781009078025818, + "logps/rejected": -0.2913815677165985, + "loss": 23.59634017944336, + "loss/core": 23.54959487915039, + "loss/preference_sft": 0.2781009078025818, + "loss/reference_anchor": 0.5954445600509644, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.5985991954803467, + "rewards/margins": 2.4494547843933105, + "rewards/rejected": 1.1491445302963257, + "step": 245 + }, + { + "drift/chosen_abs": 0.2339925318956375, + "drift/rejected_abs": 0.10722045600414276, + "epoch": 0.23886301206258212, + "grad_norm": 101.0, + "learning_rate": 4.887549861142967e-07, + "logits/chosen": -1.9105064868927002, + "logits/rejected": -1.9906113147735596, + "logps/chosen": -0.2999528646469116, + "logps/rejected": -0.29507893323898315, + "loss": 23.889766693115234, + "loss/core": 23.876558303833008, + "loss/preference_sft": 0.2999528646469116, + "loss/reference_anchor": 0.1460881531238556, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.337829351425171, + "rewards/margins": 1.2744026184082031, + "rewards/rejected": 1.0634267330169678, + "step": 250 + }, + { + "drift/chosen_abs": 0.24554595351219177, + "drift/rejected_abs": 0.16090847551822662, + "epoch": 0.24364027230383375, + "grad_norm": 64.0, + "learning_rate": 4.870809138015498e-07, + "logits/chosen": -2.220195770263672, + "logits/rejected": -2.2151570320129395, + "logps/chosen": -0.2845707833766937, + "logps/rejected": -0.28703150153160095, + "loss": 24.28571128845215, + "loss/core": 24.260061264038086, + "loss/preference_sft": 0.2845707833766937, + "loss/reference_anchor": 0.31353074312210083, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.4550528526306152, + "rewards/margins": 0.8476669192314148, + "rewards/rejected": 1.6073859930038452, + "step": 255 + }, + { + "drift/chosen_abs": 0.2403588742017746, + "drift/rejected_abs": 0.15879188477993011, + "epoch": 0.2484175325450854, + "grad_norm": 35.25, + "learning_rate": 4.852940038140927e-07, + "logits/chosen": -2.1549439430236816, + "logits/rejected": -2.227227210998535, + "logps/chosen": -0.28740060329437256, + "logps/rejected": -0.27014413475990295, + "loss": 24.529048919677734, + "loss/core": 24.506467819213867, + "loss/preference_sft": 0.28740060329437256, + "loss/reference_anchor": 0.2723291516304016, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.403057336807251, + "rewards/margins": 0.8171398043632507, + "rewards/rejected": 1.5859174728393555, + "step": 260 + }, + { + "drift/chosen_abs": 0.12319445610046387, + "drift/rejected_abs": 0.11670766770839691, + "epoch": 0.25319479278633705, + "grad_norm": 34.0, + "learning_rate": 4.833951066243004e-07, + "logits/chosen": -2.0384128093719482, + "logits/rejected": -2.0463626384735107, + "logps/chosen": -0.29939448833465576, + "logps/rejected": -0.30272746086120605, + "loss": 25.318052291870117, + "loss/core": 25.299692153930664, + "loss/preference_sft": 0.29939448833465576, + "loss/reference_anchor": 0.21487000584602356, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.2312783002853394, + "rewards/margins": 0.06888408958911896, + "rewards/rejected": 1.1623942852020264, + "step": 265 + }, + { + "drift/chosen_abs": 0.32817989587783813, + "drift/rejected_abs": 0.21317580342292786, + "epoch": 0.2579720530275887, + "grad_norm": 57.0, + "learning_rate": 4.813851260043915e-07, + "logits/chosen": -1.9417810440063477, + "logits/rejected": -2.031541347503662, + "logps/chosen": -0.28844502568244934, + "logps/rejected": -0.29021137952804565, + "loss": 24.536828994750977, + "loss/core": 24.490381240844727, + "loss/preference_sft": 0.28844502568244934, + "loss/reference_anchor": 0.5904703140258789, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.2817986011505127, + "rewards/margins": 1.1503736972808838, + "rewards/rejected": 2.13142466545105, + "step": 270 + }, + { + "drift/chosen_abs": 0.23207350075244904, + "drift/rejected_abs": 0.12252874672412872, + "epoch": 0.2627493132688403, + "grad_norm": 572.0, + "learning_rate": 4.79265018596281e-07, + "logits/chosen": -2.0927796363830566, + "logits/rejected": -2.0458498001098633, + "logps/chosen": -0.29707807302474976, + "logps/rejected": -0.3014887869358063, + "loss": 23.969472885131836, + "loss/core": 23.95018768310547, + "loss/preference_sft": 0.29707807302474976, + "loss/reference_anchor": 0.22740456461906433, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.3190271854400635, + "rewards/margins": 1.1700794696807861, + "rewards/rejected": 1.1489474773406982, + "step": 275 + }, + { + "drift/chosen_abs": 0.2051713466644287, + "drift/rejected_abs": 0.08534234762191772, + "epoch": 0.26752657351009196, + "grad_norm": 56.75, + "learning_rate": 4.770357934562704e-07, + "logits/chosen": -2.063467502593994, + "logits/rejected": -1.9795265197753906, + "logps/chosen": -0.2879626750946045, + "logps/rejected": -0.3061186373233795, + "loss": 23.862253189086914, + "loss/core": 23.851999282836914, + "loss/preference_sft": 0.2879626750946045, + "loss/reference_anchor": 0.10792496055364609, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.0476605892181396, + "rewards/margins": 1.2737023830413818, + "rewards/rejected": 0.7739582061767578, + "step": 280 + }, + { + "drift/chosen_abs": 0.2606334090232849, + "drift/rejected_abs": 0.16303770244121552, + "epoch": 0.2723038337513436, + "grad_norm": 168.0, + "learning_rate": 4.746985115747917e-07, + "logits/chosen": -2.0845389366149902, + "logits/rejected": -2.0801751613616943, + "logps/chosen": -0.29716962575912476, + "logps/rejected": -0.29298409819602966, + "loss": 24.26735496520996, + "loss/core": 24.245363235473633, + "loss/preference_sft": 0.29716962575912476, + "loss/reference_anchor": 0.26353123784065247, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.605191707611084, + "rewards/margins": 0.9785102605819702, + "rewards/rejected": 1.6266816854476929, + "step": 285 + }, + { + "drift/chosen_abs": 0.14788036048412323, + "drift/rejected_abs": 0.14346002042293549, + "epoch": 0.27708109399259523, + "grad_norm": 1488.0, + "learning_rate": 4.722542853714341e-07, + "logits/chosen": -2.1646649837493896, + "logits/rejected": -2.176093578338623, + "logps/chosen": -0.2869679927825928, + "logps/rejected": -0.28978294134140015, + "loss": 25.25179672241211, + "loss/core": 25.234987258911133, + "loss/preference_sft": 0.2869679927825928, + "loss/reference_anchor": 0.19546569883823395, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.4774421453475952, + "rewards/margins": 0.04469200596213341, + "rewards/rejected": 1.4327499866485596, + "step": 290 + }, + { + "drift/chosen_abs": 0.1308569461107254, + "drift/rejected_abs": 0.0676531195640564, + "epoch": 0.28185835423384686, + "grad_norm": 64.5, + "learning_rate": 4.697042781654913e-07, + "logits/chosen": -2.565634250640869, + "logits/rejected": -2.5284571647644043, + "logps/chosen": -0.2447257786989212, + "logps/rejected": -0.2566220164299011, + "loss": 24.383955001831055, + "loss/core": 24.379867553710938, + "loss/preference_sft": 0.2447257786989212, + "loss/reference_anchor": 0.030016642063856125, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.308022379875183, + "rewards/margins": 0.6341660022735596, + "rewards/rejected": 0.6738564968109131, + "step": 295 + }, + { + "drift/chosen_abs": 0.3740275204181671, + "drift/rejected_abs": 0.10665237903594971, + "epoch": 0.28663561447509855, + "grad_norm": 388.0, + "learning_rate": 4.670497036222856e-07, + "logits/chosen": -1.9813871383666992, + "logits/rejected": -1.9485054016113281, + "logps/chosen": -0.31088101863861084, + "logps/rejected": -0.3178117573261261, + "loss": 23.0051326751709, + "loss/core": 22.965051651000977, + "loss/preference_sft": 0.31088101863861084, + "loss/reference_anchor": 0.5033376216888428, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.739037275314331, + "rewards/margins": 2.6878647804260254, + "rewards/rejected": 1.0511722564697266, + "step": 300 + }, + { + "drift/chosen_abs": 0.2159603089094162, + "drift/rejected_abs": 0.0897536352276802, + "epoch": 0.2914128747163502, + "grad_norm": 76.0, + "learning_rate": 4.642918251755281e-07, + "logits/chosen": -2.125406265258789, + "logits/rejected": -2.185771942138672, + "logps/chosen": -0.26945987343788147, + "logps/rejected": -0.26080164313316345, + "loss": 23.85554313659668, + "loss/core": 23.84500503540039, + "loss/preference_sft": 0.26945987343788147, + "loss/reference_anchor": 0.11357325315475464, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.159000873565674, + "rewards/margins": 1.2621368169784546, + "rewards/rejected": 0.896864116191864, + "step": 305 + }, + { + "drift/chosen_abs": 0.26018840074539185, + "drift/rejected_abs": 0.2073136568069458, + "epoch": 0.2961901349576018, + "grad_norm": 976.0, + "learning_rate": 4.614319554259933e-07, + "logits/chosen": -2.12811017036438, + "logits/rejected": -2.0370516777038574, + "logps/chosen": -0.2741633355617523, + "logps/rejected": -0.2802058160305023, + "loss": 25.01598358154297, + "loss/core": 24.98000144958496, + "loss/preference_sft": 0.2741633355617523, + "loss/reference_anchor": 0.452305406332016, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.6010148525238037, + "rewards/margins": 0.5811925530433655, + "rewards/rejected": 2.019822359085083, + "step": 310 + }, + { + "drift/chosen_abs": 0.29169219732284546, + "drift/rejected_abs": 0.13470803201198578, + "epoch": 0.30096739519885346, + "grad_norm": 61.0, + "learning_rate": 4.58471455516792e-07, + "logits/chosen": -1.9604593515396118, + "logits/rejected": -2.0688371658325195, + "logps/chosen": -0.2688148617744446, + "logps/rejected": -0.2757693827152252, + "loss": 23.543533325195312, + "loss/core": 23.524099349975586, + "loss/preference_sft": 0.2688148617744446, + "loss/reference_anchor": 0.23222799599170685, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.9164979457855225, + "rewards/margins": 1.5716060400009155, + "rewards/rejected": 1.344891905784607, + "step": 315 + }, + { + "drift/chosen_abs": 0.1583484411239624, + "drift/rejected_abs": 0.12316056340932846, + "epoch": 0.3057446554401051, + "grad_norm": 51.5, + "learning_rate": 4.5541173448554095e-07, + "logits/chosen": -2.0016112327575684, + "logits/rejected": -2.0177295207977295, + "logps/chosen": -0.2901010513305664, + "logps/rejected": -0.2924564480781555, + "loss": 24.739116668701172, + "loss/core": 24.729549407958984, + "loss/preference_sft": 0.2901010513305664, + "loss/reference_anchor": 0.09852476418018341, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.5812407732009888, + "rewards/margins": 0.35098347067832947, + "rewards/rejected": 1.2302576303482056, + "step": 320 + }, + { + "drift/chosen_abs": 0.36936116218566895, + "drift/rejected_abs": 0.1534915715456009, + "epoch": 0.31052191568135673, + "grad_norm": 146.0, + "learning_rate": 4.5225424859373684e-07, + "logits/chosen": -2.257908344268799, + "logits/rejected": -2.2035529613494873, + "logps/chosen": -0.28698205947875977, + "logps/rejected": -0.28376373648643494, + "loss": 24.22592544555664, + "loss/core": 24.155797958374023, + "loss/preference_sft": 0.28698205947875977, + "loss/reference_anchor": 0.9063272476196289, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.6920619010925293, + "rewards/margins": 2.15855073928833, + "rewards/rejected": 1.5335115194320679, + "step": 325 + }, + { + "drift/chosen_abs": 0.20329172909259796, + "drift/rejected_abs": 0.11353343725204468, + "epoch": 0.31529917592260837, + "grad_norm": 40.75, + "learning_rate": 4.4900050063365547e-07, + "logits/chosen": -2.2280595302581787, + "logits/rejected": -2.217438220977783, + "logps/chosen": -0.27844610810279846, + "logps/rejected": -0.270263135433197, + "loss": 24.234060287475586, + "loss/core": 24.221086502075195, + "loss/preference_sft": 0.27844610810279846, + "loss/reference_anchor": 0.14516863226890564, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.0322957038879395, + "rewards/margins": 0.8984895944595337, + "rewards/rejected": 1.1338062286376953, + "step": 330 + }, + { + "drift/chosen_abs": 0.2127622663974762, + "drift/rejected_abs": 0.07780848443508148, + "epoch": 0.32007643616386, + "grad_norm": 165.0, + "learning_rate": 4.4565203921310344e-07, + "logits/chosen": -2.0511183738708496, + "logits/rejected": -2.0350396633148193, + "logps/chosen": -0.30186930298805237, + "logps/rejected": -0.3197532296180725, + "loss": 23.990903854370117, + "loss/core": 23.97220230102539, + "loss/preference_sft": 0.30186930298805237, + "loss/reference_anchor": 0.21921315789222717, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.124210834503174, + "rewards/margins": 1.4477331638336182, + "rewards/rejected": 0.6764776110649109, + "step": 335 + }, + { + "drift/chosen_abs": 0.2031564712524414, + "drift/rejected_abs": 0.087882399559021, + "epoch": 0.3248536964051117, + "grad_norm": 186.0, + "learning_rate": 4.422104580183649e-07, + "logits/chosen": -2.095062017440796, + "logits/rejected": -2.2184524536132812, + "logps/chosen": -0.28098320960998535, + "logps/rejected": -0.28097662329673767, + "loss": 24.006999969482422, + "loss/core": 23.993459701538086, + "loss/preference_sft": 0.28098320960998535, + "loss/reference_anchor": 0.15243388712406158, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.031026840209961, + "rewards/margins": 1.153816819190979, + "rewards/rejected": 0.8772099614143372, + "step": 340 + }, + { + "drift/chosen_abs": 0.16465213894844055, + "drift/rejected_abs": 0.0845918357372284, + "epoch": 0.3296309566463633, + "grad_norm": 98.5, + "learning_rate": 4.3867739505569303e-07, + "logits/chosen": -2.3218281269073486, + "logits/rejected": -2.3827481269836426, + "logps/chosen": -0.282081663608551, + "logps/rejected": -0.29249638319015503, + "loss": 24.309249877929688, + "loss/core": 24.29976463317871, + "loss/preference_sft": 0.282081663608551, + "loss/reference_anchor": 0.09826230257749557, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.644649863243103, + "rewards/margins": 0.799951434135437, + "rewards/rejected": 0.8446983098983765, + "step": 345 + }, + { + "drift/chosen_abs": 0.20000576972961426, + "drift/rejected_abs": 0.09551592916250229, + "epoch": 0.33440821688761496, + "grad_norm": 80.5, + "learning_rate": 4.35054531871708e-07, + "logits/chosen": -2.0598835945129395, + "logits/rejected": -2.03928804397583, + "logps/chosen": -0.26030445098876953, + "logps/rejected": -0.2558044195175171, + "loss": 24.042661666870117, + "loss/core": 24.03305435180664, + "loss/preference_sft": 0.26030445098876953, + "loss/reference_anchor": 0.1020883321762085, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.9989124536514282, + "rewards/margins": 1.0452065467834473, + "rewards/rejected": 0.953705906867981, + "step": 350 + }, + { + "drift/chosen_abs": 0.18520495295524597, + "drift/rejected_abs": 0.1412447839975357, + "epoch": 0.3391854771288666, + "grad_norm": 75.5, + "learning_rate": 4.3134359275307185e-07, + "logits/chosen": -2.106975555419922, + "logits/rejected": -2.2341411113739014, + "logps/chosen": -0.28064435720443726, + "logps/rejected": -0.2879922389984131, + "loss": 24.90737533569336, + "loss/core": 24.887197494506836, + "loss/preference_sft": 0.28064435720443726, + "loss/reference_anchor": 0.2410183846950531, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.851447343826294, + "rewards/margins": 0.45425519347190857, + "rewards/rejected": 1.3971920013427734, + "step": 355 + }, + { + "drift/chosen_abs": 0.26272064447402954, + "drift/rejected_abs": 0.1274389922618866, + "epoch": 0.34396273737011823, + "grad_norm": 472.0, + "learning_rate": 4.2754634390582133e-07, + "logits/chosen": -2.1305363178253174, + "logits/rejected": -2.192857265472412, + "logps/chosen": -0.28095608949661255, + "logps/rejected": -0.28217315673828125, + "loss": 24.006431579589844, + "loss/core": 23.981016159057617, + "loss/preference_sft": 0.28095608949661255, + "loss/reference_anchor": 0.31079161167144775, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.62534761428833, + "rewards/margins": 1.353183627128601, + "rewards/rejected": 1.272163987159729, + "step": 360 + }, + { + "drift/chosen_abs": 0.2852028012275696, + "drift/rejected_abs": 0.08169830590486526, + "epoch": 0.34873999761136987, + "grad_norm": 205.0, + "learning_rate": 4.236645926147493e-07, + "logits/chosen": -2.069857120513916, + "logits/rejected": -2.1774981021881104, + "logps/chosen": -0.287311851978302, + "logps/rejected": -0.2833508551120758, + "loss": 23.85344123840332, + "loss/core": 23.815753936767578, + "loss/preference_sft": 0.287311851978302, + "loss/reference_anchor": 0.4737482964992523, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.851012945175171, + "rewards/margins": 2.036228895187378, + "rewards/rejected": 0.8147839307785034, + "step": 365 + }, + { + "drift/chosen_abs": 0.1644589602947235, + "drift/rejected_abs": 0.10670790821313858, + "epoch": 0.3535172578526215, + "grad_norm": 188.0, + "learning_rate": 4.1970018638323547e-07, + "logits/chosen": -2.2686495780944824, + "logits/rejected": -2.3140177726745605, + "logps/chosen": -0.2849913239479065, + "logps/rejected": -0.2848779857158661, + "loss": 24.466651916503906, + "loss/core": 24.457897186279297, + "loss/preference_sft": 0.2849913239479065, + "loss/reference_anchor": 0.08822296559810638, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6427894830703735, + "rewards/margins": 0.5768302083015442, + "rewards/rejected": 1.0659592151641846, + "step": 370 + }, + { + "drift/chosen_abs": 0.3016364276409149, + "drift/rejected_abs": 0.22407476603984833, + "epoch": 0.35829451809387314, + "grad_norm": 70.5, + "learning_rate": 4.1565501205393436e-07, + "logits/chosen": -2.0465548038482666, + "logits/rejected": -2.024115562438965, + "logps/chosen": -0.25850996375083923, + "logps/rejected": -0.25814658403396606, + "loss": 24.701723098754883, + "loss/core": 24.66933822631836, + "loss/preference_sft": 0.25850996375083923, + "loss/reference_anchor": 0.4059620499610901, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.0161895751953125, + "rewards/margins": 0.7962467670440674, + "rewards/rejected": 2.2199432849884033, + "step": 375 + }, + { + "drift/chosen_abs": 0.20642168819904327, + "drift/rejected_abs": 0.16034074127674103, + "epoch": 0.36307177833512483, + "grad_norm": 106.0, + "learning_rate": 4.1153099491074093e-07, + "logits/chosen": -2.122387409210205, + "logits/rejected": -2.131068468093872, + "logps/chosen": -0.27889400720596313, + "logps/rejected": -0.2892562448978424, + "loss": 25.11880874633789, + "loss/core": 25.087757110595703, + "loss/preference_sft": 0.27889400720596313, + "loss/reference_anchor": 0.38615480065345764, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.06227445602417, + "rewards/margins": 0.45964282751083374, + "rewards/rejected": 1.6026318073272705, + "step": 380 + }, + { + "drift/chosen_abs": 0.21806609630584717, + "drift/rejected_abs": 0.15287581086158752, + "epoch": 0.36784903857637646, + "grad_norm": 131.0, + "learning_rate": 4.0733009776245937e-07, + "logits/chosen": -2.115004062652588, + "logits/rejected": -2.1338284015655518, + "logps/chosen": -0.2700949013233185, + "logps/rejected": -0.27447065711021423, + "loss": 24.529088973999023, + "loss/core": 24.512638092041016, + "loss/preference_sft": 0.2700949013233185, + "loss/reference_anchor": 0.19232222437858582, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.179309129714966, + "rewards/margins": 0.6530361175537109, + "rewards/rejected": 1.5262731313705444, + "step": 385 + }, + { + "drift/chosen_abs": 0.17654861509799957, + "drift/rejected_abs": 0.09032897651195526, + "epoch": 0.3726262988176281, + "grad_norm": 42.75, + "learning_rate": 4.0305432000861226e-07, + "logits/chosen": -1.8726367950439453, + "logits/rejected": -1.9506858587265015, + "logps/chosen": -0.3072229325771332, + "logps/rejected": -0.3017742931842804, + "loss": 24.248693466186523, + "loss/core": 24.23879051208496, + "loss/preference_sft": 0.3072229325771332, + "loss/reference_anchor": 0.10132338106632233, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.761505126953125, + "rewards/margins": 0.8584615588188171, + "rewards/rejected": 0.9030437469482422, + "step": 390 + }, + { + "drift/chosen_abs": 0.1685396432876587, + "drift/rejected_abs": 0.08101732283830643, + "epoch": 0.37740355905887973, + "grad_norm": 101.0, + "learning_rate": 3.9870569668783533e-07, + "logits/chosen": -2.223672389984131, + "logits/rejected": -2.3257579803466797, + "logps/chosen": -0.29469752311706543, + "logps/rejected": -0.28933185338974, + "loss": 24.182411193847656, + "loss/core": 24.174169540405273, + "loss/preference_sft": 0.29469752311706543, + "loss/reference_anchor": 0.08043049275875092, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 1.6851272583007812, + "rewards/margins": 0.8753724098205566, + "rewards/rejected": 0.8097550272941589, + "step": 395 + }, + { + "drift/chosen_abs": 0.29273721575737, + "drift/rejected_abs": 0.0846409797668457, + "epoch": 0.38218081930013137, + "grad_norm": 85.0, + "learning_rate": 3.942862975093084e-07, + "logits/chosen": -2.1020679473876953, + "logits/rejected": -2.1372578144073486, + "logps/chosen": -0.29274052381515503, + "logps/rejected": -0.2928157448768616, + "loss": 23.699491500854492, + "loss/core": 23.65845489501953, + "loss/preference_sft": 0.29274052381515503, + "loss/reference_anchor": 0.5178688168525696, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.925940990447998, + "rewards/margins": 2.0813891887664795, + "rewards/rejected": 0.8445518612861633, + "step": 400 + }, + { + "drift/chosen_abs": 0.2351594716310501, + "drift/rejected_abs": 0.12229617685079575, + "epoch": 0.386958079541383, + "grad_norm": 65.0, + "learning_rate": 3.8979822586768666e-07, + "logits/chosen": -2.135050058364868, + "logits/rejected": -2.189692735671997, + "logps/chosen": -0.3113452196121216, + "logps/rejected": -0.3050457835197449, + "loss": 24.190441131591797, + "loss/core": 24.1619873046875, + "loss/preference_sft": 0.3113452196121216, + "loss/reference_anchor": 0.3482899069786072, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.3515946865081787, + "rewards/margins": 1.1289160251617432, + "rewards/rejected": 1.2226788997650146, + "step": 405 + }, + { + "drift/chosen_abs": 0.20916077494621277, + "drift/rejected_abs": 0.11924557387828827, + "epoch": 0.39173533978263464, + "grad_norm": 86.0, + "learning_rate": 3.8524361784199847e-07, + "logits/chosen": -2.251237630844116, + "logits/rejected": -2.2815263271331787, + "logps/chosen": -0.27680549025535583, + "logps/rejected": -0.2878721058368683, + "loss": 24.154521942138672, + "loss/core": 24.142080307006836, + "loss/preference_sft": 0.27680549025535583, + "loss/reference_anchor": 0.1381910741329193, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.090045690536499, + "rewards/margins": 0.8996987342834473, + "rewards/rejected": 1.1903469562530518, + "step": 410 + }, + { + "drift/chosen_abs": 0.09965378046035767, + "drift/rejected_abs": 0.037877295166254044, + "epoch": 0.3965126000238863, + "grad_norm": 77.5, + "learning_rate": 3.806246411789872e-07, + "logits/chosen": -2.1194329261779785, + "logits/rejected": -2.1981375217437744, + "logps/chosen": -0.33038097620010376, + "logps/rejected": -0.32079142332077026, + "loss": 24.420391082763672, + "loss/core": 24.41567039489746, + "loss/preference_sft": 0.33038097620010376, + "loss/reference_anchor": 0.02990715578198433, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 0.9942773580551147, + "rewards/margins": 0.616710364818573, + "rewards/rejected": 0.37756696343421936, + "step": 415 + }, + { + "drift/chosen_abs": 0.2046138346195221, + "drift/rejected_abs": 0.07252134382724762, + "epoch": 0.40128986026513797, + "grad_norm": 129.0, + "learning_rate": 3.7594349426138154e-07, + "logits/chosen": -2.350454807281494, + "logits/rejected": -2.316537618637085, + "logps/chosen": -0.2832780182361603, + "logps/rejected": -0.29880502820014954, + "loss": 23.930326461791992, + "loss/core": 23.915685653686523, + "loss/preference_sft": 0.2832780182361603, + "loss/reference_anchor": 0.1668982058763504, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.046138286590576, + "rewards/margins": 1.3250911235809326, + "rewards/rejected": 0.721047043800354, + "step": 420 + }, + { + "drift/chosen_abs": 0.2236243188381195, + "drift/rejected_abs": 0.1645563691854477, + "epoch": 0.4060671205063896, + "grad_norm": 486.0, + "learning_rate": 3.712024050615843e-07, + "logits/chosen": -2.0146186351776123, + "logits/rejected": -2.0732080936431885, + "logps/chosen": -0.2916070222854614, + "logps/rejected": -0.2652800679206848, + "loss": 25.089502334594727, + "loss/core": 25.058069229125977, + "loss/preference_sft": 0.2916070222854614, + "loss/reference_anchor": 0.3899894654750824, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.235166549682617, + "rewards/margins": 0.5908761620521545, + "rewards/rejected": 1.6442902088165283, + "step": 425 + }, + { + "drift/chosen_abs": 0.44021230936050415, + "drift/rejected_abs": 0.1417957842350006, + "epoch": 0.41084438074764124, + "grad_norm": 274.0, + "learning_rate": 3.664036300812778e-07, + "logits/chosen": -2.0683376789093018, + "logits/rejected": -2.091358184814453, + "logps/chosen": -0.27976277470588684, + "logps/rejected": -0.2667387127876282, + "loss": 23.26540184020996, + "loss/core": 23.194873809814453, + "loss/preference_sft": 0.27976277470588684, + "loss/reference_anchor": 0.9124003648757935, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.400694847106934, + "rewards/margins": 2.9839932918548584, + "rewards/rejected": 1.4167020320892334, + "step": 430 + }, + { + "drift/chosen_abs": 0.16288933157920837, + "drift/rejected_abs": 0.10126354545354843, + "epoch": 0.41562164098889287, + "grad_norm": 99.5, + "learning_rate": 3.615494532774522e-07, + "logits/chosen": -2.3494062423706055, + "logits/rejected": -2.3344578742980957, + "logps/chosen": -0.2779366374015808, + "logps/rejected": -0.29263314604759216, + "loss": 24.44803810119629, + "loss/core": 24.437061309814453, + "loss/preference_sft": 0.2779366374015808, + "loss/reference_anchor": 0.11851142346858978, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.628753900527954, + "rewards/margins": 0.6175113916397095, + "rewards/rejected": 1.0112426280975342, + "step": 435 + }, + { + "drift/chosen_abs": 0.30069416761398315, + "drift/rejected_abs": 0.13946831226348877, + "epoch": 0.4203989012301445, + "grad_norm": 668.0, + "learning_rate": 3.5664218497536456e-07, + "logits/chosen": -2.05379581451416, + "logits/rejected": -2.1259214878082275, + "logps/chosen": -0.303558886051178, + "logps/rejected": -0.3001348078250885, + "loss": 24.09839630126953, + "loss/core": 24.058101654052734, + "loss/preference_sft": 0.303558886051178, + "loss/reference_anchor": 0.5069498419761658, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.003722667694092, + "rewards/margins": 1.6185331344604492, + "rewards/rejected": 1.3851896524429321, + "step": 440 + }, + { + "drift/chosen_abs": 0.29670435190200806, + "drift/rejected_abs": 0.16009071469306946, + "epoch": 0.42517616147139614, + "grad_norm": 130.0, + "learning_rate": 3.516841607689501e-07, + "logits/chosen": -1.9354779720306396, + "logits/rejected": -1.9123506546020508, + "logps/chosen": -0.29504531621932983, + "logps/rejected": -0.2956696152687073, + "loss": 23.822757720947266, + "loss/core": 23.801040649414062, + "loss/preference_sft": 0.29504531621932983, + "loss/reference_anchor": 0.2600812315940857, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.9661905765533447, + "rewards/margins": 1.388580560684204, + "rewards/rejected": 1.577609896659851, + "step": 445 + }, + { + "drift/chosen_abs": 0.345471054315567, + "drift/rejected_abs": 0.161865234375, + "epoch": 0.4299534217126478, + "grad_norm": 768.0, + "learning_rate": 3.4667774040920515e-07, + "logits/chosen": -1.9765275716781616, + "logits/rejected": -1.9926903247833252, + "logps/chosen": -0.29864731431007385, + "logps/rejected": -0.2888573110103607, + "loss": 23.699243545532227, + "loss/core": 23.66305160522461, + "loss/preference_sft": 0.29864731431007385, + "loss/reference_anchor": 0.4526439309120178, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.4545040130615234, + "rewards/margins": 1.8422412872314453, + "rewards/rejected": 1.6122627258300781, + "step": 450 + }, + { + "drift/chosen_abs": 0.34665170311927795, + "drift/rejected_abs": 0.14890094101428986, + "epoch": 0.4347306819538994, + "grad_norm": 246.0, + "learning_rate": 3.416253066810743e-07, + "logits/chosen": -1.936833143234253, + "logits/rejected": -2.127368927001953, + "logps/chosen": -0.32262879610061646, + "logps/rejected": -0.31309956312179565, + "loss": 23.719173431396484, + "loss/core": 23.66214370727539, + "loss/preference_sft": 0.32262879610061646, + "loss/reference_anchor": 0.728130042552948, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.4665169715881348, + "rewards/margins": 1.979360580444336, + "rewards/rejected": 1.4871563911437988, + "step": 455 + }, + { + "drift/chosen_abs": 0.36950841546058655, + "drift/rejected_abs": 0.18701323866844177, + "epoch": 0.4395079421951511, + "grad_norm": 141.0, + "learning_rate": 3.3652926426937325e-07, + "logits/chosen": -2.0627896785736084, + "logits/rejected": -2.0499472618103027, + "logps/chosen": -0.2930159270763397, + "logps/rejected": -0.3065492510795593, + "loss": 23.571096420288086, + "loss/core": 23.52437973022461, + "loss/preference_sft": 0.2930159270763397, + "loss/reference_anchor": 0.5935932397842407, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.694608688354492, + "rewards/margins": 1.8250821828842163, + "rewards/rejected": 1.8695262670516968, + "step": 460 + }, + { + "drift/chosen_abs": 0.23000410199165344, + "drift/rejected_abs": 0.09393614530563354, + "epoch": 0.44428520243640274, + "grad_norm": 60.25, + "learning_rate": 3.3139203861428914e-07, + "logits/chosen": -1.8922626972198486, + "logits/rejected": -1.9913305044174194, + "logps/chosen": -0.29730328917503357, + "logps/rejected": -0.29092690348625183, + "loss": 23.748533248901367, + "loss/core": 23.73752212524414, + "loss/preference_sft": 0.29730328917503357, + "loss/reference_anchor": 0.11708039045333862, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.2995853424072266, + "rewards/margins": 1.3625905513763428, + "rewards/rejected": 0.936994731426239, + "step": 465 + }, + { + "drift/chosen_abs": 0.30310288071632385, + "drift/rejected_abs": 0.12414111942052841, + "epoch": 0.4490624626776544, + "grad_norm": 208.0, + "learning_rate": 3.262160747570027e-07, + "logits/chosen": -2.1301331520080566, + "logits/rejected": -2.198742628097534, + "logps/chosen": -0.27701377868652344, + "logps/rejected": -0.26015716791152954, + "loss": 23.711040496826172, + "loss/core": 23.664438247680664, + "loss/preference_sft": 0.27701377868652344, + "loss/reference_anchor": 0.5936552286148071, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.030449151992798, + "rewards/margins": 1.86244797706604, + "rewards/rejected": 1.168001413345337, + "step": 470 + }, + { + "drift/chosen_abs": 0.18387170135974884, + "drift/rejected_abs": 0.15331368148326874, + "epoch": 0.453839722918906, + "grad_norm": 332.0, + "learning_rate": 3.210038361759807e-07, + "logits/chosen": -2.095447301864624, + "logits/rejected": -2.033949613571167, + "logps/chosen": -0.29470914602279663, + "logps/rejected": -0.3144375681877136, + "loss": 25.121068954467773, + "loss/core": 25.099435806274414, + "loss/preference_sft": 0.29470914602279663, + "loss/reference_anchor": 0.2589879631996155, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.838440179824829, + "rewards/margins": 0.307887464761734, + "rewards/rejected": 1.530552625656128, + "step": 475 + }, + { + "drift/chosen_abs": 0.2358885258436203, + "drift/rejected_abs": 0.10619956254959106, + "epoch": 0.45861698316015764, + "grad_norm": 70.0, + "learning_rate": 3.1575780361449364e-07, + "logits/chosen": -2.264855146408081, + "logits/rejected": -2.3214104175567627, + "logps/chosen": -0.2755188047885895, + "logps/rejected": -0.27138248085975647, + "loss": 24.032466888427734, + "loss/core": 24.010181427001953, + "loss/preference_sft": 0.2755188047885895, + "loss/reference_anchor": 0.2696211636066437, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.3573050498962402, + "rewards/margins": 1.3217544555664062, + "rewards/rejected": 1.0355504751205444, + "step": 480 + }, + { + "drift/chosen_abs": 0.1745290607213974, + "drift/rejected_abs": 0.08883767575025558, + "epoch": 0.4633942434014093, + "grad_norm": 74.0, + "learning_rate": 3.104804738999169e-07, + "logits/chosen": -2.205657482147217, + "logits/rejected": -2.2127983570098877, + "logps/chosen": -0.29136472940444946, + "logps/rejected": -0.29586881399154663, + "loss": 24.19198226928711, + "loss/core": 24.183902740478516, + "loss/preference_sft": 0.29136472940444946, + "loss/reference_anchor": 0.07860977947711945, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.74505615234375, + "rewards/margins": 0.8590679168701172, + "rewards/rejected": 0.8859883546829224, + "step": 485 + }, + { + "drift/chosen_abs": 0.20774760842323303, + "drift/rejected_abs": 0.10662759840488434, + "epoch": 0.4681715036426609, + "grad_norm": 99.5, + "learning_rate": 3.0517435875537536e-07, + "logits/chosen": -2.1565749645233154, + "logits/rejected": -2.198429822921753, + "logps/chosen": -0.30721935629844666, + "logps/rejected": -0.29325515031814575, + "loss": 24.078283309936523, + "loss/core": 24.06472396850586, + "loss/preference_sft": 0.30721935629844666, + "loss/reference_anchor": 0.15004561841487885, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.0768425464630127, + "rewards/margins": 1.012283444404602, + "rewards/rejected": 1.0645591020584106, + "step": 490 + }, + { + "drift/chosen_abs": 0.2635328471660614, + "drift/rejected_abs": 0.08991514146327972, + "epoch": 0.47294876388391255, + "grad_norm": 133.0, + "learning_rate": 2.998419836042993e-07, + "logits/chosen": -2.2264866828918457, + "logits/rejected": -2.3216514587402344, + "logps/chosen": -0.29546287655830383, + "logps/rejected": -0.2902601361274719, + "loss": 23.78059959411621, + "loss/core": 23.750764846801758, + "loss/preference_sft": 0.29546287655830383, + "loss/reference_anchor": 0.36827731132507324, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.6349475383758545, + "rewards/margins": 1.7373361587524414, + "rewards/rejected": 0.897611141204834, + "step": 495 + }, + { + "drift/chosen_abs": 0.4189828336238861, + "drift/rejected_abs": 0.20560988783836365, + "epoch": 0.47772602412516424, + "grad_norm": 284.0, + "learning_rate": 2.9448588636846043e-07, + "logits/chosen": -2.023249387741089, + "logits/rejected": -1.972783088684082, + "logps/chosen": -0.2784698009490967, + "logps/rejected": -0.2997981607913971, + "loss": 24.244253158569336, + "loss/core": 24.155033111572266, + "loss/preference_sft": 0.2784698009490967, + "loss/reference_anchor": 1.161747694015503, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 4.189828395843506, + "rewards/margins": 2.1354339122772217, + "rewards/rejected": 2.054394245147705, + "step": 500 + }, + { + "drift/chosen_abs": 0.3628165125846863, + "drift/rejected_abs": 0.19208696484565735, + "epoch": 0.4825032843664159, + "grad_norm": 76.0, + "learning_rate": 2.8910861626005773e-07, + "logits/chosen": -2.1512835025787354, + "logits/rejected": -2.1834206581115723, + "logps/chosen": -0.2839333713054657, + "logps/rejected": -0.278773695230484, + "loss": 24.71078109741211, + "loss/core": 24.65015983581543, + "loss/preference_sft": 0.2839333713054657, + "loss/reference_anchor": 0.7799011468887329, + "rewards/accuracies": 0.8125, + "rewards/chosen": 3.6272101402282715, + "rewards/margins": 1.7839975357055664, + "rewards/rejected": 1.8432128429412842, + "step": 505 + }, + { + "drift/chosen_abs": 0.2025258094072342, + "drift/rejected_abs": 0.08022260665893555, + "epoch": 0.4872805446076675, + "grad_norm": 27.5, + "learning_rate": 2.8371273256843074e-07, + "logits/chosen": -2.1484246253967285, + "logits/rejected": -2.282355546951294, + "logps/chosen": -0.3147265315055847, + "logps/rejected": -0.2912803888320923, + "loss": 24.102039337158203, + "loss/core": 24.08344078063965, + "loss/preference_sft": 0.3147265315055847, + "loss/reference_anchor": 0.21651633083820343, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.022645950317383, + "rewards/margins": 1.2219388484954834, + "rewards/rejected": 0.8007070422172546, + "step": 510 + }, + { + "drift/chosen_abs": 0.2555674910545349, + "drift/rejected_abs": 0.16905184090137482, + "epoch": 0.49205780484891914, + "grad_norm": 129.0, + "learning_rate": 2.783008034419767e-07, + "logits/chosen": -2.032346487045288, + "logits/rejected": -2.036229372024536, + "logps/chosen": -0.2818189561367035, + "logps/rejected": -0.30472418665885925, + "loss": 24.25507164001465, + "loss/core": 24.236268997192383, + "loss/preference_sft": 0.2818189561367035, + "loss/reference_anchor": 0.22251644730567932, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.553283214569092, + "rewards/margins": 0.8822887539863586, + "rewards/rejected": 1.670994758605957, + "step": 515 + }, + { + "drift/chosen_abs": 0.30994483828544617, + "drift/rejected_abs": 0.09894011914730072, + "epoch": 0.4968350650901708, + "grad_norm": 111.0, + "learning_rate": 2.7287540466585064e-07, + "logits/chosen": -2.059782028198242, + "logits/rejected": -2.228891372680664, + "logps/chosen": -0.3199217915534973, + "logps/rejected": -0.333048939704895, + "loss": 23.464609146118164, + "loss/core": 23.430858612060547, + "loss/preference_sft": 0.3199217915534973, + "loss/reference_anchor": 0.41807693243026733, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.0952601432800293, + "rewards/margins": 2.12144136428833, + "rewards/rejected": 0.9738188982009888, + "step": 520 + }, + { + "drift/chosen_abs": 0.43741607666015625, + "drift/rejected_abs": 0.14899471402168274, + "epoch": 0.5016123253314224, + "grad_norm": 668.0, + "learning_rate": 2.674391184360313e-07, + "logits/chosen": -2.195627450942993, + "logits/rejected": -2.1632399559020996, + "logps/chosen": -0.2940196394920349, + "logps/rejected": -0.2978762984275818, + "loss": 23.785734176635742, + "loss/core": 23.704212188720703, + "loss/preference_sft": 0.2940196394920349, + "loss/reference_anchor": 1.057555079460144, + "rewards/accuracies": 0.875, + "rewards/chosen": 4.371184349060059, + "rewards/margins": 2.8844447135925293, + "rewards/rejected": 1.4867396354675293, + "step": 525 + }, + { + "drift/chosen_abs": 0.3064503073692322, + "drift/rejected_abs": 0.1510452926158905, + "epoch": 0.5063895855726741, + "grad_norm": 27.0, + "learning_rate": 2.6199453213033593e-07, + "logits/chosen": -2.0389158725738525, + "logits/rejected": -2.002628803253174, + "logps/chosen": -0.28475403785705566, + "logps/rejected": -0.28787198662757874, + "loss": 23.84844970703125, + "loss/core": 23.816450119018555, + "loss/preference_sft": 0.28475403785705566, + "loss/reference_anchor": 0.3981722593307495, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.063196897506714, + "rewards/margins": 1.5666277408599854, + "rewards/rejected": 1.496569275856018, + "step": 530 + }, + { + "drift/chosen_abs": 0.3102155029773712, + "drift/rejected_abs": 0.10850729793310165, + "epoch": 0.5111668458139257, + "grad_norm": 58.0, + "learning_rate": 2.565442370769683e-07, + "logits/chosen": -2.1870720386505127, + "logits/rejected": -2.109102487564087, + "logps/chosen": -0.2847766578197479, + "logps/rejected": -0.27864956855773926, + "loss": 23.705360412597656, + "loss/core": 23.66876220703125, + "loss/preference_sft": 0.2847766578197479, + "loss/reference_anchor": 0.4595085680484772, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.1017348766326904, + "rewards/margins": 2.018510580062866, + "rewards/rejected": 1.0832241773605347, + "step": 535 + }, + { + "drift/chosen_abs": 0.15850523114204407, + "drift/rejected_abs": 0.15349014103412628, + "epoch": 0.5159441060551774, + "grad_norm": 40.5, + "learning_rate": 2.510908273211866e-07, + "logits/chosen": -2.1092724800109863, + "logits/rejected": -2.0874218940734863, + "logps/chosen": -0.27907878160476685, + "logps/rejected": -0.2851378917694092, + "loss": 25.15981674194336, + "loss/core": 25.145463943481445, + "loss/preference_sft": 0.27907878160476685, + "loss/reference_anchor": 0.163437157869339, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.5850121974945068, + "rewards/margins": 0.05249936506152153, + "rewards/rejected": 1.532512903213501, + "step": 540 + }, + { + "drift/chosen_abs": 0.2638896703720093, + "drift/rejected_abs": 0.09009301662445068, + "epoch": 0.520721366296429, + "grad_norm": 344.0, + "learning_rate": 2.456368983906791e-07, + "logits/chosen": -2.334498882293701, + "logits/rejected": -2.1754422187805176, + "logps/chosen": -0.28355711698532104, + "logps/rejected": -0.2911638617515564, + "loss": 23.94521141052246, + "loss/core": 23.90779685974121, + "loss/preference_sft": 0.28355711698532104, + "loss/reference_anchor": 0.47052621841430664, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.637960433959961, + "rewards/margins": 1.7409608364105225, + "rewards/rejected": 0.8969996571540833, + "step": 545 + }, + { + "drift/chosen_abs": 0.2771431505680084, + "drift/rejected_abs": 0.16234190762043, + "epoch": 0.5254986265376806, + "grad_norm": 94.0, + "learning_rate": 2.401850460602329e-07, + "logits/chosen": -1.901158332824707, + "logits/rejected": -1.9195445775985718, + "logps/chosen": -0.27585816383361816, + "logps/rejected": -0.29788321256637573, + "loss": 23.990930557250977, + "loss/core": 23.959190368652344, + "loss/preference_sft": 0.27585816383361816, + "loss/reference_anchor": 0.3956103026866913, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.7710883617401123, + "rewards/margins": 1.148720383644104, + "rewards/rejected": 1.6223682165145874, + "step": 550 + }, + { + "drift/chosen_abs": 0.33259114623069763, + "drift/rejected_abs": 0.10077937692403793, + "epoch": 0.5302758867789323, + "grad_norm": 55.5, + "learning_rate": 2.3473786511628575e-07, + "logits/chosen": -2.095702648162842, + "logits/rejected": -2.1536788940429688, + "logps/chosen": -0.308633029460907, + "logps/rejected": -0.2737549841403961, + "loss": 23.32325553894043, + "loss/core": 23.289531707763672, + "loss/preference_sft": 0.308633029460907, + "loss/reference_anchor": 0.4187850058078766, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.3254246711730957, + "rewards/margins": 2.317995548248291, + "rewards/rejected": 1.0074293613433838, + "step": 555 + }, + { + "drift/chosen_abs": 0.2825181186199188, + "drift/rejected_abs": 0.15401551127433777, + "epoch": 0.5350531470201839, + "grad_norm": 135.0, + "learning_rate": 2.2929794812194894e-07, + "logits/chosen": -2.1838390827178955, + "logits/rejected": -2.171027660369873, + "logps/chosen": -0.2748723030090332, + "logps/rejected": -0.2799665927886963, + "loss": 23.88632583618164, + "loss/core": 23.858224868774414, + "loss/preference_sft": 0.2748723030090332, + "loss/reference_anchor": 0.3471968472003937, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.8238039016723633, + "rewards/margins": 1.2850908041000366, + "rewards/rejected": 1.5387130975723267, + "step": 560 + }, + { + "drift/chosen_abs": 0.280038982629776, + "drift/rejected_abs": 0.16534456610679626, + "epoch": 0.5398304072614356, + "grad_norm": 140.0, + "learning_rate": 2.2386788418308665e-07, + "logits/chosen": -2.077315092086792, + "logits/rejected": -2.1030631065368652, + "logps/chosen": -0.28564533591270447, + "logps/rejected": -0.3059332072734833, + "loss": 24.126893997192383, + "loss/core": 24.075992584228516, + "loss/preference_sft": 0.28564533591270447, + "loss/reference_anchor": 0.6501015424728394, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.7995214462280273, + "rewards/margins": 1.1462886333465576, + "rewards/rejected": 1.6532329320907593, + "step": 565 + }, + { + "drift/chosen_abs": 0.3362275958061218, + "drift/rejected_abs": 0.1704963594675064, + "epoch": 0.5446076675026872, + "grad_norm": 316.0, + "learning_rate": 2.184502577160426e-07, + "logits/chosen": -2.07682466506958, + "logits/rejected": -2.1273488998413086, + "logps/chosen": -0.286944180727005, + "logps/rejected": -0.2869289517402649, + "loss": 23.644794464111328, + "loss/core": 23.60843276977539, + "loss/preference_sft": 0.286944180727005, + "loss/reference_anchor": 0.4561174809932709, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.3612422943115234, + "rewards/margins": 1.6572866439819336, + "rewards/rejected": 1.703955888748169, + "step": 570 + }, + { + "drift/chosen_abs": 0.18439261615276337, + "drift/rejected_abs": 0.0657983049750328, + "epoch": 0.5493849277439389, + "grad_norm": 43.0, + "learning_rate": 2.1304764721759732e-07, + "logits/chosen": -2.1571121215820312, + "logits/rejected": -2.188605546951294, + "logps/chosen": -0.3006740212440491, + "logps/rejected": -0.32234296202659607, + "loss": 24.048736572265625, + "loss/core": 24.03409194946289, + "loss/preference_sft": 0.3006740212440491, + "loss/reference_anchor": 0.1651538759469986, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.8426148891448975, + "rewards/margins": 1.273414134979248, + "rewards/rejected": 0.569200873374939, + "step": 575 + }, + { + "drift/chosen_abs": 0.14222335815429688, + "drift/rejected_abs": 0.09565664827823639, + "epoch": 0.5541621879851905, + "grad_norm": 30.25, + "learning_rate": 2.0766262403774385e-07, + "logits/chosen": -2.4316046237945557, + "logits/rejected": -2.4871370792388916, + "logps/chosen": -0.26630157232284546, + "logps/rejected": -0.2813344895839691, + "loss": 24.60654067993164, + "loss/core": 24.598276138305664, + "loss/preference_sft": 0.26630157232284546, + "loss/reference_anchor": 0.08357779681682587, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.4222335815429688, + "rewards/margins": 0.46712565422058105, + "rewards/rejected": 0.9551078677177429, + "step": 580 + }, + { + "drift/chosen_abs": 0.22317564487457275, + "drift/rejected_abs": 0.13778264820575714, + "epoch": 0.5589394482264421, + "grad_norm": 68.0, + "learning_rate": 2.0229775115586378e-07, + "logits/chosen": -2.1374573707580566, + "logits/rejected": -2.1010804176330566, + "logps/chosen": -0.26939648389816284, + "logps/rejected": -0.2822672426700592, + "loss": 24.302589416503906, + "loss/core": 24.24777603149414, + "loss/preference_sft": 0.26939648389816284, + "loss/reference_anchor": 0.7039062976837158, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.230914354324341, + "rewards/margins": 0.8574030995368958, + "rewards/rejected": 1.3735109567642212, + "step": 585 + }, + { + "drift/chosen_abs": 0.3644371032714844, + "drift/rejected_abs": 0.104056216776371, + "epoch": 0.5637167084676937, + "grad_norm": 51.0, + "learning_rate": 1.9695558196088844e-07, + "logits/chosen": -2.0276551246643066, + "logits/rejected": -2.1235756874084473, + "logps/chosen": -0.2852238714694977, + "logps/rejected": -0.2887842059135437, + "loss": 23.151445388793945, + "loss/core": 23.115150451660156, + "loss/preference_sft": 0.2852238714694977, + "loss/reference_anchor": 0.45538654923439026, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.6441140174865723, + "rewards/margins": 2.6299378871917725, + "rewards/rejected": 1.0141762495040894, + "step": 590 + }, + { + "drift/chosen_abs": 0.1715865433216095, + "drift/rejected_abs": 0.09020796418190002, + "epoch": 0.5684939687089454, + "grad_norm": 69.5, + "learning_rate": 1.9163865903602372e-07, + "logits/chosen": -2.0477294921875, + "logits/rejected": -2.0416600704193115, + "logps/chosen": -0.2748182415962219, + "logps/rejected": -0.27176275849342346, + "loss": 24.21047592163086, + "loss/core": 24.204145431518555, + "loss/preference_sft": 0.2748182415962219, + "loss/reference_anchor": 0.05691518262028694, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.7147114276885986, + "rewards/margins": 0.8242376446723938, + "rewards/rejected": 0.8904738426208496, + "step": 595 + }, + { + "drift/chosen_abs": 0.18755662441253662, + "drift/rejected_abs": 0.10950811952352524, + "epoch": 0.5732712289501971, + "grad_norm": 103.0, + "learning_rate": 1.8634951294861806e-07, + "logits/chosen": -2.2788078784942627, + "logits/rejected": -2.2681713104248047, + "logps/chosen": -0.266190767288208, + "logps/rejected": -0.2825302183628082, + "loss": 24.342437744140625, + "loss/core": 24.329063415527344, + "loss/preference_sft": 0.266190767288208, + "loss/reference_anchor": 0.15168902277946472, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.8744417428970337, + "rewards/margins": 0.7936787009239197, + "rewards/rejected": 1.0807631015777588, + "step": 600 + }, + { + "drift/chosen_abs": 0.42088302969932556, + "drift/rejected_abs": 0.1206652969121933, + "epoch": 0.5780484891914487, + "grad_norm": 122.5, + "learning_rate": 1.810906610457502e-07, + "logits/chosen": -2.1177077293395996, + "logits/rejected": -2.1916587352752686, + "logps/chosen": -0.2778300940990448, + "logps/rejected": -0.275773286819458, + "loss": 23.372730255126953, + "loss/core": 23.302967071533203, + "loss/preference_sft": 0.2778300940990448, + "loss/reference_anchor": 0.9023693203926086, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.206851482391357, + "rewards/margins": 3.001718759536743, + "rewards/rejected": 1.2051331996917725, + "step": 605 + }, + { + "drift/chosen_abs": 0.4304037094116211, + "drift/rejected_abs": 0.19502094388008118, + "epoch": 0.5828257494327004, + "grad_norm": 145.0, + "learning_rate": 1.7586460625610726e-07, + "logits/chosen": -2.118242025375366, + "logits/rejected": -2.1343255043029785, + "logps/chosen": -0.27321526408195496, + "logps/rejected": -0.27437523007392883, + "loss": 23.397014617919922, + "loss/core": 23.323720932006836, + "loss/preference_sft": 0.27321526408195496, + "loss/reference_anchor": 0.9498865008354187, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 4.302402973175049, + "rewards/margins": 2.3571293354034424, + "rewards/rejected": 1.945273995399475, + "step": 610 + }, + { + "drift/chosen_abs": 0.27419549226760864, + "drift/rejected_abs": 0.1494186818599701, + "epoch": 0.587603009673952, + "grad_norm": 69.0, + "learning_rate": 1.70673835898727e-07, + "logits/chosen": -1.967576026916504, + "logits/rejected": -2.034571409225464, + "logps/chosen": -0.3140396773815155, + "logps/rejected": -0.3035440444946289, + "loss": 23.98012351989746, + "loss/core": 23.95610237121582, + "loss/preference_sft": 0.3140396773815155, + "loss/reference_anchor": 0.28889569640159607, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.739633321762085, + "rewards/margins": 1.2492014169692993, + "rewards/rejected": 1.4904316663742065, + "step": 615 + }, + { + "drift/chosen_abs": 0.16910451650619507, + "drift/rejected_abs": 0.14680986106395721, + "epoch": 0.5923802699152036, + "grad_norm": 61.25, + "learning_rate": 1.6552082049916824e-07, + "logits/chosen": -2.143625259399414, + "logits/rejected": -2.1699137687683105, + "logps/chosen": -0.286045640707016, + "logps/rejected": -0.2911421060562134, + "loss": 25.047000885009766, + "loss/core": 25.030277252197266, + "loss/preference_sft": 0.286045640707016, + "loss/reference_anchor": 0.19438524544239044, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.690314531326294, + "rewards/margins": 0.22506046295166016, + "rewards/rejected": 1.4652539491653442, + "step": 620 + }, + { + "drift/chosen_abs": 0.388736367225647, + "drift/rejected_abs": 0.1826535165309906, + "epoch": 0.5971575301564552, + "grad_norm": 95.5, + "learning_rate": 1.6040801261367493e-07, + "logits/chosen": -1.9324958324432373, + "logits/rejected": -1.961570143699646, + "logps/chosen": -0.304928719997406, + "logps/rejected": -0.33531856536865234, + "loss": 24.033096313476562, + "loss/core": 23.950403213500977, + "loss/preference_sft": 0.304928719997406, + "loss/reference_anchor": 1.0720922946929932, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.885380268096924, + "rewards/margins": 2.3656325340270996, + "rewards/rejected": 1.5197473764419556, + "step": 625 + }, + { + "drift/chosen_abs": 0.29494503140449524, + "drift/rejected_abs": 0.12427574396133423, + "epoch": 0.6019347903977069, + "grad_norm": 57.0, + "learning_rate": 1.5533784566189175e-07, + "logits/chosen": -1.9347823858261108, + "logits/rejected": -2.0498507022857666, + "logps/chosen": -0.27183371782302856, + "logps/rejected": -0.2836715281009674, + "loss": 23.67662239074707, + "loss/core": 23.64113998413086, + "loss/preference_sft": 0.27183371782302856, + "loss/reference_anchor": 0.4459313452243805, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.949350595474243, + "rewards/margins": 1.7082860469818115, + "rewards/rejected": 1.2410645484924316, + "step": 630 + }, + { + "drift/chosen_abs": 0.23291683197021484, + "drift/rejected_abs": 0.10206238925457001, + "epoch": 0.6067120506389586, + "grad_norm": 222.0, + "learning_rate": 1.5031273276868845e-07, + "logits/chosen": -1.984095573425293, + "logits/rejected": -2.0696752071380615, + "logps/chosen": -0.3027849495410919, + "logps/rejected": -0.2994707524776459, + "loss": 23.96482276916504, + "loss/core": 23.94552230834961, + "loss/preference_sft": 0.3027849495410919, + "loss/reference_anchor": 0.2270624190568924, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.327078342437744, + "rewards/margins": 1.3064545392990112, + "rewards/rejected": 1.0206239223480225, + "step": 635 + }, + { + "drift/chosen_abs": 0.1955791413784027, + "drift/rejected_abs": 0.08006999641656876, + "epoch": 0.6114893108802102, + "grad_norm": 49.5, + "learning_rate": 1.4533506561564305e-07, + "logits/chosen": -2.2103934288024902, + "logits/rejected": -2.2860031127929688, + "logps/chosen": -0.3017186224460602, + "logps/rejected": -0.2985337972640991, + "loss": 24.067489624023438, + "loss/core": 24.050811767578125, + "loss/preference_sft": 0.3017186224460602, + "loss/reference_anchor": 0.19217993319034576, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.9552767276763916, + "rewards/margins": 1.1555898189544678, + "rewards/rejected": 0.7996870279312134, + "step": 640 + }, + { + "drift/chosen_abs": 0.16859070956707, + "drift/rejected_abs": 0.12346317619085312, + "epoch": 0.6162665711214619, + "grad_norm": 117.5, + "learning_rate": 1.404072133027306e-07, + "logits/chosen": -2.1554760932922363, + "logits/rejected": -2.13128662109375, + "logps/chosen": -0.2895345687866211, + "logps/rejected": -0.2951858341693878, + "loss": 24.680572509765625, + "loss/core": 24.669631958007812, + "loss/preference_sft": 0.2895345687866211, + "loss/reference_anchor": 0.11693717539310455, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.6848987340927124, + "rewards/margins": 0.4532623291015625, + "rewards/rejected": 1.2316365242004395, + "step": 645 + }, + { + "drift/chosen_abs": 0.13376560807228088, + "drift/rejected_abs": 0.08641010522842407, + "epoch": 0.6210438313627135, + "grad_norm": 66.5, + "learning_rate": 1.3553152122076078e-07, + "logits/chosen": -2.2198781967163086, + "logits/rejected": -2.170907974243164, + "logps/chosen": -0.28950920701026917, + "logps/rejected": -0.2986505925655365, + "loss": 24.61799430847168, + "loss/core": 24.610118865966797, + "loss/preference_sft": 0.28950920701026917, + "loss/reference_anchor": 0.07604795694351196, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.3332895040512085, + "rewards/margins": 0.4888167381286621, + "rewards/rejected": 0.8444727063179016, + "step": 650 + }, + { + "drift/chosen_abs": 0.2010502815246582, + "drift/rejected_abs": 0.0992017537355423, + "epoch": 0.6258210916039652, + "grad_norm": 304.0, + "learning_rate": 1.3071030993509787e-07, + "logits/chosen": -2.059145450592041, + "logits/rejected": -2.1219570636749268, + "logps/chosen": -0.2817350924015045, + "logps/rejected": -0.28943654894828796, + "loss": 24.06000328063965, + "loss/core": 24.04839515686035, + "loss/preference_sft": 0.2817350924015045, + "loss/reference_anchor": 0.12662085890769958, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.009782314300537, + "rewards/margins": 1.0185507535934448, + "rewards/rejected": 0.9912316203117371, + "step": 655 + }, + { + "drift/chosen_abs": 0.2789165675640106, + "drift/rejected_abs": 0.1331566572189331, + "epoch": 0.6305983518452167, + "grad_norm": 45.75, + "learning_rate": 1.2594587408119804e-07, + "logits/chosen": -2.0298891067504883, + "logits/rejected": -2.0245847702026367, + "logps/chosen": -0.29279690980911255, + "logps/rejected": -0.30883678793907166, + "loss": 24.24443244934082, + "loss/core": 24.194019317626953, + "loss/preference_sft": 0.29279690980911255, + "loss/reference_anchor": 0.6429013609886169, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.7881667613983154, + "rewards/margins": 1.5086616277694702, + "rewards/rejected": 1.2795050144195557, + "step": 660 + }, + { + "drift/chosen_abs": 0.23610401153564453, + "drift/rejected_abs": 0.1503429114818573, + "epoch": 0.6353756120864684, + "grad_norm": 123.5, + "learning_rate": 1.2124048127248644e-07, + "logits/chosen": -2.188072681427002, + "logits/rejected": -2.164827823638916, + "logps/chosen": -0.28306499123573303, + "logps/rejected": -0.27927109599113464, + "loss": 24.441707611083984, + "loss/core": 24.4201717376709, + "loss/preference_sft": 0.28306499123573303, + "loss/reference_anchor": 0.25882014632225037, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.3602824211120605, + "rewards/margins": 0.8587505221366882, + "rewards/rejected": 1.5015321969985962, + "step": 665 + }, + { + "drift/chosen_abs": 0.3666432499885559, + "drift/rejected_abs": 0.1319333016872406, + "epoch": 0.64015287232772, + "grad_norm": 44.0, + "learning_rate": 1.1659637102109712e-07, + "logits/chosen": -1.9824602603912354, + "logits/rejected": -2.0293171405792236, + "logps/chosen": -0.2721920907497406, + "logps/rejected": -0.31627121567726135, + "loss": 23.281818389892578, + "loss/core": 23.24245834350586, + "loss/preference_sft": 0.2721920907497406, + "loss/reference_anchor": 0.49754709005355835, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.666236162185669, + "rewards/margins": 2.3700363636016846, + "rewards/rejected": 1.2962002754211426, + "step": 670 + }, + { + "drift/chosen_abs": 0.162507563829422, + "drift/rejected_abs": 0.09013229608535767, + "epoch": 0.6449301325689717, + "grad_norm": 93.0, + "learning_rate": 1.1201575367198546e-07, + "logits/chosen": -2.038818359375, + "logits/rejected": -2.0799050331115723, + "logps/chosen": -0.28820866346359253, + "logps/rejected": -0.28691327571868896, + "loss": 24.332555770874023, + "loss/core": 24.325624465942383, + "loss/preference_sft": 0.28820866346359253, + "loss/reference_anchor": 0.06361403316259384, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6243938207626343, + "rewards/margins": 0.730514645576477, + "rewards/rejected": 0.893879234790802, + "step": 675 + }, + { + "drift/chosen_abs": 0.2139333039522171, + "drift/rejected_abs": 0.13963082432746887, + "epoch": 0.6497073928102234, + "grad_norm": 43.75, + "learning_rate": 1.0750080935092423e-07, + "logits/chosen": -2.1613402366638184, + "logits/rejected": -2.079357385635376, + "logps/chosen": -0.28284314274787903, + "logps/rejected": -0.2906249463558197, + "loss": 24.423181533813477, + "loss/core": 24.4051456451416, + "loss/preference_sft": 0.28284314274787903, + "loss/reference_anchor": 0.2122153490781784, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.1380627155303955, + "rewards/margins": 0.7430015802383423, + "rewards/rejected": 1.3950612545013428, + "step": 680 + }, + { + "drift/chosen_abs": 0.28872284293174744, + "drift/rejected_abs": 0.15741479396820068, + "epoch": 0.654484653051475, + "grad_norm": 57.25, + "learning_rate": 1.0305368692688174e-07, + "logits/chosen": -2.088265895843506, + "logits/rejected": -2.072359561920166, + "logps/chosen": -0.253498375415802, + "logps/rejected": -0.2732449471950531, + "loss": 24.38766098022461, + "loss/core": 24.349506378173828, + "loss/preference_sft": 0.253498375415802, + "loss/reference_anchor": 0.4833889901638031, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.8872287273406982, + "rewards/margins": 1.3145825862884521, + "rewards/rejected": 1.5726460218429565, + "step": 685 + }, + { + "drift/chosen_abs": 0.17745468020439148, + "drift/rejected_abs": 0.1100490465760231, + "epoch": 0.6592619132927267, + "grad_norm": 51.5, + "learning_rate": 9.867650298927643e-08, + "logits/chosen": -1.8866245746612549, + "logits/rejected": -1.9085317850112915, + "logps/chosen": -0.307719886302948, + "logps/rejected": -0.32763782143592834, + "loss": 24.435400009155273, + "loss/core": 24.425445556640625, + "loss/preference_sft": 0.307719886302948, + "loss/reference_anchor": 0.10196495056152344, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7728378772735596, + "rewards/margins": 0.6770357489585876, + "rewards/rejected": 1.0958020687103271, + "step": 690 + }, + { + "drift/chosen_abs": 0.1973785161972046, + "drift/rejected_abs": 0.06951657682657242, + "epoch": 0.6640391735339782, + "grad_norm": 65.5, + "learning_rate": 9.437134084059515e-08, + "logits/chosen": -2.272217273712158, + "logits/rejected": -2.280867576599121, + "logps/chosen": -0.3060583770275116, + "logps/rejected": -0.29238781332969666, + "loss": 24.02126121520996, + "loss/core": 24.004791259765625, + "loss/preference_sft": 0.3060583770275116, + "loss/reference_anchor": 0.18900753557682037, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.972781777381897, + "rewards/margins": 1.285192847251892, + "rewards/rejected": 0.6875890493392944, + "step": 695 + }, + { + "drift/chosen_abs": 0.22317072749137878, + "drift/rejected_abs": 0.16237227618694305, + "epoch": 0.6688164337752299, + "grad_norm": 1552.0, + "learning_rate": 9.014024950485383e-08, + "logits/chosen": -2.1077773571014404, + "logits/rejected": -2.214353561401367, + "logps/chosen": -0.29673534631729126, + "logps/rejected": -0.3178013861179352, + "loss": 24.56673240661621, + "loss/core": 24.545368194580078, + "loss/preference_sft": 0.29673534631729126, + "loss/reference_anchor": 0.25515273213386536, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.231229305267334, + "rewards/margins": 0.6277844309806824, + "rewards/rejected": 1.603445053100586, + "step": 700 + }, + { + "drift/chosen_abs": 0.341328501701355, + "drift/rejected_abs": 0.15423105657100677, + "epoch": 0.6735936940164815, + "grad_norm": 132.0, + "learning_rate": 8.598524275237321e-08, + "logits/chosen": -2.1965928077697754, + "logits/rejected": -2.154327630996704, + "logps/chosen": -0.27952122688293457, + "logps/rejected": -0.2664346396923065, + "loss": 23.59442138671875, + "loss/core": 23.561824798583984, + "loss/preference_sft": 0.27952122688293457, + "loss/reference_anchor": 0.40662726759910583, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.412951946258545, + "rewards/margins": 1.871299386024475, + "rewards/rejected": 1.5416524410247803, + "step": 705 + }, + { + "drift/chosen_abs": 0.3609875440597534, + "drift/rejected_abs": 0.19022467732429504, + "epoch": 0.6783709542577332, + "grad_norm": 576.0, + "learning_rate": 8.190829814133293e-08, + "logits/chosen": -1.8862717151641846, + "logits/rejected": -1.87451171875, + "logps/chosen": -0.2760660648345947, + "logps/rejected": -0.2974444329738617, + "loss": 23.68022346496582, + "loss/core": 23.627628326416016, + "loss/preference_sft": 0.2760660648345947, + "loss/reference_anchor": 0.6736692786216736, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.608205795288086, + "rewards/margins": 1.707396149635315, + "rewards/rejected": 1.90080988407135, + "step": 710 + }, + { + "drift/chosen_abs": 0.18668507039546967, + "drift/rejected_abs": 0.07842984795570374, + "epoch": 0.6831482144989849, + "grad_norm": 57.25, + "learning_rate": 7.791135607656146e-08, + "logits/chosen": -2.2538669109344482, + "logits/rejected": -2.2352232933044434, + "logps/chosen": -0.28401559591293335, + "logps/rejected": -0.3294680714607239, + "loss": 24.112934112548828, + "loss/core": 24.091781616210938, + "loss/preference_sft": 0.28401559591293335, + "loss/reference_anchor": 0.25365859270095825, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8665748834609985, + "rewards/margins": 1.0951749086380005, + "rewards/rejected": 0.771399974822998, + "step": 715 + }, + { + "drift/chosen_abs": 0.21051010489463806, + "drift/rejected_abs": 0.1500566005706787, + "epoch": 0.6879254747402365, + "grad_norm": 136.0, + "learning_rate": 7.399631888600796e-08, + "logits/chosen": -1.9951555728912354, + "logits/rejected": -2.00339674949646, + "logps/chosen": -0.2962464392185211, + "logps/rejected": -0.30031222105026245, + "loss": 24.666812896728516, + "loss/core": 24.645822525024414, + "loss/preference_sft": 0.2962464392185211, + "loss/reference_anchor": 0.25020235776901245, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.1030123233795166, + "rewards/margins": 0.6037241816520691, + "rewards/rejected": 1.4992883205413818, + "step": 720 + }, + { + "drift/chosen_abs": 0.15870442986488342, + "drift/rejected_abs": 0.07348223030567169, + "epoch": 0.6927027349814882, + "grad_norm": 66.5, + "learning_rate": 7.016504991533726e-08, + "logits/chosen": -2.1241562366485596, + "logits/rejected": -2.2400271892547607, + "logps/chosen": -0.28562217950820923, + "logps/rejected": -0.2771227955818176, + "loss": 24.232282638549805, + "loss/core": 24.223468780517578, + "loss/preference_sft": 0.28562217950820923, + "loss/reference_anchor": 0.08894944936037064, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.5855097770690918, + "rewards/margins": 0.8525517582893372, + "rewards/rejected": 0.7329580783843994, + "step": 725 + }, + { + "drift/chosen_abs": 0.102408766746521, + "drift/rejected_abs": 0.08807560056447983, + "epoch": 0.6974799952227397, + "grad_norm": 956.0, + "learning_rate": 6.641937264107867e-08, + "logits/chosen": -2.2225677967071533, + "logits/rejected": -2.271383285522461, + "logps/chosen": -0.28616029024124146, + "logps/rejected": -0.293459951877594, + "loss": 24.99797821044922, + "loss/core": 24.98850440979004, + "loss/preference_sft": 0.28616029024124146, + "loss/reference_anchor": 0.09770476073026657, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.022210717201233, + "rewards/margins": 0.14174190163612366, + "rewards/rejected": 0.8804686665534973, + "step": 730 + }, + { + "drift/chosen_abs": 0.1421697437763214, + "drift/rejected_abs": 0.06707651913166046, + "epoch": 0.7022572554639914, + "grad_norm": 119.0, + "learning_rate": 6.276106980274944e-08, + "logits/chosen": -2.0496909618377686, + "logits/rejected": -2.097294330596924, + "logps/chosen": -0.31965214014053345, + "logps/rejected": -0.31716209650039673, + "loss": 24.297515869140625, + "loss/core": 24.289684295654297, + "loss/preference_sft": 0.31965214014053345, + "loss/reference_anchor": 0.07245230674743652, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.4212677478790283, + "rewards/margins": 0.7517526149749756, + "rewards/rejected": 0.6695151329040527, + "step": 735 + }, + { + "drift/chosen_abs": 0.2564278841018677, + "drift/rejected_abs": 0.18303130567073822, + "epoch": 0.707034515705243, + "grad_norm": 71.0, + "learning_rate": 5.919188255436777e-08, + "logits/chosen": -2.1012163162231445, + "logits/rejected": -2.1193010807037354, + "logps/chosen": -0.3130371868610382, + "logps/rejected": -0.3160560727119446, + "loss": 25.59921646118164, + "loss/core": 25.540393829345703, + "loss/preference_sft": 0.3130371868610382, + "loss/reference_anchor": 0.7530360221862793, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.564279079437256, + "rewards/margins": 0.7340623736381531, + "rewards/rejected": 1.830216646194458, + "step": 740 + }, + { + "drift/chosen_abs": 0.3141566514968872, + "drift/rejected_abs": 0.15219292044639587, + "epoch": 0.7118117759464947, + "grad_norm": 126.0, + "learning_rate": 5.571350963575727e-08, + "logits/chosen": -2.0531277656555176, + "logits/rejected": -2.09647798538208, + "logps/chosen": -0.2884739339351654, + "logps/rejected": -0.2817172408103943, + "loss": 23.735965728759766, + "loss/core": 23.71042823791504, + "loss/preference_sft": 0.2884739339351654, + "loss/reference_anchor": 0.3116436302661896, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.141024351119995, + "rewards/margins": 1.6229345798492432, + "rewards/rejected": 1.5180898904800415, + "step": 745 + }, + { + "drift/chosen_abs": 0.12072908878326416, + "drift/rejected_abs": 0.12682561576366425, + "epoch": 0.7165890361877463, + "grad_norm": 340.0, + "learning_rate": 5.232760656403923e-08, + "logits/chosen": -2.1889564990997314, + "logits/rejected": -2.1800601482391357, + "logps/chosen": -0.3116965889930725, + "logps/rejected": -0.3028327226638794, + "loss": 25.507654190063477, + "loss/core": 25.483829498291016, + "loss/preference_sft": 0.3116965889930725, + "loss/reference_anchor": 0.28652527928352356, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.2060551643371582, + "rewards/margins": -0.05827074497938156, + "rewards/rejected": 1.2643258571624756, + "step": 750 + }, + { + "drift/chosen_abs": 0.24006719887256622, + "drift/rejected_abs": 0.17291317880153656, + "epoch": 0.721366296428998, + "grad_norm": 133.0, + "learning_rate": 4.9035784845695675e-08, + "logits/chosen": -1.9279276132583618, + "logits/rejected": -1.8381156921386719, + "logps/chosen": -0.3074774742126465, + "logps/rejected": -0.29301613569259644, + "loss": 24.52367401123047, + "loss/core": 24.506526947021484, + "loss/preference_sft": 0.3074774742126465, + "loss/reference_anchor": 0.19787776470184326, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.3972103595733643, + "rewards/margins": 0.6746050715446472, + "rewards/rejected": 1.7226054668426514, + "step": 755 + }, + { + "drift/chosen_abs": 0.19889570772647858, + "drift/rejected_abs": 0.08718482404947281, + "epoch": 0.7261435566702497, + "grad_norm": 64.0, + "learning_rate": 4.583961120958027e-08, + "logits/chosen": -2.186131715774536, + "logits/rejected": -2.203399419784546, + "logps/chosen": -0.30299732089042664, + "logps/rejected": -0.29353067278862, + "loss": 24.01272964477539, + "loss/core": 24.000045776367188, + "loss/preference_sft": 0.30299732089042664, + "loss/reference_anchor": 0.13879242539405823, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9878475666046143, + "rewards/margins": 1.1234432458877563, + "rewards/rejected": 0.8644043803215027, + "step": 760 + }, + { + "drift/chosen_abs": 0.1797488033771515, + "drift/rejected_abs": 0.1066058874130249, + "epoch": 0.7309208169115012, + "grad_norm": 106.5, + "learning_rate": 4.2740606861239594e-08, + "logits/chosen": -2.08768892288208, + "logits/rejected": -2.158001184463501, + "logps/chosen": -0.275774210691452, + "logps/rejected": -0.27186232805252075, + "loss": 24.34197425842285, + "loss/core": 24.330942153930664, + "loss/preference_sft": 0.275774210691452, + "loss/reference_anchor": 0.11952831596136093, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.7971556186676025, + "rewards/margins": 0.7334049344062805, + "rewards/rejected": 1.0637508630752563, + "step": 765 + }, + { + "drift/chosen_abs": 0.3747478127479553, + "drift/rejected_abs": 0.2356497049331665, + "epoch": 0.7356980771527529, + "grad_norm": 222.0, + "learning_rate": 3.974024675890189e-08, + "logits/chosen": -1.9823691844940186, + "logits/rejected": -1.9497228860855103, + "logps/chosen": -0.3086339831352234, + "logps/rejected": -0.28454282879829407, + "loss": 24.525297164916992, + "loss/core": 24.43834114074707, + "loss/preference_sft": 0.3086339831352234, + "loss/reference_anchor": 1.1285443305969238, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.744788408279419, + "rewards/margins": 1.3908659219741821, + "rewards/rejected": 2.3539223670959473, + "step": 770 + }, + { + "drift/chosen_abs": 0.15283748507499695, + "drift/rejected_abs": 0.09471901506185532, + "epoch": 0.7404753373940045, + "grad_norm": 32.75, + "learning_rate": 3.683995891147695e-08, + "logits/chosen": -2.0964419841766357, + "logits/rejected": -2.1264514923095703, + "logps/chosen": -0.30610623955726624, + "logps/rejected": -0.29977482557296753, + "loss": 24.4835147857666, + "loss/core": 24.475711822509766, + "loss/preference_sft": 0.30610623955726624, + "loss/reference_anchor": 0.0734710842370987, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.5278440713882446, + "rewards/margins": 0.5806539058685303, + "rewards/rejected": 0.9471901059150696, + "step": 775 + }, + { + "drift/chosen_abs": 0.1896996945142746, + "drift/rejected_abs": 0.13039933145046234, + "epoch": 0.7452525976352562, + "grad_norm": 102.5, + "learning_rate": 3.404112369890108e-08, + "logits/chosen": -2.1795246601104736, + "logits/rejected": -2.1940202713012695, + "logps/chosen": -0.3016170859336853, + "logps/rejected": -0.30035489797592163, + "loss": 24.48188018798828, + "loss/core": 24.470382690429688, + "loss/preference_sft": 0.3016170859336853, + "loss/reference_anchor": 0.1231255754828453, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8962314128875732, + "rewards/margins": 0.5943207144737244, + "rewards/rejected": 1.3019107580184937, + "step": 780 + }, + { + "drift/chosen_abs": 0.3444829285144806, + "drift/rejected_abs": 0.13935281336307526, + "epoch": 0.7500298578765078, + "grad_norm": 42.5, + "learning_rate": 3.134507321515106e-08, + "logits/chosen": -1.9970579147338867, + "logits/rejected": -1.953643560409546, + "logps/chosen": -0.30979782342910767, + "logps/rejected": -0.2892361879348755, + "loss": 24.037694931030273, + "loss/core": 23.987783432006836, + "loss/preference_sft": 0.30979782342910767, + "loss/reference_anchor": 0.6345094442367554, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.4429678916931152, + "rewards/margins": 2.0524790287017822, + "rewards/rejected": 1.390488862991333, + "step": 785 + }, + { + "drift/chosen_abs": 0.26237040758132935, + "drift/rejected_abs": 0.19609911739826202, + "epoch": 0.7548071181177595, + "grad_norm": 780.0, + "learning_rate": 2.875309063423956e-08, + "logits/chosen": -2.1686906814575195, + "logits/rejected": -1.9951951503753662, + "logps/chosen": -0.28583893179893494, + "logps/rejected": -0.315361887216568, + "loss": 24.721479415893555, + "loss/core": 24.67513656616211, + "loss/preference_sft": 0.28583893179893494, + "loss/reference_anchor": 0.5893004536628723, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.6218254566192627, + "rewards/margins": 0.6629058122634888, + "rewards/rejected": 1.9589197635650635, + "step": 790 + }, + { + "drift/chosen_abs": 0.24005870521068573, + "drift/rejected_abs": 0.11813025176525116, + "epoch": 0.7595843783590112, + "grad_norm": 140.0, + "learning_rate": 2.626640959949375e-08, + "logits/chosen": -1.971401572227478, + "logits/rejected": -1.991015076637268, + "logps/chosen": -0.30549412965774536, + "logps/rejected": -0.296720415353775, + "loss": 24.04024887084961, + "loss/core": 23.997535705566406, + "loss/preference_sft": 0.30549412965774536, + "loss/reference_anchor": 0.5389922857284546, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.399735927581787, + "rewards/margins": 1.2197308540344238, + "rewards/rejected": 1.1800049543380737, + "step": 795 + }, + { + "drift/chosen_abs": 0.2291211634874344, + "drift/rejected_abs": 0.12003247439861298, + "epoch": 0.7643616386002627, + "grad_norm": 59.75, + "learning_rate": 2.388621363640797e-08, + "logits/chosen": -2.1559276580810547, + "logits/rejected": -2.1329169273376465, + "logps/chosen": -0.28992101550102234, + "logps/rejected": -0.2854466736316681, + "loss": 24.079273223876953, + "loss/core": 24.065502166748047, + "loss/preference_sft": 0.28992101550102234, + "loss/reference_anchor": 0.15460558235645294, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.290127754211426, + "rewards/margins": 1.0904570817947388, + "rewards/rejected": 1.1996705532073975, + "step": 800 + }, + { + "drift/chosen_abs": 0.22606876492500305, + "drift/rejected_abs": 0.2016010731458664, + "epoch": 0.7691388988415144, + "grad_norm": 104.0, + "learning_rate": 2.1613635589349756e-08, + "logits/chosen": -2.028775453567505, + "logits/rejected": -2.0167717933654785, + "logps/chosen": -0.2745590806007385, + "logps/rejected": -0.2676575779914856, + "loss": 24.971622467041016, + "loss/core": 24.949968338012695, + "loss/preference_sft": 0.2745590806007385, + "loss/reference_anchor": 0.2612423002719879, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.2601115703582764, + "rewards/margins": 0.24478550255298615, + "rewards/rejected": 2.0153260231018066, + "step": 805 + }, + { + "drift/chosen_abs": 0.15367405116558075, + "drift/rejected_abs": 0.14802715182304382, + "epoch": 0.773916159082766, + "grad_norm": 51.5, + "learning_rate": 1.944975708238708e-08, + "logits/chosen": -2.2288622856140137, + "logits/rejected": -2.1330811977386475, + "logps/chosen": -0.30788537859916687, + "logps/rejected": -0.31650078296661377, + "loss": 25.090599060058594, + "loss/core": 25.078384399414062, + "loss/preference_sft": 0.30788537859916687, + "loss/reference_anchor": 0.13205906748771667, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.534415364265442, + "rewards/margins": 0.05462510138750076, + "rewards/rejected": 1.4797903299331665, + "step": 810 + }, + { + "drift/chosen_abs": 0.2130316197872162, + "drift/rejected_abs": 0.11016812175512314, + "epoch": 0.7786934193240177, + "grad_norm": 145.0, + "learning_rate": 1.7395608004493884e-08, + "logits/chosen": -2.0962109565734863, + "logits/rejected": -2.1853909492492676, + "logps/chosen": -0.2830978333950043, + "logps/rejected": -0.2838820815086365, + "loss": 24.020706176757812, + "loss/core": 24.00937843322754, + "loss/preference_sft": 0.2830978333950043, + "loss/reference_anchor": 0.12274503707885742, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.1303162574768066, + "rewards/margins": 1.032294750213623, + "rewards/rejected": 1.0980216264724731, + "step": 815 + }, + { + "drift/chosen_abs": 0.1970573216676712, + "drift/rejected_abs": 0.07308342307806015, + "epoch": 0.7834706795652693, + "grad_norm": 45.25, + "learning_rate": 1.5452166019378987e-08, + "logits/chosen": -2.099004030227661, + "logits/rejected": -2.1342720985412598, + "logps/chosen": -0.2875714898109436, + "logps/rejected": -0.31725940108299255, + "loss": 23.889062881469727, + "loss/core": 23.877830505371094, + "loss/preference_sft": 0.2875714898109436, + "loss/reference_anchor": 0.12097527086734772, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.9672977924346924, + "rewards/margins": 1.2818208932876587, + "rewards/rejected": 0.685477077960968, + "step": 820 + }, + { + "drift/chosen_abs": 0.2467678040266037, + "drift/rejected_abs": 0.12167330831289291, + "epoch": 0.788247939806521, + "grad_norm": 183.0, + "learning_rate": 1.3620356100170788e-08, + "logits/chosen": -2.1444103717803955, + "logits/rejected": -2.247798442840576, + "logps/chosen": -0.3017248809337616, + "logps/rejected": -0.28473973274230957, + "loss": 23.85074234008789, + "loss/core": 23.836971282958984, + "loss/preference_sft": 0.3017248809337616, + "loss/reference_anchor": 0.15346962213516235, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.466996669769287, + "rewards/margins": 1.2521899938583374, + "rewards/rejected": 1.2148066759109497, + "step": 825 + }, + { + "drift/chosen_abs": 0.17280618846416473, + "drift/rejected_abs": 0.16574548184871674, + "epoch": 0.7930252000477725, + "grad_norm": 91.0, + "learning_rate": 1.190105008918041e-08, + "logits/chosen": -2.1808249950408936, + "logits/rejected": -2.1297428607940674, + "logps/chosen": -0.2784227430820465, + "logps/rejected": -0.30759522318840027, + "loss": 25.10189437866211, + "loss/core": 25.08694839477539, + "loss/preference_sft": 0.2784227430820465, + "loss/reference_anchor": 0.17144832015037537, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7274526357650757, + "rewards/margins": 0.07077233493328094, + "rewards/rejected": 1.6566804647445679, + "step": 830 + }, + { + "drift/chosen_abs": 0.24050016701221466, + "drift/rejected_abs": 0.11438360065221786, + "epoch": 0.7978024602890242, + "grad_norm": 58.75, + "learning_rate": 1.0295066282951737e-08, + "logits/chosen": -2.096271276473999, + "logits/rejected": -2.096055507659912, + "logps/chosen": -0.2547900378704071, + "logps/rejected": -0.3017795979976654, + "loss": 23.816970825195312, + "loss/core": 23.80051040649414, + "loss/preference_sft": 0.2547900378704071, + "loss/reference_anchor": 0.19398611783981323, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.4034855365753174, + "rewards/margins": 1.5796620845794678, + "rewards/rejected": 0.8238231539726257, + "step": 835 + }, + { + "drift/chosen_abs": 0.28091713786125183, + "drift/rejected_abs": 0.10494539886713028, + "epoch": 0.8025797205302759, + "grad_norm": 796.0, + "learning_rate": 8.803169042796765e-09, + "logits/chosen": -2.0359435081481934, + "logits/rejected": -2.085045576095581, + "logps/chosen": -0.3219825327396393, + "logps/rejected": -0.3002759516239166, + "loss": 23.931066513061523, + "loss/core": 23.892078399658203, + "loss/preference_sft": 0.3219825327396393, + "loss/reference_anchor": 0.4876865744590759, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.8080499172210693, + "rewards/margins": 1.7595913410186768, + "rewards/rejected": 1.0484585762023926, + "step": 840 + }, + { + "drift/chosen_abs": 0.141829252243042, + "drift/rejected_abs": 0.06329073011875153, + "epoch": 0.8073569807715275, + "grad_norm": 54.5, + "learning_rate": 7.4260684310008815e-09, + "logits/chosen": -2.1739792823791504, + "logits/rejected": -2.248871326446533, + "logps/chosen": -0.29366642236709595, + "logps/rejected": -0.3113446831703186, + "loss": 24.187685012817383, + "loss/core": 24.17953109741211, + "loss/preference_sft": 0.29366642236709595, + "loss/reference_anchor": 0.07929544895887375, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.4174824953079224, + "rewards/margins": 0.963509202003479, + "rewards/rejected": 0.45397335290908813, + "step": 845 + }, + { + "drift/chosen_abs": 0.15600529313087463, + "drift/rejected_abs": 0.09067943692207336, + "epoch": 0.8121342410127792, + "grad_norm": 314.0, + "learning_rate": 6.164419872871835e-09, + "logits/chosen": -2.2953596115112305, + "logits/rejected": -2.3431358337402344, + "logps/chosen": -0.2520250082015991, + "logps/rejected": -0.24778322875499725, + "loss": 24.37624168395996, + "loss/core": 24.37070083618164, + "loss/preference_sft": 0.2520250082015991, + "loss/reference_anchor": 0.04869046062231064, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.5578166246414185, + "rewards/margins": 0.6599420309066772, + "rewards/rejected": 0.8978745341300964, + "step": 850 + }, + { + "drift/chosen_abs": 0.27003389596939087, + "drift/rejected_abs": 0.12391098588705063, + "epoch": 0.8169115012540308, + "grad_norm": 143.0, + "learning_rate": 5.018823844792602e-09, + "logits/chosen": -2.0034213066101074, + "logits/rejected": -2.040942668914795, + "logps/chosen": -0.2998163104057312, + "logps/rejected": -0.2942708730697632, + "loss": 23.890283584594727, + "loss/core": 23.85587501525879, + "loss/preference_sft": 0.2998163104057312, + "loss/reference_anchor": 0.42880433797836304, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.6989176273345947, + "rewards/margins": 1.4601688385009766, + "rewards/rejected": 1.2387486696243286, + "step": 855 + }, + { + "drift/chosen_abs": 0.409585177898407, + "drift/rejected_abs": 0.19676879048347473, + "epoch": 0.8216887614952825, + "grad_norm": 37.25, + "learning_rate": 3.989825588427281e-09, + "logits/chosen": -1.9499990940093994, + "logits/rejected": -2.038193702697754, + "logps/chosen": -0.2986469268798828, + "logps/rejected": -0.2962108850479126, + "loss": 23.53319549560547, + "loss/core": 23.46843910217285, + "loss/preference_sft": 0.2986469268798828, + "loss/reference_anchor": 0.8335119485855103, + "rewards/accuracies": 0.875, + "rewards/chosen": 4.0953145027160645, + "rewards/margins": 2.138956308364868, + "rewards/rejected": 1.956358551979065, + "step": 860 + }, + { + "drift/chosen_abs": 0.24526743590831757, + "drift/rejected_abs": 0.15863600373268127, + "epoch": 0.826466021736534, + "grad_norm": 85.5, + "learning_rate": 3.077914851215585e-09, + "logits/chosen": -1.95840585231781, + "logits/rejected": -2.0312142372131348, + "logps/chosen": -0.29089808464050293, + "logps/rejected": -0.29388031363487244, + "loss": 24.42396354675293, + "loss/core": 24.399206161499023, + "loss/preference_sft": 0.29089808464050293, + "loss/reference_anchor": 0.30098140239715576, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.4520413875579834, + "rewards/margins": 0.8663004040718079, + "rewards/rejected": 1.5857408046722412, + "step": 865 + }, + { + "drift/chosen_abs": 0.3167889714241028, + "drift/rejected_abs": 0.09579071402549744, + "epoch": 0.8312432819777857, + "grad_norm": 91.0, + "learning_rate": 2.2835256532794387e-09, + "logits/chosen": -2.1105716228485107, + "logits/rejected": -2.119257926940918, + "logps/chosen": -0.3130267262458801, + "logps/rejected": -0.2952987551689148, + "loss": 23.687124252319336, + "loss/core": 23.64649772644043, + "loss/preference_sft": 0.3130267262458801, + "loss/reference_anchor": 0.5103899240493774, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.165898561477661, + "rewards/margins": 2.2113327980041504, + "rewards/rejected": 0.9545663595199585, + "step": 870 + }, + { + "drift/chosen_abs": 0.1863785833120346, + "drift/rejected_abs": 0.10340656340122223, + "epoch": 0.8360205422190374, + "grad_norm": 66.0, + "learning_rate": 1.6070360808531359e-09, + "logits/chosen": -2.1814465522766113, + "logits/rejected": -2.22865891456604, + "logps/chosen": -0.3077881932258606, + "logps/rejected": -0.3306838274002075, + "loss": 24.516956329345703, + "loss/core": 24.487125396728516, + "loss/preference_sft": 0.3077881932258606, + "loss/reference_anchor": 0.36692848801612854, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.8635761737823486, + "rewards/margins": 0.8385453224182129, + "rewards/rejected": 1.0250307321548462, + "step": 875 + }, + { + "drift/chosen_abs": 0.24594731628894806, + "drift/rejected_abs": 0.18863597512245178, + "epoch": 0.840797802460289, + "grad_norm": 194.0, + "learning_rate": 1.0487681063345854e-09, + "logits/chosen": -2.1177544593811035, + "logits/rejected": -2.1003167629241943, + "logps/chosen": -0.2906046509742737, + "logps/rejected": -0.3307008743286133, + "loss": 25.163158416748047, + "loss/core": 25.12496566772461, + "loss/preference_sft": 0.2906046509742737, + "loss/reference_anchor": 0.4801712930202484, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.458425521850586, + "rewards/margins": 0.6850012540817261, + "rewards/rejected": 1.7734243869781494, + "step": 880 + }, + { + "drift/chosen_abs": 0.27374953031539917, + "drift/rejected_abs": 0.11317259073257446, + "epoch": 0.8455750627015407, + "grad_norm": 96.0, + "learning_rate": 6.089874350439505e-10, + "logits/chosen": -2.0427865982055664, + "logits/rejected": -2.1189193725585938, + "logps/chosen": -0.2813889980316162, + "logps/rejected": -0.2768685817718506, + "loss": 23.570148468017578, + "loss/core": 23.551849365234375, + "loss/preference_sft": 0.2813889980316162, + "loss/reference_anchor": 0.215844064950943, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.7366836071014404, + "rewards/margins": 1.611907958984375, + "rewards/rejected": 1.1247756481170654, + "step": 885 + }, + { + "drift/chosen_abs": 0.246209055185318, + "drift/rejected_abs": 0.1439971625804901, + "epoch": 0.8503523229427923, + "grad_norm": 130.0, + "learning_rate": 2.8790337876233305e-10, + "logits/chosen": -1.9758459329605103, + "logits/rejected": -2.097522735595703, + "logps/chosen": -0.29943814873695374, + "logps/rejected": -0.27435192465782166, + "loss": 24.266326904296875, + "loss/core": 24.23988914489746, + "loss/preference_sft": 0.29943814873695374, + "loss/reference_anchor": 0.3225429058074951, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.461657762527466, + "rewards/margins": 1.0233776569366455, + "rewards/rejected": 1.4382802248001099, + "step": 890 + }, + { + "drift/chosen_abs": 0.23041865229606628, + "drift/rejected_abs": 0.15222403407096863, + "epoch": 0.855129583184044, + "grad_norm": 172.0, + "learning_rate": 8.566875611068503e-11, + "logits/chosen": -2.044543504714966, + "logits/rejected": -2.07798433303833, + "logps/chosen": -0.27221474051475525, + "logps/rejected": -0.2905210852622986, + "loss": 24.323848724365234, + "loss/core": 24.309627532958984, + "loss/preference_sft": 0.27221474051475525, + "loss/reference_anchor": 0.162424698472023, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.3031530380249023, + "rewards/margins": 0.7896323800086975, + "rewards/rejected": 1.51352059841156, + "step": 895 + }, + { + "drift/chosen_abs": 0.31285884976387024, + "drift/rejected_abs": 0.08625397086143494, + "epoch": 0.8599068434252956, + "grad_norm": 135.0, + "learning_rate": 2.3798198163782478e-12, + "logits/chosen": -2.055809736251831, + "logits/rejected": -2.176356315612793, + "logps/chosen": -0.2678307294845581, + "logps/rejected": -0.268280953168869, + "loss": 23.679527282714844, + "loss/core": 23.63846778869629, + "loss/preference_sft": 0.2678307294845581, + "loss/reference_anchor": 0.5206762552261353, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 3.1283411979675293, + "rewards/margins": 2.278930187225342, + "rewards/rejected": 0.8494111895561218, + "step": 900 + }, + { + "epoch": 0.8599068434252956, + "step": 900, + "total_flos": 0.0, + "train_loss": 24.184869588216145, + "train_runtime": 7109.0777, + "train_samples_per_second": 2.026, + "train_steps_per_second": 0.127 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..b5d92e0 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5117975d32aa6d5ee887a27ed5d586ce2b1ab4f86ce47449497c97462ed54443 +size 6993 diff --git a/training_status.json b/training_status.json new file mode 100644 index 0000000..97926e3 --- /dev/null +++ b/training_status.json @@ -0,0 +1,20 @@ +{ + "status": "completed", + "candidate_id": "inpo_b", + "method": "inpo_avg", + "seed": 42, + "gpu": 0, + "pid": 1079779, + "config": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/fair_demo_20260715/sweep/candidates/inpo_b/config.yaml", + "dataset": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/avg", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "098086245ba4", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/098086245ba4", + "started_at": "2026-07-15T15:36:21+09:00", + "returncode": 0, + "model_complete": true, + "finite_results": true, + "measured_step": null, + "completed_at": "2026-07-15T17:35:44+09:00" +}