From 17d80b4a2fe413352e032c05181b87fd80ebd423 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 25 Jul 2026 17:52:10 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: promotion/qwen3-8b-aaai27-flagship-ronpo-full-expect-s43 Source: Original Platform --- .gitattributes | 36 + README.md | 24 + all_results.json | 9 + chat_template.jinja | 89 + config.json | 71 + config.yaml | 59 + generation_config.json | 12 + job_status.json | 23 + model.safetensors | 3 + objective_protocol.json | 69 + pair_manifest.json | 34 + provenance.json | 12 + run_status.json | 13 + split_manifest.json | 23 + stability_gate.json | 38 + tokenizer.json | 3 + tokenizer_config.json | 30 + train_results.json | 9 + trainer_state.json | 4903 +++++++++++++++++++++++++++++++++++++++ training_args.bin | 3 + 20 files changed, 5463 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 config.yaml create mode 100644 generation_config.json create mode 100644 job_status.json create mode 100644 model.safetensors create mode 100644 objective_protocol.json create mode 100644 pair_manifest.json create mode 100644 provenance.json create mode 100644 run_status.json create mode 100644 split_manifest.json create mode 100644 stability_gate.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..d7894f5 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +--- +library_name: transformers +base_model: Qwen/Qwen3-8B +tags: +- ronpo +- mnpo +- preference-optimization +--- + +# qwen3-8b-aaai27-flagship-ronpo-full-expect-s43 + +Research checkpoint for the RONPO AAAI revision experiments. + +- Method: ronpo_full_expect +- Base model: `Qwen/Qwen3-8B` +- Seed: 43 +- Local source at upload time: `/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/ronpo_full_expect/seed43/attempt1` +- Uploaded at UTC: 2026-07-12T17:16:35Z +- Run status metadata: `{"attempt": 1, "effective_batch_size": 16, "method": "ronpo_full_expect", "objective_protocol": "objective_protocol.json", "optimizer_steps": 900, "seed": 43, "split_manifest": "split_manifest.json", "stability_gate": "stability_gate.json", "status": "completed", "wandb_run_id": "bd99ef514a5f", "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/bd99ef514a5f"}` + +AAAI-27 P1 matched budget: 900 optimizer steps, effective batch 16; passed non-thinking and collapse stability gates; sealed-test metrics are not used for checkpoint selection. + +Intended use: reproducibility and evaluation for the RONPO paper. This +checkpoint is not intended as a production assistant. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..cdae505 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.2860548271752086, + "total_flos": 0.0, + "train_loss": 0.48261189058423043, + "train_runtime": 8509.445, + "train_samples": 50340, + "train_samples_per_second": 1.692, + "train_steps_per_second": 0.106 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..2c42049 --- /dev/null +++ b/config.yaml @@ -0,0 +1,59 @@ +model_name_or_path: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218 +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + ? /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/ronpo_full_expect + : 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: ronpo +eta: 0.0075 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.05 +preference_sft_weight: 0.005 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 1.0e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 43 +gradient_accumulation_steps: 8 +gradient_checkpointing: false +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 2 +per_device_eval_batch_size: 2 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/ronpo_full_expect/seed43/attempt1 +run_name: aaai27-flagship-full-ronpo_full_expect-s43-a1 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/job_status.json b/job_status.json new file mode 100644 index 0000000..1b74acf --- /dev/null +++ b/job_status.json @@ -0,0 +1,23 @@ +{ + "status": "completed", + "stage": "full", + "method": "ronpo_full_expect", + "seed": 43, + "attempt": 1, + "gpu": 1, + "gpus": [ + 1 + ], + "returncode": 0, + "finite_metrics": true, + "oom": false, + "traceback": false, + "reason": "passed", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "bd99ef514a5f", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/bd99ef514a5f", + "output_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/ronpo_full_expect/seed43/attempt1", + "log": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/logs/full_ronpo_full_expect_s43_a1.log", + "completed_at": "2026-07-12T17:12:04Z" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..39964b3 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0304be907896916e4ed1d5ff5d588afd82a461d59dbb040efde35289f5ee0ca +size 16381517208 diff --git a/objective_protocol.json b/objective_protocol.json new file mode 100644 index 0000000..35b3b44 --- /dev/null +++ b/objective_protocol.json @@ -0,0 +1,69 @@ +{ + "schema_version": 1, + "frozen_at_kst": "2026-07-12T21:40:00+09:00", + "base_model": "Qwen/Qwen3-8B", + "primary_triple": { + "helpfulness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "ultrafeedback-helpfulness", + "transform": "identity" + }, + "safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "conciseness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "helpsteer-verbosity", + "transform": "negate" + } + }, + "gate_thresholds": { + "raw_score_std_strictly_greater_than": 0.01, + "aggregate_prompt_pair_spearman_median_strictly_less_than": 0.0, + "aggregate_pairwise_top1_mismatch_at_least": 0.5 + }, + "gate_pool": "source-train Qwen3-8B response pool split into deterministic train/validation; sealed source-test excluded", + "training_seeds": [42, 43, 44], + "optimizer_steps": 900, + "effective_batch_size": 16, + "decode": { + "enable_thinking": false, + "temperature": 0.7, + "top_p": 0.9, + "max_new_tokens": 2048, + "decode_seed": 42 + }, + "methods": [ + "ronpo_full_expect", + "ronpo_k_only", + "dpo", + "ipo", + "simpo", + "kto", + "sppo_avg", + "inpo_avg", + "ht_mnpo_helpfulness", + "ht_mnpo_safety", + "ht_mnpo_conciseness" + ], + "selection_policy": "No sealed-test selection. One preregistered configuration per method; failures use only the declared stability retry recipe, at most three attempts.", + "stability_gate": { + "validation_prompts": 128, + "min_candidate_base_mean_word_ratio": 0.33, + "max_candidate_base_mean_word_ratio": 2.0, + "max_consecutive_identical_word_run": 20, + "think_tag_count": 0, + "empty_response_count": 0, + "retry_recipe": "attempt 2 halves learning rate, enables gradient checkpointing, and adds reference=0.02/SFT=0.002 anchoring to unanchored unified losses; attempt 3 quarters learning rate and uses reference=0.05/SFT=0.005; RONPO uses reference=0.05/SFT=0.005 from attempt 1" + }, + "primary_metric": "mean prompt-level minimum of per-prompt min-max-normalized helpfulness, safety, conciseness", + "bootstrap": { + "unit": "prompt", + "paired": true, + "resamples": 2000, + "interval": "percentile_95", + "seed": 42 + } +} diff --git a/pair_manifest.json b/pair_manifest.json new file mode 100644 index 0000000..be034b0 --- /dev/null +++ b/pair_manifest.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "objectives": [ + "helpfulness", + "safety", + "conciseness" + ], + "normalization": "per_prompt_minmax", + "adversary": { + "steps": 25, + "alpha": 1.0, + "kappa": 0.05, + "preference_scale": 8.0 + }, + "ronpo_estimator": "Rao-Blackwellized full sigma expectation with three common response pairs per prompt", + "counts": { + "train": { + "avg": 16746, + "ronpo_full_expect": 50340, + "ronpo_k_only": 50340, + "ht_mnpo_helpfulness": 16743, + "ht_mnpo_safety": 16764, + "ht_mnpo_conciseness": 16755 + }, + "validation": { + "avg": 1859, + "ronpo_full_expect": 5592, + "ronpo_k_only": 5592, + "ht_mnpo_helpfulness": 1864, + "ht_mnpo_safety": 1862, + "ht_mnpo_conciseness": 1858 + } + } +} diff --git a/provenance.json b/provenance.json new file mode 100644 index 0000000..5012b49 --- /dev/null +++ b/provenance.json @@ -0,0 +1,12 @@ +{ + "method": "ronpo_full_expect", + "seed": 43, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "bd99ef514a5f", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/bd99ef514a5f", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "stability_gate": "stability_gate.json" +} diff --git a/run_status.json b/run_status.json new file mode 100644 index 0000000..5654c1b --- /dev/null +++ b/run_status.json @@ -0,0 +1,13 @@ +{ + "method": "ronpo_full_expect", + "seed": 43, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "bd99ef514a5f", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/bd99ef514a5f", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "stability_gate": "stability_gate.json", + "status": "completed" +} diff --git a/split_manifest.json b/split_manifest.json new file mode 100644 index 0000000..4fb96bd --- /dev/null +++ b/split_manifest.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "split_rule": "sort source-train prompts by sha256('flagship-v1||'+prompt_id); first round(10%) validation", + "validation_fraction": 0.1, + "source_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/avg_oracle", + "base_model": "Qwen/Qwen3-8B", + "counts": { + "train": 16780, + "validation": 1864, + "sealed_test": 604 + }, + "prompt_id_sha256": { + "train": "fcf6f2ea5a0b7bcd41179e6f5288f4d850b2d04bdb729da91907bd18bf733a69", + "validation": "a87ba6f519192164daa487b109a6547dd8cbde46a27a373d6da201ca2d061b8f", + "sealed_test": "b398ced6af6cf73d731e1a5f32a0631d8060323d51eda1e375695566bf5f1867" + }, + "overlaps": { + "train_validation": 0, + "train_sealed_test": 0, + "validation_sealed_test": 0 + }, + "sealed_test_policy": "prompts only until S4a; no objective scores or model generations used in S0-S3" +} diff --git a/stability_gate.json b/stability_gate.json new file mode 100644 index 0000000..2df3381 --- /dev/null +++ b/stability_gate.json @@ -0,0 +1,38 @@ +{ + "status": "passed", + "passed": true, + "base": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 388.1484375, + "max_words": 1379, + "max_repeat_run": 4 + }, + "candidate": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 397.2734375, + "max_words": 1191, + "max_repeat_run": 4 + }, + "candidate_base_mean_word_ratio": 1.0235090473602642, + "thresholds": { + "min_length_ratio": 0.33, + "max_length_ratio": 2.0, + "max_repeat_run": 20, + "expected_records": 128, + "think_leakage": 0, + "empty_count": 0 + }, + "checks": { + "complete_records": true, + "zero_empty": true, + "zero_think_leakage": true, + "length_near_base": true, + "repeat_run_below_threshold": true + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ffd19d2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..cdae505 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.2860548271752086, + "total_flos": 0.0, + "train_loss": 0.48261189058423043, + "train_runtime": 8509.445, + "train_samples": 50340, + "train_samples_per_second": 1.692, + "train_steps_per_second": 0.106 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..a3bdf09 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,4903 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.2860548271752086, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "drift/chosen_abs": 0.19945812225341797, + "drift/rejected_abs": 0.08849310874938965, + "epoch": 0.0015891934843067143, + "grad_norm": 4.5625, + "learning_rate": 4.444444444444444e-09, + "logits/chosen": -1.5555963516235352, + "logits/rejected": -1.5219662189483643, + "logps/chosen": -0.3077252507209778, + "logps/rejected": -0.29369106888771057, + "loss": 0.131231427192688, + "loss/core": 0.12094147503376007, + "loss/preference_sft": 0.3084653615951538, + "loss/reference_anchor": 0.17495320737361908, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.9923360347747803, + "rewards/margins": 1.110528826713562, + "rewards/rejected": 0.8818074464797974, + "ronpo/logit": 0.11105290800333023, + "ronpo/residual": 0.1199965700507164, + "ronpo/residual_abs": 0.1629294753074646, + "ronpo/target": -0.008943679742515087, + "ronpo/target_abs": 0.07324962317943573, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 5 + }, + { + "drift/chosen_abs": 0.2019089162349701, + "drift/rejected_abs": 0.095692940056324, + "epoch": 0.0031783869686134287, + "grad_norm": 76.5, + "learning_rate": 1e-08, + "logits/chosen": -1.7638843059539795, + "logits/rejected": -1.8241863250732422, + "logps/chosen": -0.30898669362068176, + "logps/rejected": -0.2883630692958832, + "loss": 0.1183585375547409, + "loss/core": 0.1104976087808609, + "loss/preference_sft": 0.29389849305152893, + "loss/reference_anchor": 0.12782886624336243, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.0182459354400635, + "rewards/margins": 1.062389850616455, + "rewards/rejected": 0.9558561444282532, + "ronpo/logit": 0.10623898357152939, + "ronpo/residual": 0.11770063638687134, + "ronpo/residual_abs": 0.1581176221370697, + "ronpo/target": -0.011461647227406502, + "ronpo/target_abs": 0.07125519216060638, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 10 + }, + { + "drift/chosen_abs": 0.36563432216644287, + "drift/rejected_abs": 0.12579265236854553, + "epoch": 0.004767580452920143, + "grad_norm": 0.640625, + "learning_rate": 1.5555555555555554e-08, + "logits/chosen": -1.926892876625061, + "logits/rejected": -1.9229015111923218, + "logps/chosen": -0.27894189953804016, + "logps/rejected": -0.29260116815567017, + "loss": 0.8448435068130493, + "loss/core": 0.8088053464889526, + "loss/preference_sft": 0.2909303307533264, + "loss/reference_anchor": 0.6916698813438416, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.655860185623169, + "rewards/margins": 2.3997905254364014, + "rewards/rejected": 1.2560697793960571, + "ronpo/logit": 0.2399790734052658, + "ronpo/residual": 0.256378710269928, + "ronpo/residual_abs": 0.3004401624202728, + "ronpo/target": -0.016399653628468513, + "ronpo/target_abs": 0.0730489045381546, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 15 + }, + { + "drift/chosen_abs": 0.26957765221595764, + "drift/rejected_abs": 0.06649773567914963, + "epoch": 0.006356773937226857, + "grad_norm": 0.79296875, + "learning_rate": 2.111111111111111e-08, + "logits/chosen": -1.8648866415023804, + "logits/rejected": -1.967498779296875, + "logps/chosen": -0.30245983600616455, + "logps/rejected": -0.2743304967880249, + "loss": 0.7822330594062805, + "loss/core": 0.7592211365699768, + "loss/preference_sft": 0.27905479073524475, + "loss/reference_anchor": 0.43233203887939453, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.6956257820129395, + "rewards/margins": 2.0322680473327637, + "rewards/rejected": 0.6633577942848206, + "ronpo/logit": 0.20322680473327637, + "ronpo/residual": 0.2019917517900467, + "ronpo/residual_abs": 0.2696748375892639, + "ronpo/target": 0.0012350535253062844, + "ronpo/target_abs": 0.0822242870926857, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 20 + }, + { + "drift/chosen_abs": 0.29503053426742554, + "drift/rejected_abs": 0.09023113548755646, + "epoch": 0.007945967421533572, + "grad_norm": 1.828125, + "learning_rate": 2.6666666666666664e-08, + "logits/chosen": -1.8297017812728882, + "logits/rejected": -1.94186532497406, + "logps/chosen": -0.27752918004989624, + "logps/rejected": -0.29865169525146484, + "loss": 0.48796534538269043, + "loss/core": 0.47148722410202026, + "loss/preference_sft": 0.29656797647476196, + "loss/reference_anchor": 0.2999069392681122, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.9502077102661133, + "rewards/margins": 2.120311737060547, + "rewards/rejected": 0.8298957943916321, + "ronpo/logit": 0.21203120052814484, + "ronpo/residual": 0.21126720309257507, + "ronpo/residual_abs": 0.2526918649673462, + "ronpo/target": 0.0007639724644832313, + "ronpo/target_abs": 0.0715799331665039, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 25 + }, + { + "drift/chosen_abs": 0.26316240429878235, + "drift/rejected_abs": 0.1573808193206787, + "epoch": 0.009535160905840286, + "grad_norm": 84.0, + "learning_rate": 3.2222222222222224e-08, + "logits/chosen": -1.9381355047225952, + "logits/rejected": -1.8474152088165283, + "logps/chosen": -0.2904534339904785, + "logps/rejected": -0.3180440664291382, + "loss": 0.5810376405715942, + "loss/core": 0.5595085024833679, + "loss/preference_sft": 0.30132001638412476, + "loss/reference_anchor": 0.4004506468772888, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.6302835941314697, + "rewards/margins": 1.1446410417556763, + "rewards/rejected": 1.485642671585083, + "ronpo/logit": 0.11446411907672882, + "ronpo/residual": 0.12280688434839249, + "ronpo/residual_abs": 0.26302680373191833, + "ronpo/target": -0.008342771790921688, + "ronpo/target_abs": 0.07446413487195969, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 30 + }, + { + "drift/chosen_abs": 0.25724148750305176, + "drift/rejected_abs": 0.1849718689918518, + "epoch": 0.011124354390147001, + "grad_norm": 6.40625, + "learning_rate": 3.7777777777777774e-08, + "logits/chosen": -1.877733588218689, + "logits/rejected": -1.8011366128921509, + "logps/chosen": -0.2792748808860779, + "logps/rejected": -0.2892039120197296, + "loss": 0.25435659289360046, + "loss/core": 0.23435644805431366, + "loss/preference_sft": 0.28339701890945435, + "loss/reference_anchor": 0.37166398763656616, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.5721800327301025, + "rewards/margins": 0.7232538461685181, + "rewards/rejected": 1.8489259481430054, + "ronpo/logit": 0.07232538610696793, + "ronpo/residual": 0.07871496677398682, + "ronpo/residual_abs": 0.19634081423282623, + "ronpo/target": -0.0063895778730511665, + "ronpo/target_abs": 0.06378191709518433, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 35 + }, + { + "drift/chosen_abs": 0.2774297595024109, + "drift/rejected_abs": 0.10878102481365204, + "epoch": 0.012713547874453715, + "grad_norm": 3.296875, + "learning_rate": 4.333333333333333e-08, + "logits/chosen": -1.8935270309448242, + "logits/rejected": -1.898699164390564, + "logps/chosen": -0.25949639081954956, + "logps/rejected": -0.26641005277633667, + "loss": 0.20093131065368652, + "loss/core": 0.18912899494171143, + "loss/preference_sft": 0.2602488696575165, + "loss/reference_anchor": 0.2100217044353485, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.7733113765716553, + "rewards/margins": 1.7004846334457397, + "rewards/rejected": 1.0728265047073364, + "ronpo/logit": 0.17004846036434174, + "ronpo/residual": 0.1733754575252533, + "ronpo/residual_abs": 0.21049460768699646, + "ronpo/target": -0.003326986450701952, + "ronpo/target_abs": 0.07367634028196335, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 40 + }, + { + "drift/chosen_abs": 0.2615305185317993, + "drift/rejected_abs": 0.0843420997262001, + "epoch": 0.014302741358760428, + "grad_norm": 1.3515625, + "learning_rate": 4.888888888888889e-08, + "logits/chosen": -1.9090964794158936, + "logits/rejected": -2.0183889865875244, + "logps/chosen": -0.28550487756729126, + "logps/rejected": -0.26880985498428345, + "loss": 0.8131304979324341, + "loss/core": 0.7812657356262207, + "loss/preference_sft": 0.28103402256965637, + "loss/reference_anchor": 0.6091889142990112, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.6137125492095947, + "rewards/margins": 1.7702915668487549, + "rewards/rejected": 0.8434208631515503, + "ronpo/logit": 0.17702916264533997, + "ronpo/residual": 0.17782527208328247, + "ronpo/residual_abs": 0.23299217224121094, + "ronpo/target": -0.0007961191004142165, + "ronpo/target_abs": 0.07125305384397507, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 45 + }, + { + "drift/chosen_abs": 0.2804063558578491, + "drift/rejected_abs": 0.14204956591129303, + "epoch": 0.015891934843067144, + "grad_norm": 1.703125, + "learning_rate": 5.444444444444444e-08, + "logits/chosen": -1.752661943435669, + "logits/rejected": -1.7710239887237549, + "logps/chosen": -0.2862663269042969, + "logps/rejected": -0.2904655933380127, + "loss": 0.3519284725189209, + "loss/core": 0.31644538044929504, + "loss/preference_sft": 0.2913067936897278, + "loss/reference_anchor": 0.6805309653282166, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.80353045463562, + "rewards/margins": 1.38631272315979, + "rewards/rejected": 1.41721773147583, + "ronpo/logit": 0.13863126933574677, + "ronpo/residual": 0.14011706411838531, + "ronpo/residual_abs": 0.1854199469089508, + "ronpo/target": -0.0014857802307233214, + "ronpo/target_abs": 0.06841117143630981, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 50 + }, + { + "drift/chosen_abs": 0.298065185546875, + "drift/rejected_abs": 0.15730881690979004, + "epoch": 0.01748112832737386, + "grad_norm": 56.0, + "learning_rate": 6e-08, + "logits/chosen": -1.8553667068481445, + "logits/rejected": -1.7833325862884521, + "logps/chosen": -0.28672200441360474, + "logps/rejected": -0.28446197509765625, + "loss": 0.23183119297027588, + "loss/core": 0.21043534576892853, + "loss/preference_sft": 0.28606119751930237, + "loss/reference_anchor": 0.3993109464645386, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.98065185546875, + "rewards/margins": 1.4075638055801392, + "rewards/rejected": 1.57308828830719, + "ronpo/logit": 0.14075636863708496, + "ronpo/residual": 0.1200438141822815, + "ronpo/residual_abs": 0.1800813376903534, + "ronpo/target": 0.020712558180093765, + "ronpo/target_abs": 0.08266838639974594, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 55 + }, + { + "drift/chosen_abs": 0.18171152472496033, + "drift/rejected_abs": 0.06555067002773285, + "epoch": 0.01907032181168057, + "grad_norm": 81.5, + "learning_rate": 6.555555555555555e-08, + "logits/chosen": -1.7977211475372314, + "logits/rejected": -1.767542839050293, + "logps/chosen": -0.29876524209976196, + "logps/rejected": -0.3064151406288147, + "loss": 0.10823305696249008, + "loss/core": 0.10275810956954956, + "loss/preference_sft": 0.3003951907157898, + "loss/reference_anchor": 0.07945962250232697, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8157427310943604, + "rewards/margins": 1.168804407119751, + "rewards/rejected": 0.6469382047653198, + "ronpo/logit": 0.11688046157360077, + "ronpo/residual": 0.10604210197925568, + "ronpo/residual_abs": 0.1665903478860855, + "ronpo/target": 0.010838357731699944, + "ronpo/target_abs": 0.0728057250380516, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 60 + }, + { + "drift/chosen_abs": 0.2507414221763611, + "drift/rejected_abs": 0.07751794159412384, + "epoch": 0.020659515295987287, + "grad_norm": 19.625, + "learning_rate": 7.111111111111111e-08, + "logits/chosen": -1.930026650428772, + "logits/rejected": -1.948189377784729, + "logps/chosen": -0.28983187675476074, + "logps/rejected": -0.28243488073349, + "loss": 0.34065335988998413, + "loss/core": 0.32895928621292114, + "loss/preference_sft": 0.2844352424144745, + "loss/reference_anchor": 0.20543844997882843, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.5072410106658936, + "rewards/margins": 1.733149766921997, + "rewards/rejected": 0.7740914225578308, + "ronpo/logit": 0.17331495881080627, + "ronpo/residual": 0.15960010886192322, + "ronpo/residual_abs": 0.21261218190193176, + "ronpo/target": 0.013714875094592571, + "ronpo/target_abs": 0.07702931016683578, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 65 + }, + { + "drift/chosen_abs": 0.2543380856513977, + "drift/rejected_abs": 0.10286681354045868, + "epoch": 0.022248708780294002, + "grad_norm": 6.0, + "learning_rate": 7.666666666666666e-08, + "logits/chosen": -1.7886531352996826, + "logits/rejected": -1.9062068462371826, + "logps/chosen": -0.2821660041809082, + "logps/rejected": -0.2808881998062134, + "loss": 0.3984472155570984, + "loss/core": 0.38331031799316406, + "loss/preference_sft": 0.2853568494319916, + "loss/reference_anchor": 0.2742030918598175, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.5432658195495605, + "rewards/margins": 1.5179535150527954, + "rewards/rejected": 1.0253121852874756, + "ronpo/logit": 0.15179535746574402, + "ronpo/residual": 0.14931924641132355, + "ronpo/residual_abs": 0.20238187909126282, + "ronpo/target": 0.0024761161766946316, + "ronpo/target_abs": 0.08015619218349457, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 70 + }, + { + "drift/chosen_abs": 0.24210648238658905, + "drift/rejected_abs": 0.10418188571929932, + "epoch": 0.023837902264600714, + "grad_norm": 19.375, + "learning_rate": 8.222222222222221e-08, + "logits/chosen": -1.8596198558807373, + "logits/rejected": -1.8323400020599365, + "logps/chosen": -0.25651809573173523, + "logps/rejected": -0.2563713490962982, + "loss": 0.15341153740882874, + "loss/core": 0.14641332626342773, + "loss/preference_sft": 0.2539079785346985, + "loss/reference_anchor": 0.11457308381795883, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.4205851554870605, + "rewards/margins": 1.379667043685913, + "rewards/rejected": 1.0409181118011475, + "ronpo/logit": 0.13796670734882355, + "ronpo/residual": 0.14936383068561554, + "ronpo/residual_abs": 0.1982424557209015, + "ronpo/target": -0.01139712892472744, + "ronpo/target_abs": 0.07977505028247833, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 75 + }, + { + "drift/chosen_abs": 0.2583889365196228, + "drift/rejected_abs": 0.1300758719444275, + "epoch": 0.02542709574890743, + "grad_norm": 508.0, + "learning_rate": 8.777777777777778e-08, + "logits/chosen": -1.824342966079712, + "logits/rejected": -1.8747689723968506, + "logps/chosen": -0.2892422080039978, + "logps/rejected": -0.28287678956985474, + "loss": 0.6578446626663208, + "loss/core": 0.638018012046814, + "loss/preference_sft": 0.28955796360969543, + "loss/reference_anchor": 0.3675774931907654, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.583889961242676, + "rewards/margins": 1.3049442768096924, + "rewards/rejected": 1.2789456844329834, + "ronpo/logit": 0.13049443066120148, + "ronpo/residual": 0.12255732715129852, + "ronpo/residual_abs": 0.27486568689346313, + "ronpo/target": 0.007937087677419186, + "ronpo/target_abs": 0.08260422199964523, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 80 + }, + { + "drift/chosen_abs": 0.24885332584381104, + "drift/rejected_abs": 0.11546947062015533, + "epoch": 0.027016289233214145, + "grad_norm": 14.25, + "learning_rate": 9.333333333333334e-08, + "logits/chosen": -1.8859790563583374, + "logits/rejected": -1.8343837261199951, + "logps/chosen": -0.32009655237197876, + "logps/rejected": -0.31681352853775024, + "loss": 0.40035802125930786, + "loss/core": 0.38102349638938904, + "loss/preference_sft": 0.3166871666908264, + "loss/reference_anchor": 0.35502058267593384, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.4881718158721924, + "rewards/margins": 1.3343416452407837, + "rewards/rejected": 1.1538300514221191, + "ronpo/logit": 0.13343417644500732, + "ronpo/residual": 0.1329519897699356, + "ronpo/residual_abs": 0.18393748998641968, + "ronpo/target": 0.00048218583106063306, + "ronpo/target_abs": 0.07522345334291458, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 85 + }, + { + "drift/chosen_abs": 0.22070515155792236, + "drift/rejected_abs": 0.16914622485637665, + "epoch": 0.028605482717520857, + "grad_norm": 1.21875, + "learning_rate": 9.888888888888889e-08, + "logits/chosen": -1.731783151626587, + "logits/rejected": -1.822945237159729, + "logps/chosen": -0.2804669737815857, + "logps/rejected": -0.2766118347644806, + "loss": 0.7024124264717102, + "loss/core": 0.6772509813308716, + "loss/preference_sft": 0.2828295826911926, + "loss/reference_anchor": 0.4749454855918884, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.2057957649230957, + "rewards/margins": 0.515635073184967, + "rewards/rejected": 1.6901605129241943, + "ronpo/logit": 0.051563508808612823, + "ronpo/residual": 0.048208534717559814, + "ronpo/residual_abs": 0.24764589965343475, + "ronpo/target": 0.003354982938617468, + "ronpo/target_abs": 0.07700905948877335, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 90 + }, + { + "drift/chosen_abs": 0.34500211477279663, + "drift/rejected_abs": 0.1328619420528412, + "epoch": 0.030194676201827572, + "grad_norm": 21.0, + "learning_rate": 9.999398298646737e-08, + "logits/chosen": -1.738399863243103, + "logits/rejected": -1.7765659093856812, + "logps/chosen": -0.28980618715286255, + "logps/rejected": -0.31215035915374756, + "loss": 0.6481006741523743, + "loss/core": 0.6118947267532349, + "loss/preference_sft": 0.2964279055595398, + "loss/reference_anchor": 0.6944752335548401, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.4496827125549316, + "rewards/margins": 2.1238207817077637, + "rewards/rejected": 1.3258618116378784, + "ronpo/logit": 0.21238207817077637, + "ronpo/residual": 0.19465920329093933, + "ronpo/residual_abs": 0.24489042162895203, + "ronpo/target": 0.017722856253385544, + "ronpo/target_abs": 0.07983126491308212, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 95 + }, + { + "drift/chosen_abs": 0.26026713848114014, + "drift/rejected_abs": 0.09202776104211807, + "epoch": 0.03178386968613429, + "grad_norm": 2.0, + "learning_rate": 9.996954135095479e-08, + "logits/chosen": -1.9308345317840576, + "logits/rejected": -1.9035898447036743, + "logps/chosen": -0.2800833582878113, + "logps/rejected": -0.2895572781562805, + "loss": 0.4711598753929138, + "loss/core": 0.451214462518692, + "loss/preference_sft": 0.28193235397338867, + "loss/reference_anchor": 0.37071505188941956, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.6015172004699707, + "rewards/margins": 1.6833524703979492, + "rewards/rejected": 0.9181646108627319, + "ronpo/logit": 0.16833525896072388, + "ronpo/residual": 0.1610206663608551, + "ronpo/residual_abs": 0.2178463488817215, + "ronpo/target": 0.0073145912028849125, + "ronpo/target_abs": 0.08214585483074188, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 100 + }, + { + "drift/chosen_abs": 0.27864447236061096, + "drift/rejected_abs": 0.12608464062213898, + "epoch": 0.033373063170441, + "grad_norm": 9.75, + "learning_rate": 9.992630821454457e-08, + "logits/chosen": -2.042860984802246, + "logits/rejected": -1.9781955480575562, + "logps/chosen": -0.28923875093460083, + "logps/rejected": -0.2902441620826721, + "loss": 0.3296965956687927, + "loss/core": 0.3039691150188446, + "loss/preference_sft": 0.29211506247520447, + "loss/reference_anchor": 0.4853378236293793, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.786428928375244, + "rewards/margins": 1.5330383777618408, + "rewards/rejected": 1.2533905506134033, + "ronpo/logit": 0.1533038467168808, + "ronpo/residual": 0.14926132559776306, + "ronpo/residual_abs": 0.2045602798461914, + "ronpo/target": 0.004042509943246841, + "ronpo/target_abs": 0.07107819616794586, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 105 + }, + { + "drift/chosen_abs": 0.18290773034095764, + "drift/rejected_abs": 0.10879548639059067, + "epoch": 0.03496225665474772, + "grad_norm": 3.71875, + "learning_rate": 9.986429983545125e-08, + "logits/chosen": -1.8636986017227173, + "logits/rejected": -1.8569608926773071, + "logps/chosen": -0.31769293546676636, + "logps/rejected": -0.318257600069046, + "loss": 0.056669317185878754, + "loss/core": 0.04444245249032974, + "loss/preference_sft": 0.31768161058425903, + "loss/reference_anchor": 0.21276922523975372, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8289499282836914, + "rewards/margins": 0.741577684879303, + "rewards/rejected": 1.0873721837997437, + "ronpo/logit": 0.07415778189897537, + "ronpo/residual": 0.08273956924676895, + "ronpo/residual_abs": 0.13138622045516968, + "ronpo/target": -0.008581787347793579, + "ronpo/target_abs": 0.07794088870286942, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 110 + }, + { + "drift/chosen_abs": 0.27672436833381653, + "drift/rejected_abs": 0.2216198742389679, + "epoch": 0.03655145013905443, + "grad_norm": 2.65625, + "learning_rate": 9.978353953249021e-08, + "logits/chosen": -1.8198753595352173, + "logits/rejected": -1.8555314540863037, + "logps/chosen": -0.29371920228004456, + "logps/rejected": -0.28360024094581604, + "loss": 0.3811081051826477, + "loss/core": 0.3532530963420868, + "loss/preference_sft": 0.2940063774585724, + "loss/reference_anchor": 0.5276997089385986, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.7668750286102295, + "rewards/margins": 0.5547674894332886, + "rewards/rejected": 2.2121076583862305, + "ronpo/logit": 0.05547674745321274, + "ronpo/residual": 0.051227767020463943, + "ronpo/residual_abs": 0.20360875129699707, + "ronpo/target": 0.004248985089361668, + "ronpo/target_abs": 0.07594992220401764, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 115 + }, + { + "drift/chosen_abs": 0.3134176731109619, + "drift/rejected_abs": 0.17082719504833221, + "epoch": 0.03814064362336114, + "grad_norm": 40.75, + "learning_rate": 9.968405767630856e-08, + "logits/chosen": -1.703507423400879, + "logits/rejected": -1.6853641271591187, + "logps/chosen": -0.2665267586708069, + "logps/rejected": -0.2603232264518738, + "loss": 0.5027941465377808, + "loss/core": 0.470782607793808, + "loss/preference_sft": 0.26472344994544983, + "loss/reference_anchor": 0.6137583255767822, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.134176731109619, + "rewards/margins": 1.4267246723175049, + "rewards/rejected": 1.7074520587921143, + "ronpo/logit": 0.14267246425151825, + "ronpo/residual": 0.12057846784591675, + "ronpo/residual_abs": 0.21640296280384064, + "ronpo/target": 0.02209400199353695, + "ronpo/target_abs": 0.08593425899744034, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 120 + }, + { + "drift/chosen_abs": 0.2831875681877136, + "drift/rejected_abs": 0.1215093731880188, + "epoch": 0.03972983710766786, + "grad_norm": 7.09375, + "learning_rate": 9.956589167796391e-08, + "logits/chosen": -1.780076026916504, + "logits/rejected": -1.7246229648590088, + "logps/chosen": -0.29332637786865234, + "logps/rejected": -0.31054961681365967, + "loss": 0.4214431643486023, + "loss/core": 0.38739684224128723, + "loss/preference_sft": 0.30493617057800293, + "loss/reference_anchor": 0.6504331231117249, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.8314144611358643, + "rewards/margins": 1.6172332763671875, + "rewards/rejected": 1.2141809463500977, + "ronpo/logit": 0.161723330616951, + "ronpo/residual": 0.15208038687705994, + "ronpo/residual_abs": 0.20944802463054657, + "ronpo/target": 0.009642958641052246, + "ronpo/target_abs": 0.07323791086673737, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 125 + }, + { + "drift/chosen_abs": 0.35686972737312317, + "drift/rejected_abs": 0.12469740957021713, + "epoch": 0.04131903059197457, + "grad_norm": 5.59375, + "learning_rate": 9.942908597485558e-08, + "logits/chosen": -1.721999168395996, + "logits/rejected": -1.697299599647522, + "logps/chosen": -0.25378936529159546, + "logps/rejected": -0.2636905908584595, + "loss": 0.6324118971824646, + "loss/core": 0.606758713722229, + "loss/preference_sft": 0.2626436650753021, + "loss/reference_anchor": 0.4867998957633972, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.5683834552764893, + "rewards/margins": 2.3224399089813232, + "rewards/rejected": 1.245943307876587, + "ronpo/logit": 0.23224398493766785, + "ronpo/residual": 0.2395981252193451, + "ronpo/residual_abs": 0.2776317298412323, + "ronpo/target": -0.007354182191193104, + "ronpo/target_abs": 0.07926900684833527, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 130 + }, + { + "drift/chosen_abs": 0.34960705041885376, + "drift/rejected_abs": 0.13111598789691925, + "epoch": 0.04290822407628129, + "grad_norm": 5.9375, + "learning_rate": 9.927369201401357e-08, + "logits/chosen": -1.7188808917999268, + "logits/rejected": -1.7290832996368408, + "logps/chosen": -0.2801087200641632, + "logps/rejected": -0.2907566428184509, + "loss": 0.7031916379928589, + "loss/core": 0.6620404720306396, + "loss/preference_sft": 0.27922695875167847, + "loss/reference_anchor": 0.7951001524925232, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.4959259033203125, + "rewards/margins": 2.1855595111846924, + "rewards/rejected": 1.3103667497634888, + "ronpo/logit": 0.21855595707893372, + "ronpo/residual": 0.2089838683605194, + "ronpo/residual_abs": 0.273912638425827, + "ronpo/target": 0.009572053328156471, + "ronpo/target_abs": 0.09021146595478058, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 135 + }, + { + "drift/chosen_abs": 0.28669261932373047, + "drift/rejected_abs": 0.23457542061805725, + "epoch": 0.044497417560588004, + "grad_norm": 26.875, + "learning_rate": 9.909976823275142e-08, + "logits/chosen": -1.638810396194458, + "logits/rejected": -1.6302299499511719, + "logps/chosen": -0.31148478388786316, + "logps/rejected": -0.3387099504470825, + "loss": 0.41511544585227966, + "loss/core": 0.3812304139137268, + "loss/preference_sft": 0.3309209644794464, + "loss/reference_anchor": 0.6446089744567871, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.8666906356811523, + "rewards/margins": 0.520958662033081, + "rewards/rejected": 2.3457322120666504, + "ronpo/logit": 0.05209586024284363, + "ronpo/residual": 0.04535352438688278, + "ronpo/residual_abs": 0.25816768407821655, + "ronpo/target": 0.0067423381842672825, + "ronpo/target_abs": 0.08022363483905792, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 140 + }, + { + "drift/chosen_abs": 0.40737858414649963, + "drift/rejected_abs": 0.19810840487480164, + "epoch": 0.04608661104489471, + "grad_norm": 10.625, + "learning_rate": 9.890738003669028e-08, + "logits/chosen": -1.8872385025024414, + "logits/rejected": -1.8847707509994507, + "logps/chosen": -0.25984078645706177, + "logps/rejected": -0.25426381826400757, + "loss": 0.8179662823677063, + "loss/core": 0.7683526277542114, + "loss/preference_sft": 0.25788503885269165, + "loss/reference_anchor": 0.9664837121963501, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 4.073483467102051, + "rewards/margins": 2.0932326316833496, + "rewards/rejected": 1.9802510738372803, + "ronpo/logit": 0.20932325720787048, + "ronpo/residual": 0.1968086212873459, + "ronpo/residual_abs": 0.3058253526687622, + "ronpo/target": 0.012514638714492321, + "ronpo/target_abs": 0.0701758936047554, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 145 + }, + { + "drift/chosen_abs": 0.261055052280426, + "drift/rejected_abs": 0.11264361441135406, + "epoch": 0.04767580452920143, + "grad_norm": 2.046875, + "learning_rate": 9.86965997751626e-08, + "logits/chosen": -1.7386753559112549, + "logits/rejected": -1.7452685832977295, + "logps/chosen": -0.2891419529914856, + "logps/rejected": -0.2879791855812073, + "loss": 0.33339086174964905, + "loss/core": 0.30610546469688416, + "loss/preference_sft": 0.28957223892211914, + "loss/reference_anchor": 0.5167508125305176, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.6090314388275146, + "rewards/margins": 1.486400842666626, + "rewards/rejected": 1.1226304769515991, + "ronpo/logit": 0.14864009618759155, + "ronpo/residual": 0.14336484670639038, + "ronpo/residual_abs": 0.18963593244552612, + "ronpo/target": 0.005275248549878597, + "ronpo/target_abs": 0.06922867149114609, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 150 + }, + { + "drift/chosen_abs": 0.22688674926757812, + "drift/rejected_abs": 0.0957280695438385, + "epoch": 0.04926499801350814, + "grad_norm": 552.0, + "learning_rate": 9.846750671400445e-08, + "logits/chosen": -1.7219197750091553, + "logits/rejected": -1.7531629800796509, + "logps/chosen": -0.2790243923664093, + "logps/rejected": -0.29044419527053833, + "loss": 0.3570828139781952, + "loss/core": 0.3397815227508545, + "loss/preference_sft": 0.2791074812412262, + "loss/reference_anchor": 0.3181149959564209, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.2685012817382812, + "rewards/margins": 1.3128230571746826, + "rewards/rejected": 0.9556782841682434, + "ronpo/logit": 0.13128229975700378, + "ronpo/residual": 0.11850941181182861, + "ronpo/residual_abs": 0.17623624205589294, + "ronpo/target": 0.012772892601788044, + "ronpo/target_abs": 0.07684291899204254, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 155 + }, + { + "drift/chosen_abs": 0.3770247995853424, + "drift/rejected_abs": 0.15572576224803925, + "epoch": 0.05085419149781486, + "grad_norm": 51.25, + "learning_rate": 9.822018700574695e-08, + "logits/chosen": -1.5965344905853271, + "logits/rejected": -1.5869042873382568, + "logps/chosen": -0.29843515157699585, + "logps/rejected": -0.2949294447898865, + "loss": 0.4583340287208557, + "loss/core": 0.4307394027709961, + "loss/preference_sft": 0.30313509702682495, + "loss/reference_anchor": 0.5215789079666138, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.7689895629882812, + "rewards/margins": 2.212876558303833, + "rewards/rejected": 1.5561130046844482, + "ronpo/logit": 0.22128765285015106, + "ronpo/residual": 0.22782309353351593, + "ronpo/residual_abs": 0.266335129737854, + "ronpo/target": -0.0065354108810424805, + "ronpo/target_abs": 0.08122516423463821, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 160 + }, + { + "drift/chosen_abs": 0.2590554654598236, + "drift/rejected_abs": 0.10995769500732422, + "epoch": 0.052443384982121574, + "grad_norm": 46.25, + "learning_rate": 9.79547336572177e-08, + "logits/chosen": -1.6693989038467407, + "logits/rejected": -1.6664832830429077, + "logps/chosen": -0.3052573502063751, + "logps/rejected": -0.30682119727134705, + "loss": 0.22655808925628662, + "loss/core": 0.21496911346912384, + "loss/preference_sft": 0.30755677819252014, + "loss/reference_anchor": 0.2010241001844406, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.5874056816101074, + "rewards/margins": 1.5105947256088257, + "rewards/rejected": 1.0768110752105713, + "ronpo/logit": 0.15105947852134705, + "ronpo/residual": 0.1594732105731964, + "ronpo/residual_abs": 0.21000726521015167, + "ronpo/target": -0.008413726463913918, + "ronpo/target_abs": 0.08734320104122162, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 165 + }, + { + "drift/chosen_abs": 0.26380234956741333, + "drift/rejected_abs": 0.1027008444070816, + "epoch": 0.05403257846642829, + "grad_norm": 124.0, + "learning_rate": 9.767124649456482e-08, + "logits/chosen": -1.8073135614395142, + "logits/rejected": -1.8926242589950562, + "logps/chosen": -0.25851204991340637, + "logps/rejected": -0.25199228525161743, + "loss": 0.32083290815353394, + "loss/core": 0.30648404359817505, + "loss/preference_sft": 0.25841936469078064, + "loss/reference_anchor": 0.2611349821090698, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.6353073120117188, + "rewards/margins": 1.6092818975448608, + "rewards/rejected": 1.0260252952575684, + "ronpo/logit": 0.16092820465564728, + "ronpo/residual": 0.14506474137306213, + "ronpo/residual_abs": 0.2043898105621338, + "ronpo/target": 0.015863457694649696, + "ronpo/target_abs": 0.08150150626897812, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 170 + }, + { + "drift/chosen_abs": 0.30660122632980347, + "drift/rejected_abs": 0.16423708200454712, + "epoch": 0.055621771950735005, + "grad_norm": 5.59375, + "learning_rate": 9.736983212571646e-08, + "logits/chosen": -1.4690876007080078, + "logits/rejected": -1.5064828395843506, + "logps/chosen": -0.2989944815635681, + "logps/rejected": -0.2933068871498108, + "loss": 0.18233861029148102, + "loss/core": 0.15927359461784363, + "loss/preference_sft": 0.29980307817459106, + "loss/reference_anchor": 0.43132057785987854, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.0638973712921143, + "rewards/margins": 1.4231058359146118, + "rewards/rejected": 1.640791654586792, + "ronpo/logit": 0.14231057465076447, + "ronpo/residual": 0.14812180399894714, + "ronpo/residual_abs": 0.19247175753116608, + "ronpo/target": -0.00581121863797307, + "ronpo/target_abs": 0.0764647126197815, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 175 + }, + { + "drift/chosen_abs": 0.21774089336395264, + "drift/rejected_abs": 0.12308623641729355, + "epoch": 0.057210965435041714, + "grad_norm": 2.0625, + "learning_rate": 9.705060390028978e-08, + "logits/chosen": -1.8982244729995728, + "logits/rejected": -1.8753488063812256, + "logps/chosen": -0.2724835276603699, + "logps/rejected": -0.26976484060287476, + "loss": 0.2106136530637741, + "loss/core": 0.19985875487327576, + "loss/preference_sft": 0.2746678590774536, + "loss/reference_anchor": 0.18763145804405212, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.176549196243286, + "rewards/margins": 0.9472295045852661, + "rewards/rejected": 1.22931969165802, + "ronpo/logit": 0.09472296386957169, + "ronpo/residual": 0.09877835214138031, + "ronpo/residual_abs": 0.19839002192020416, + "ronpo/target": -0.004055389203131199, + "ronpo/target_abs": 0.07911963760852814, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 180 + }, + { + "drift/chosen_abs": 0.20750541985034943, + "drift/rejected_abs": 0.07880739867687225, + "epoch": 0.05880015891934843, + "grad_norm": 0.66015625, + "learning_rate": 9.671368186696487e-08, + "logits/chosen": -1.7366540431976318, + "logits/rejected": -1.6725021600723267, + "logps/chosen": -0.2818494439125061, + "logps/rejected": -0.29854708909988403, + "loss": 0.2648107707500458, + "loss/core": 0.25557440519332886, + "loss/preference_sft": 0.29618388414382935, + "loss/reference_anchor": 0.1551087200641632, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.072450876235962, + "rewards/margins": 1.4119017124176025, + "rewards/rejected": 0.6605492830276489, + "ronpo/logit": 0.14119017124176025, + "ronpo/residual": 0.13928617537021637, + "ronpo/residual_abs": 0.18467354774475098, + "ronpo/target": 0.0019040137995034456, + "ronpo/target_abs": 0.06882323324680328, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 185 + }, + { + "drift/chosen_abs": 0.3288821876049042, + "drift/rejected_abs": 0.08692674338817596, + "epoch": 0.060389352403655144, + "grad_norm": 1.0234375, + "learning_rate": 9.635919272833936e-08, + "logits/chosen": -1.8826234340667725, + "logits/rejected": -1.8909105062484741, + "logps/chosen": -0.28138142824172974, + "logps/rejected": -0.2874371409416199, + "loss": 0.9378984570503235, + "loss/core": 0.9062137603759766, + "loss/preference_sft": 0.28439387679100037, + "loss/reference_anchor": 0.6052535772323608, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.2882328033447266, + "rewards/margins": 2.4242477416992188, + "rewards/rejected": 0.8639850616455078, + "ronpo/logit": 0.24242480099201202, + "ronpo/residual": 0.24681024253368378, + "ronpo/residual_abs": 0.28807663917541504, + "ronpo/target": -0.004385476466268301, + "ronpo/target_abs": 0.06821522861719131, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 190 + }, + { + "drift/chosen_abs": 0.48508280515670776, + "drift/rejected_abs": 0.12983950972557068, + "epoch": 0.06197854588796186, + "grad_norm": 1.3203125, + "learning_rate": 9.598726979328077e-08, + "logits/chosen": -1.7950727939605713, + "logits/rejected": -1.7613496780395508, + "logps/chosen": -0.2707488238811493, + "logps/rejected": -0.2625698447227478, + "loss": 1.6498689651489258, + "loss/core": 1.5984039306640625, + "loss/preference_sft": 0.27121761441230774, + "loss/reference_anchor": 1.002181887626648, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 4.849929332733154, + "rewards/margins": 3.582291841506958, + "rewards/rejected": 1.267637848854065, + "ronpo/logit": 0.3582291901111603, + "ronpo/residual": 0.3614763617515564, + "ronpo/residual_abs": 0.3961934447288513, + "ronpo/target": -0.0032471846789121628, + "ronpo/target_abs": 0.06749024987220764, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 195 + }, + { + "drift/chosen_abs": 0.3213449716567993, + "drift/rejected_abs": 0.18218696117401123, + "epoch": 0.06356773937226858, + "grad_norm": 209.0, + "learning_rate": 9.559805292679444e-08, + "logits/chosen": -1.7774794101715088, + "logits/rejected": -1.779119849205017, + "logps/chosen": -0.28203001618385315, + "logps/rejected": -0.28085026144981384, + "loss": 0.3768664002418518, + "loss/core": 0.3414820730686188, + "loss/preference_sft": 0.2905542552471161, + "loss/reference_anchor": 0.6786319017410278, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.2131171226501465, + "rewards/margins": 1.3922719955444336, + "rewards/rejected": 1.8208452463150024, + "ronpo/logit": 0.13922719657421112, + "ronpo/residual": 0.14935089647769928, + "ronpo/residual_abs": 0.21834442019462585, + "ronpo/target": -0.010123697109520435, + "ronpo/target_abs": 0.0792742595076561, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 200 + }, + { + "drift/chosen_abs": 0.30102676153182983, + "drift/rejected_abs": 0.12649984657764435, + "epoch": 0.06515693285657528, + "grad_norm": 7.59375, + "learning_rate": 9.519168849742603e-08, + "logits/chosen": -1.7655588388442993, + "logits/rejected": -1.8515663146972656, + "logps/chosen": -0.28426095843315125, + "logps/rejected": -0.2876798212528229, + "loss": 0.44624027609825134, + "loss/core": 0.4282476007938385, + "loss/preference_sft": 0.28018221259117126, + "loss/reference_anchor": 0.33183637261390686, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.0090479850769043, + "rewards/margins": 1.7447162866592407, + "rewards/rejected": 1.2643316984176636, + "ronpo/logit": 0.1744716465473175, + "ronpo/residual": 0.17375977337360382, + "ronpo/residual_abs": 0.23645329475402832, + "ronpo/target": 0.0007118784124031663, + "ronpo/target_abs": 0.09003230184316635, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 205 + }, + { + "drift/chosen_abs": 0.13736221194267273, + "drift/rejected_abs": 0.07152147591114044, + "epoch": 0.066746126340882, + "grad_norm": 1.109375, + "learning_rate": 9.476832932221834e-08, + "logits/chosen": -1.9468104839324951, + "logits/rejected": -2.017134666442871, + "logps/chosen": -0.2806920111179352, + "logps/rejected": -0.28426095843315125, + "loss": 0.03696361929178238, + "loss/core": 0.033381301909685135, + "loss/preference_sft": 0.2832663953304291, + "loss/reference_anchor": 0.0433196984231472, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.3729757070541382, + "rewards/margins": 0.6626027822494507, + "rewards/rejected": 0.7103729248046875, + "ronpo/logit": 0.06626029312610626, + "ronpo/residual": 0.06426443159580231, + "ronpo/residual_abs": 0.12241494655609131, + "ronpo/target": 0.001995860366150737, + "ronpo/target_abs": 0.0830467939376831, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 210 + }, + { + "drift/chosen_abs": 0.23734159767627716, + "drift/rejected_abs": 0.12650033831596375, + "epoch": 0.06833531982518871, + "grad_norm": 2.3125, + "learning_rate": 9.432813460924306e-08, + "logits/chosen": -1.6526628732681274, + "logits/rejected": -1.7219336032867432, + "logps/chosen": -0.2975338399410248, + "logps/rejected": -0.30417123436927795, + "loss": 0.06939711421728134, + "loss/core": 0.057345591485500336, + "loss/preference_sft": 0.301188588142395, + "loss/reference_anchor": 0.21091172099113464, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.3707804679870605, + "rewards/margins": 1.1078386306762695, + "rewards/rejected": 1.262941837310791, + "ronpo/logit": 0.11078387498855591, + "ronpo/residual": 0.09446224570274353, + "ronpo/residual_abs": 0.14788493514060974, + "ronpo/target": 0.01632162369787693, + "ronpo/target_abs": 0.07581789791584015, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 215 + }, + { + "drift/chosen_abs": 0.199172705411911, + "drift/rejected_abs": 0.09683258831501007, + "epoch": 0.06992451330949544, + "grad_norm": 7.71875, + "learning_rate": 9.387126989772909e-08, + "logits/chosen": -1.8001289367675781, + "logits/rejected": -1.7968250513076782, + "logps/chosen": -0.27282461524009705, + "logps/rejected": -0.27572038769721985, + "loss": 0.20236146450042725, + "loss/core": 0.19408978521823883, + "loss/preference_sft": 0.27387744188308716, + "loss/reference_anchor": 0.13804566860198975, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9908561706542969, + "rewards/margins": 1.023787260055542, + "rewards/rejected": 0.9670687913894653, + "ronpo/logit": 0.10237874835729599, + "ronpo/residual": 0.10155370086431503, + "ronpo/residual_abs": 0.16425423324108124, + "ronpo/target": 0.0008250333485193551, + "ronpo/target_abs": 0.08938291668891907, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 220 + }, + { + "drift/chosen_abs": 0.16761744022369385, + "drift/rejected_abs": 0.09081226587295532, + "epoch": 0.07151370679380215, + "grad_norm": 3.21875, + "learning_rate": 9.339790699581003e-08, + "logits/chosen": -1.9558331966400146, + "logits/rejected": -1.9702494144439697, + "logps/chosen": -0.24233689904212952, + "logps/rejected": -0.24989719688892365, + "loss": 0.055875737220048904, + "loss/core": 0.050802744925022125, + "loss/preference_sft": 0.24640336632728577, + "loss/reference_anchor": 0.07681950181722641, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.673455834388733, + "rewards/margins": 0.7707704901695251, + "rewards/rejected": 0.9026853442192078, + "ronpo/logit": 0.07707705348730087, + "ronpo/residual": 0.07403306663036346, + "ronpo/residual_abs": 0.1344565600156784, + "ronpo/target": 0.003043986391276121, + "ronpo/target_abs": 0.08832766115665436, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 225 + }, + { + "drift/chosen_abs": 0.4040397107601166, + "drift/rejected_abs": 0.20574426651000977, + "epoch": 0.07310290027810885, + "grad_norm": 1.2265625, + "learning_rate": 9.290822391591417e-08, + "logits/chosen": -1.8052107095718384, + "logits/rejected": -1.7864511013031006, + "logps/chosen": -0.2899695038795471, + "logps/rejected": -0.30955272912979126, + "loss": 0.5364000797271729, + "loss/core": 0.4829653203487396, + "loss/preference_sft": 0.3120691180229187, + "loss/reference_anchor": 1.0374867916107178, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 4.037346839904785, + "rewards/margins": 2.1194443702697754, + "rewards/rejected": 1.9179027080535889, + "ronpo/logit": 0.21194446086883545, + "ronpo/residual": 0.20892982184886932, + "ronpo/residual_abs": 0.2797432839870453, + "ronpo/target": 0.0030146308708935976, + "ronpo/target_abs": 0.07361957430839539, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 230 + }, + { + "drift/chosen_abs": 0.35165148973464966, + "drift/rejected_abs": 0.18592815101146698, + "epoch": 0.07469209376241558, + "grad_norm": 1.4609375, + "learning_rate": 9.240240480782128e-08, + "logits/chosen": -1.792009711265564, + "logits/rejected": -1.8102452754974365, + "logps/chosen": -0.29723426699638367, + "logps/rejected": -0.28721755743026733, + "loss": 0.3889918923377991, + "loss/core": 0.3515408933162689, + "loss/preference_sft": 0.28965941071510315, + "loss/reference_anchor": 0.7200536131858826, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.5157008171081543, + "rewards/margins": 1.6589634418487549, + "rewards/rejected": 1.856737494468689, + "ronpo/logit": 0.16589637100696564, + "ronpo/residual": 0.158092200756073, + "ronpo/residual_abs": 0.2130337655544281, + "ronpo/target": 0.007804171647876501, + "ronpo/target_abs": 0.07572085410356522, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 235 + }, + { + "drift/chosen_abs": 0.23401710391044617, + "drift/rejected_abs": 0.11872808635234833, + "epoch": 0.07628128724672228, + "grad_norm": 4.40625, + "learning_rate": 9.188063988941146e-08, + "logits/chosen": -1.817143201828003, + "logits/rejected": -1.893754243850708, + "logps/chosen": -0.2975711226463318, + "logps/rejected": -0.2947074770927429, + "loss": 0.11131559312343597, + "loss/core": 0.09855598956346512, + "loss/preference_sft": 0.2999657392501831, + "loss/reference_anchor": 0.22519561648368835, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.339799642562866, + "rewards/margins": 1.156066656112671, + "rewards/rejected": 1.1837332248687744, + "ronpo/logit": 0.1156066507101059, + "ronpo/residual": 0.10619938373565674, + "ronpo/residual_abs": 0.15284030139446259, + "ronpo/target": 0.009407281875610352, + "ronpo/target_abs": 0.07423986494541168, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 240 + }, + { + "drift/chosen_abs": 0.3331848680973053, + "drift/rejected_abs": 0.11539693921804428, + "epoch": 0.07787048073102901, + "grad_norm": 234.0, + "learning_rate": 9.134312537513186e-08, + "logits/chosen": -1.8041841983795166, + "logits/rejected": -1.761364221572876, + "logps/chosen": -0.3000238239765167, + "logps/rejected": -0.30252355337142944, + "loss": 0.8759735822677612, + "loss/core": 0.8312212228775024, + "loss/preference_sft": 0.30587834119796753, + "loss/reference_anchor": 0.8644579648971558, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.3306591510772705, + "rewards/margins": 2.1822597980499268, + "rewards/rejected": 1.1483994722366333, + "ronpo/logit": 0.21822600066661835, + "ronpo/residual": 0.21011772751808167, + "ronpo/residual_abs": 0.2682834267616272, + "ronpo/target": 0.008108270354568958, + "ronpo/target_abs": 0.09494705498218536, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 245 + }, + { + "drift/chosen_abs": 0.35135117173194885, + "drift/rejected_abs": 0.15672887861728668, + "epoch": 0.07945967421533572, + "grad_norm": 352.0, + "learning_rate": 9.07900634022086e-08, + "logits/chosen": -1.749436378479004, + "logits/rejected": -1.7206599712371826, + "logps/chosen": -0.2767244577407837, + "logps/rejected": -0.2613469660282135, + "loss": 0.6066776514053345, + "loss/core": 0.5774033069610596, + "loss/preference_sft": 0.27615392208099365, + "loss/reference_anchor": 0.5578705072402954, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.513512134552002, + "rewards/margins": 1.9805538654327393, + "rewards/rejected": 1.5329577922821045, + "ronpo/logit": 0.19805540144443512, + "ronpo/residual": 0.19628280401229858, + "ronpo/residual_abs": 0.2545327842235565, + "ronpo/target": 0.0017725772922858596, + "ronpo/target_abs": 0.07362736016511917, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 250 + }, + { + "drift/chosen_abs": 0.09521497786045074, + "drift/rejected_abs": 0.04314413666725159, + "epoch": 0.08104886769964244, + "grad_norm": 1.703125, + "learning_rate": 9.02216619546311e-08, + "logits/chosen": -2.127336263656616, + "logits/rejected": -2.1303391456604004, + "logps/chosen": -0.2789131999015808, + "logps/rejected": -0.2843419015407562, + "loss": 0.02066163904964924, + "loss/core": 0.018253901973366737, + "loss/preference_sft": 0.28216642141342163, + "loss/reference_anchor": 0.019938113167881966, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.9484003186225891, + "rewards/margins": 0.5196825861930847, + "rewards/rejected": 0.428717702627182, + "ronpo/logit": 0.05196825787425041, + "ronpo/residual": 0.041401177644729614, + "ronpo/residual_abs": 0.0943097323179245, + "ronpo/target": 0.01056708488613367, + "ronpo/target_abs": 0.0819949060678482, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 255 + }, + { + "drift/chosen_abs": 0.37315139174461365, + "drift/rejected_abs": 0.16794715821743011, + "epoch": 0.08263806118394915, + "grad_norm": 11.5625, + "learning_rate": 8.963813478493788e-08, + "logits/chosen": -1.6915229558944702, + "logits/rejected": -1.7035410404205322, + "logps/chosen": -0.2909337878227234, + "logps/rejected": -0.2909685969352722, + "loss": 0.5750402808189392, + "loss/core": 0.5253722071647644, + "loss/preference_sft": 0.2933681607246399, + "loss/reference_anchor": 0.9640249013900757, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.7303051948547363, + "rewards/margins": 2.0620627403259277, + "rewards/rejected": 1.66824209690094, + "ronpo/logit": 0.2062062919139862, + "ronpo/residual": 0.1919330209493637, + "ronpo/residual_abs": 0.2425989806652069, + "ronpo/target": 0.014273243956267834, + "ronpo/target_abs": 0.06451570987701416, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 260 + }, + { + "drift/chosen_abs": 0.37269216775894165, + "drift/rejected_abs": 0.09161754697561264, + "epoch": 0.08422725466825585, + "grad_norm": 21.125, + "learning_rate": 8.903970133383297e-08, + "logits/chosen": -1.644796371459961, + "logits/rejected": -1.5450862646102905, + "logps/chosen": -0.280166894197464, + "logps/rejected": -0.2835793197154999, + "loss": 0.9720993041992188, + "loss/core": 0.943614661693573, + "loss/preference_sft": 0.28308966755867004, + "loss/reference_anchor": 0.5413841009140015, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.7253494262695312, + "rewards/margins": 2.8579189777374268, + "rewards/rejected": 0.8674303293228149, + "ronpo/logit": 0.28579190373420715, + "ronpo/residual": 0.2972434461116791, + "ronpo/residual_abs": 0.3468155562877655, + "ronpo/target": -0.011451545171439648, + "ronpo/target_abs": 0.08932532370090485, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 265 + }, + { + "drift/chosen_abs": 0.1571657955646515, + "drift/rejected_abs": 0.07838908582925797, + "epoch": 0.08581644815256258, + "grad_norm": 10.375, + "learning_rate": 8.842658664766316e-08, + "logits/chosen": -1.829146146774292, + "logits/rejected": -1.841604471206665, + "logps/chosen": -0.2962016463279724, + "logps/rejected": -0.29258397221565247, + "loss": 0.0605291910469532, + "loss/core": 0.054931849241256714, + "loss/preference_sft": 0.2943229079246521, + "loss/reference_anchor": 0.08251471817493439, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.5701178312301636, + "rewards/margins": 0.78829425573349, + "rewards/rejected": 0.7818235158920288, + "ronpo/logit": 0.07882943749427795, + "ronpo/residual": 0.09931907802820206, + "ronpo/residual_abs": 0.13743585348129272, + "ronpo/target": -0.0204896442592144, + "ronpo/target_abs": 0.08406566083431244, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 270 + }, + { + "drift/chosen_abs": 0.34716716408729553, + "drift/rejected_abs": 0.14563432335853577, + "epoch": 0.08740564163686929, + "grad_norm": 6.90625, + "learning_rate": 8.779902129378738e-08, + "logits/chosen": -1.930598497390747, + "logits/rejected": -1.9004106521606445, + "logps/chosen": -0.29537269473075867, + "logps/rejected": -0.28550130128860474, + "loss": 0.7920290231704712, + "loss/core": 0.7438088655471802, + "loss/preference_sft": 0.2878211736679077, + "loss/reference_anchor": 0.9356187582015991, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.469741106033325, + "rewards/margins": 2.0159237384796143, + "rewards/rejected": 1.4538179636001587, + "ronpo/logit": 0.201592355966568, + "ronpo/residual": 0.20089873671531677, + "ronpo/residual_abs": 0.24797526001930237, + "ronpo/target": 0.0006936040008440614, + "ronpo/target_abs": 0.0743836909532547, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 275 + }, + { + "drift/chosen_abs": 0.3208922743797302, + "drift/rejected_abs": 0.11808327585458755, + "epoch": 0.08899483512117601, + "grad_norm": 21.375, + "learning_rate": 8.715724127386971e-08, + "logits/chosen": -1.851441740989685, + "logits/rejected": -1.7920417785644531, + "logps/chosen": -0.291858971118927, + "logps/rejected": -0.27598926424980164, + "loss": 0.43859606981277466, + "loss/core": 0.4198855459690094, + "loss/preference_sft": 0.27710169553756714, + "loss/reference_anchor": 0.3464996814727783, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.2073960304260254, + "rewards/margins": 2.0731959342956543, + "rewards/rejected": 1.1342003345489502, + "ronpo/logit": 0.20731958746910095, + "ronpo/residual": 0.21950101852416992, + "ronpo/residual_abs": 0.2542910575866699, + "ronpo/target": -0.012181434780359268, + "ronpo/target_abs": 0.07466322928667068, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 280 + }, + { + "drift/chosen_abs": 0.422764390707016, + "drift/rejected_abs": 0.2523038387298584, + "epoch": 0.09058402860548272, + "grad_norm": 1.7890625, + "learning_rate": 8.650148793512872e-08, + "logits/chosen": -1.6715549230575562, + "logits/rejected": -1.730615258216858, + "logps/chosen": -0.27390456199645996, + "logps/rejected": -0.27246493101119995, + "loss": 0.37580451369285583, + "loss/core": 0.3173265755176544, + "loss/preference_sft": 0.27324944734573364, + "loss/reference_anchor": 1.1422338485717773, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 4.225140571594238, + "rewards/margins": 1.8235841989517212, + "rewards/rejected": 2.4015564918518066, + "ronpo/logit": 0.18235842883586884, + "ronpo/residual": 0.18450120091438293, + "ronpo/residual_abs": 0.24420492351055145, + "ronpo/target": -0.0021427671890705824, + "ronpo/target_abs": 0.07870659232139587, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 285 + }, + { + "drift/chosen_abs": 0.339019238948822, + "drift/rejected_abs": 0.1414244920015335, + "epoch": 0.09217322208978942, + "grad_norm": 5.90625, + "learning_rate": 8.58320078795768e-08, + "logits/chosen": -1.9007600545883179, + "logits/rejected": -1.8737331628799438, + "logps/chosen": -0.2887403964996338, + "logps/rejected": -0.3162103295326233, + "loss": 0.5720577239990234, + "loss/core": 0.5303006768226624, + "loss/preference_sft": 0.3076707720756531, + "loss/reference_anchor": 0.8043742179870605, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 3.390192747116089, + "rewards/margins": 2.156050682067871, + "rewards/rejected": 1.2341418266296387, + "ronpo/logit": 0.21560506522655487, + "ronpo/residual": 0.21163058280944824, + "ronpo/residual_abs": 0.25786668062210083, + "ronpo/target": 0.003974485211074352, + "ronpo/target_abs": 0.08457241952419281, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 290 + }, + { + "drift/chosen_abs": 0.38282856345176697, + "drift/rejected_abs": 0.10466931760311127, + "epoch": 0.09376241557409615, + "grad_norm": 11.125, + "learning_rate": 8.514905287128308e-08, + "logits/chosen": -1.9592187404632568, + "logits/rejected": -1.918391466140747, + "logps/chosen": -0.29651257395744324, + "logps/rejected": -0.29091230034828186, + "loss": 1.480013132095337, + "loss/core": 1.4311615228652954, + "loss/preference_sft": 0.29933270812034607, + "loss/reference_anchor": 0.9471020698547363, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.8279099464416504, + "rewards/margins": 2.7820181846618652, + "rewards/rejected": 1.0458917617797852, + "ronpo/logit": 0.27820178866386414, + "ronpo/residual": 0.2564352750778198, + "ronpo/residual_abs": 0.34638240933418274, + "ronpo/target": 0.0217665396630764, + "ronpo/target_abs": 0.0853576809167862, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 295 + }, + { + "drift/chosen_abs": 0.23551969230175018, + "drift/rejected_abs": 0.11991935968399048, + "epoch": 0.09535160905840286, + "grad_norm": 4.71875, + "learning_rate": 8.44528797416954e-08, + "logits/chosen": -1.7564899921417236, + "logits/rejected": -1.8135544061660767, + "logps/chosen": -0.2883635461330414, + "logps/rejected": -0.2851993441581726, + "loss": 0.13555917143821716, + "loss/core": 0.12408057600259781, + "loss/preference_sft": 0.28471457958221436, + "loss/reference_anchor": 0.20110058784484863, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.353811740875244, + "rewards/margins": 1.1569559574127197, + "rewards/rejected": 1.1968556642532349, + "ronpo/logit": 0.11569561064243317, + "ronpo/residual": 0.11366703361272812, + "ronpo/residual_abs": 0.1557668149471283, + "ronpo/target": 0.0020285695791244507, + "ronpo/target_abs": 0.06852469593286514, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 300 + }, + { + "drift/chosen_abs": 0.3189460337162018, + "drift/rejected_abs": 0.1352815330028534, + "epoch": 0.09694080254270958, + "grad_norm": 31.5, + "learning_rate": 8.374375029305638e-08, + "logits/chosen": -1.7944324016571045, + "logits/rejected": -1.7328087091445923, + "logps/chosen": -0.27897724509239197, + "logps/rejected": -0.2867986559867859, + "loss": 0.629349946975708, + "loss/core": 0.5993045568466187, + "loss/preference_sft": 0.2911568880081177, + "loss/reference_anchor": 0.5717930197715759, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.189460277557373, + "rewards/margins": 1.8377918004989624, + "rewards/rejected": 1.3516688346862793, + "ronpo/logit": 0.18377919495105743, + "ronpo/residual": 0.16819463670253754, + "ronpo/residual_abs": 0.22840599715709686, + "ronpo/target": 0.015584555454552174, + "ronpo/target_abs": 0.08353091031312943, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 305 + }, + { + "drift/chosen_abs": 0.22331972420215607, + "drift/rejected_abs": 0.10017452389001846, + "epoch": 0.09852999602701629, + "grad_norm": 0.9765625, + "learning_rate": 8.302193119995037e-08, + "logits/chosen": -1.83052659034729, + "logits/rejected": -1.8760980367660522, + "logps/chosen": -0.3000250458717346, + "logps/rejected": -0.302760511636734, + "loss": 0.1551820933818817, + "loss/core": 0.1419788897037506, + "loss/preference_sft": 0.3033260107040405, + "loss/reference_anchor": 0.23373158276081085, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.2327847480773926, + "rewards/margins": 1.2316186428070068, + "rewards/rejected": 1.0011662244796753, + "ronpo/logit": 0.12316185235977173, + "ronpo/residual": 0.12490953505039215, + "ronpo/residual_abs": 0.15779349207878113, + "ronpo/target": -0.0017476793145760894, + "ronpo/target_abs": 0.0755293220281601, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 310 + }, + { + "drift/chosen_abs": 0.32748138904571533, + "drift/rejected_abs": 0.12331084161996841, + "epoch": 0.10011918951132301, + "grad_norm": 330.0, + "learning_rate": 8.22876939090181e-08, + "logits/chosen": -1.8843028545379639, + "logits/rejected": -1.8397438526153564, + "logps/chosen": -0.2662331163883209, + "logps/rejected": -0.2638329565525055, + "loss": 0.6274744272232056, + "loss/core": 0.6013913750648499, + "loss/preference_sft": 0.2718098759651184, + "loss/reference_anchor": 0.4944801330566406, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.274233341217041, + "rewards/margins": 2.0433292388916016, + "rewards/rejected": 1.2309037446975708, + "ronpo/logit": 0.20433291792869568, + "ronpo/residual": 0.2113933265209198, + "ronpo/residual_abs": 0.2690233290195465, + "ronpo/target": -0.007060416974127293, + "ronpo/target_abs": 0.08037460595369339, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 315 + }, + { + "drift/chosen_abs": 0.31982502341270447, + "drift/rejected_abs": 0.1502150595188141, + "epoch": 0.10170838299562972, + "grad_norm": 9.0, + "learning_rate": 8.154131453687656e-08, + "logits/chosen": -1.6950571537017822, + "logits/rejected": -1.6967027187347412, + "logps/chosen": -0.29967036843299866, + "logps/rejected": -0.2906130254268646, + "loss": 0.22242435812950134, + "loss/core": 0.20424409210681915, + "loss/preference_sft": 0.29998618364334106, + "loss/reference_anchor": 0.3336065411567688, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.1972603797912598, + "rewards/margins": 1.7026844024658203, + "rewards/rejected": 1.4945757389068604, + "ronpo/logit": 0.1702684462070465, + "ronpo/residual": 0.16520681977272034, + "ronpo/residual_abs": 0.21789643168449402, + "ronpo/target": 0.005061624571681023, + "ronpo/target_abs": 0.07731436938047409, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 320 + }, + { + "drift/chosen_abs": 0.32239073514938354, + "drift/rejected_abs": 0.14487099647521973, + "epoch": 0.10329757647993643, + "grad_norm": 2.515625, + "learning_rate": 8.07830737662829e-08, + "logits/chosen": -1.7711395025253296, + "logits/rejected": -1.7771180868148804, + "logps/chosen": -0.27507781982421875, + "logps/rejected": -0.26983487606048584, + "loss": 0.46221646666526794, + "loss/core": 0.43381762504577637, + "loss/preference_sft": 0.274006187915802, + "loss/reference_anchor": 0.5405768156051636, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.223078489303589, + "rewards/margins": 1.7749700546264648, + "rewards/rejected": 1.4481085538864136, + "ronpo/logit": 0.177496999502182, + "ronpo/residual": 0.17691360414028168, + "ronpo/residual_abs": 0.22765477001667023, + "ronpo/target": 0.0005833914619870484, + "ronpo/target_abs": 0.07749475538730621, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 325 + }, + { + "drift/chosen_abs": 0.2092551738023758, + "drift/rejected_abs": 0.05679240822792053, + "epoch": 0.10488676996424315, + "grad_norm": 13.8125, + "learning_rate": 8.001325674058123e-08, + "logits/chosen": -1.9151840209960938, + "logits/rejected": -1.9076945781707764, + "logps/chosen": -0.3010541498661041, + "logps/rejected": -0.298443078994751, + "loss": 0.29586261510849, + "loss/core": 0.28491804003715515, + "loss/preference_sft": 0.3068598210811615, + "loss/reference_anchor": 0.18820573389530182, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.0916950702667236, + "rewards/margins": 1.525160312652588, + "rewards/rejected": 0.5665348172187805, + "ronpo/logit": 0.15251603722572327, + "ronpo/residual": 0.1487692892551422, + "ronpo/residual_abs": 0.19916026294231415, + "ronpo/target": 0.003746759844943881, + "ronpo/target_abs": 0.07705475389957428, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 330 + }, + { + "drift/chosen_abs": 0.2681005001068115, + "drift/rejected_abs": 0.13281159102916718, + "epoch": 0.10647596344854986, + "grad_norm": 4.9375, + "learning_rate": 7.923215295647166e-08, + "logits/chosen": -1.8540802001953125, + "logits/rejected": -1.8404872417449951, + "logps/chosen": -0.28104037046432495, + "logps/rejected": -0.28956103324890137, + "loss": 0.3578251004219055, + "loss/core": 0.33799657225608826, + "loss/preference_sft": 0.28577661514282227, + "loss/reference_anchor": 0.36799150705337524, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.680506944656372, + "rewards/margins": 1.3567620515823364, + "rewards/rejected": 1.323744773864746, + "ronpo/logit": 0.13567620515823364, + "ronpo/residual": 0.1216505616903305, + "ronpo/residual_abs": 0.2040475606918335, + "ronpo/target": 0.014025667682290077, + "ronpo/target_abs": 0.05643507093191147, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 335 + }, + { + "drift/chosen_abs": 0.20863750576972961, + "drift/rejected_abs": 0.23069891333580017, + "epoch": 0.10806515693285658, + "grad_norm": 416.0, + "learning_rate": 7.844005615514259e-08, + "logits/chosen": -1.861107587814331, + "logits/rejected": -1.8314059972763062, + "logps/chosen": -0.26891106367111206, + "logps/rejected": -0.2669699788093567, + "loss": 1.0711219310760498, + "loss/core": 1.038293480873108, + "loss/preference_sft": 0.2746712565422058, + "loss/reference_anchor": 0.6291025280952454, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.0860025882720947, + "rewards/margins": -0.2168906182050705, + "rewards/rejected": 2.3028931617736816, + "ronpo/logit": -0.02168908901512623, + "ronpo/residual": -0.020689811557531357, + "ronpo/residual_abs": 0.2519962787628174, + "ronpo/target": -0.00099927035626024, + "ronpo/target_abs": 0.07252907007932663, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 340 + }, + { + "drift/chosen_abs": 0.3522050678730011, + "drift/rejected_abs": 0.1813865453004837, + "epoch": 0.10965435041716329, + "grad_norm": 812.0, + "learning_rate": 7.763726421180662e-08, + "logits/chosen": -1.597299337387085, + "logits/rejected": -1.547257423400879, + "logps/chosen": -0.28551051020622253, + "logps/rejected": -0.3178768754005432, + "loss": 0.6678337454795837, + "loss/core": 0.6344470381736755, + "loss/preference_sft": 0.289370059967041, + "loss/reference_anchor": 0.6387966871261597, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.520669937133789, + "rewards/margins": 1.727455496788025, + "rewards/rejected": 1.7932144403457642, + "ronpo/logit": 0.17274554073810577, + "ronpo/residual": 0.14491984248161316, + "ronpo/residual_abs": 0.2273443192243576, + "ronpo/target": 0.027825692668557167, + "ronpo/target_abs": 0.07946719974279404, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 345 + }, + { + "drift/chosen_abs": 0.3626919686794281, + "drift/rejected_abs": 0.11279211193323135, + "epoch": 0.11124354390147001, + "grad_norm": 288.0, + "learning_rate": 7.682407902368189e-08, + "logits/chosen": -1.664306640625, + "logits/rejected": -1.6896919012069702, + "logps/chosen": -0.2869907021522522, + "logps/rejected": -0.2831929326057434, + "loss": 1.0064929723739624, + "loss/core": 0.9767097234725952, + "loss/preference_sft": 0.284060537815094, + "loss/reference_anchor": 0.5672638416290283, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.626801013946533, + "rewards/margins": 2.499552011489868, + "rewards/rejected": 1.127249002456665, + "ronpo/logit": 0.2499551773071289, + "ronpo/residual": 0.2459290474653244, + "ronpo/residual_abs": 0.3209651708602905, + "ronpo/target": 0.004026124719530344, + "ronpo/target_abs": 0.07355201244354248, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 350 + }, + { + "drift/chosen_abs": 0.26780807971954346, + "drift/rejected_abs": 0.08546146005392075, + "epoch": 0.11283273738577672, + "grad_norm": 1.890625, + "learning_rate": 7.600080639646076e-08, + "logits/chosen": -1.7517549991607666, + "logits/rejected": -1.759873390197754, + "logps/chosen": -0.3038514256477356, + "logps/rejected": -0.2689318358898163, + "loss": 0.7741256952285767, + "loss/core": 0.749835193157196, + "loss/preference_sft": 0.30647626519203186, + "loss/reference_anchor": 0.45516258478164673, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.6770687103271484, + "rewards/margins": 1.8246443271636963, + "rewards/rejected": 0.8524240255355835, + "ronpo/logit": 0.18246445059776306, + "ronpo/residual": 0.15374180674552917, + "ronpo/residual_abs": 0.23904690146446228, + "ronpo/target": 0.02872266247868538, + "ronpo/target_abs": 0.0866011381149292, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 355 + }, + { + "drift/chosen_abs": 0.16474132239818573, + "drift/rejected_abs": 0.12225530296564102, + "epoch": 0.11442193087008343, + "grad_norm": 106.0, + "learning_rate": 7.516775592930899e-08, + "logits/chosen": -1.9887319803237915, + "logits/rejected": -1.9524673223495483, + "logps/chosen": -0.27265065908432007, + "logps/rejected": -0.2855183184146881, + "loss": 0.09760759770870209, + "loss/core": 0.08684544265270233, + "loss/preference_sft": 0.2860459089279175, + "loss/reference_anchor": 0.18663868308067322, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.647254228591919, + "rewards/margins": 0.4279909133911133, + "rewards/rejected": 1.2192633152008057, + "ronpo/logit": 0.04279909282922745, + "ronpo/residual": 0.05301178619265556, + "ronpo/residual_abs": 0.15181580185890198, + "ronpo/target": -0.010212697088718414, + "ronpo/target_abs": 0.08592395484447479, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 360 + }, + { + "drift/chosen_abs": 0.3150689899921417, + "drift/rejected_abs": 0.11829020828008652, + "epoch": 0.11601112435439015, + "grad_norm": 114.0, + "learning_rate": 7.4325240898438e-08, + "logits/chosen": -1.7951710224151611, + "logits/rejected": -1.8196980953216553, + "logps/chosen": -0.3030449450016022, + "logps/rejected": -0.3088149428367615, + "loss": 0.6980522871017456, + "loss/core": 0.6711905598640442, + "loss/preference_sft": 0.29767075181007385, + "loss/reference_anchor": 0.5074678659439087, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.149803876876831, + "rewards/margins": 1.9678528308868408, + "rewards/rejected": 1.1819511651992798, + "ronpo/logit": 0.19678528606891632, + "ronpo/residual": 0.2056920975446701, + "ronpo/residual_abs": 0.2885230481624603, + "ronpo/target": -0.00890684686601162, + "ronpo/target_abs": 0.08591794967651367, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 365 + }, + { + "drift/chosen_abs": 0.2871250510215759, + "drift/rejected_abs": 0.19906646013259888, + "epoch": 0.11760031783869686, + "grad_norm": 6.375, + "learning_rate": 7.347357813929453e-08, + "logits/chosen": -1.6759065389633179, + "logits/rejected": -1.616633415222168, + "logps/chosen": -0.2482568770647049, + "logps/rejected": -0.2896507978439331, + "loss": 0.6905705332756042, + "loss/core": 0.6696237325668335, + "loss/preference_sft": 0.28166383504867554, + "loss/reference_anchor": 0.39076724648475647, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.871251106262207, + "rewards/margins": 0.9647668600082397, + "rewards/rejected": 1.9064840078353882, + "ronpo/logit": 0.09647669643163681, + "ronpo/residual": 0.09969191998243332, + "ronpo/residual_abs": 0.2865365743637085, + "ronpo/target": -0.003215217497199774, + "ronpo/target_abs": 0.08571871370077133, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 370 + }, + { + "drift/chosen_abs": 0.5225389003753662, + "drift/rejected_abs": 0.21518293023109436, + "epoch": 0.11918951132300358, + "grad_norm": 33.0, + "learning_rate": 7.261308792741188e-08, + "logits/chosen": -1.641985297203064, + "logits/rejected": -1.6296402215957642, + "logps/chosen": -0.3135181665420532, + "logps/rejected": -0.31274524331092834, + "loss": 1.3733961582183838, + "loss/core": 1.2828857898712158, + "loss/preference_sft": 0.314228355884552, + "loss/reference_anchor": 1.7787834405899048, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 5.22520112991333, + "rewards/margins": 3.074826717376709, + "rewards/rejected": 2.150374412536621, + "ronpo/logit": 0.30748268961906433, + "ronpo/residual": 0.3128319978713989, + "ronpo/residual_abs": 0.36880940198898315, + "ronpo/target": -0.005349296145141125, + "ronpo/target_abs": 0.07712045311927795, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 375 + }, + { + "drift/chosen_abs": 0.35133180022239685, + "drift/rejected_abs": 0.23328383266925812, + "epoch": 0.12077870480731029, + "grad_norm": 16.875, + "learning_rate": 7.174409385796725e-08, + "logits/chosen": -1.8136361837387085, + "logits/rejected": -1.7627443075180054, + "logps/chosen": -0.26291412115097046, + "logps/rejected": -0.26428166031837463, + "loss": 0.9764726758003235, + "loss/core": 0.9314364194869995, + "loss/preference_sft": 0.2695896327495575, + "loss/reference_anchor": 0.8737646341323853, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 3.5123963356018066, + "rewards/margins": 1.181367039680481, + "rewards/rejected": 2.3310296535491943, + "ronpo/logit": 0.11813666671514511, + "ronpo/residual": 0.12999659776687622, + "ronpo/residual_abs": 0.31305280327796936, + "ronpo/target": -0.011859911493957043, + "ronpo/target_abs": 0.0706958919763565, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 380 + }, + { + "drift/chosen_abs": 0.36728110909461975, + "drift/rejected_abs": 0.14616873860359192, + "epoch": 0.12236789829161701, + "grad_norm": 7.125, + "learning_rate": 7.08669227240909e-08, + "logits/chosen": -1.8517367839813232, + "logits/rejected": -1.8667278289794922, + "logps/chosen": -0.28371182084083557, + "logps/rejected": -0.30156999826431274, + "loss": 0.6238721609115601, + "loss/core": 0.5842751860618591, + "loss/preference_sft": 0.2967758774757385, + "loss/reference_anchor": 0.7622602581977844, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.6717803478240967, + "rewards/margins": 2.2115306854248047, + "rewards/rejected": 1.4602497816085815, + "ronpo/logit": 0.22115305066108704, + "ronpo/residual": 0.2303275614976883, + "ronpo/residual_abs": 0.2650774121284485, + "ronpo/target": -0.009174478240311146, + "ronpo/target_abs": 0.06511794030666351, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 385 + }, + { + "drift/chosen_abs": 0.22626559436321259, + "drift/rejected_abs": 0.08026406168937683, + "epoch": 0.12395709177592372, + "grad_norm": 0.546875, + "learning_rate": 6.99819043939726e-08, + "logits/chosen": -1.9849250316619873, + "logits/rejected": -1.9792630672454834, + "logps/chosen": -0.2899254560470581, + "logps/rejected": -0.2959898114204407, + "loss": 0.16416332125663757, + "loss/core": 0.15644721686840057, + "loss/preference_sft": 0.2958739101886749, + "loss/reference_anchor": 0.12473498284816742, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.262193202972412, + "rewards/margins": 1.4670231342315674, + "rewards/rejected": 0.7951699495315552, + "ronpo/logit": 0.1467023342847824, + "ronpo/residual": 0.15477152168750763, + "ronpo/residual_abs": 0.1925136297941208, + "ronpo/target": -0.00806919950991869, + "ronpo/target_abs": 0.07871512323617935, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 390 + }, + { + "drift/chosen_abs": 0.36781710386276245, + "drift/rejected_abs": 0.17324690520763397, + "epoch": 0.12554628526023043, + "grad_norm": 44.25, + "learning_rate": 6.908937168681174e-08, + "logits/chosen": -1.6767114400863647, + "logits/rejected": -1.7050321102142334, + "logps/chosen": -0.26027780771255493, + "logps/rejected": -0.25346577167510986, + "loss": 0.45207443833351135, + "loss/core": 0.40612488985061646, + "loss/preference_sft": 0.25639814138412476, + "loss/reference_anchor": 0.893352210521698, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.676452159881592, + "rewards/margins": 1.9461982250213623, + "rewards/rejected": 1.73025381565094, + "ronpo/logit": 0.19461986422538757, + "ronpo/residual": 0.19026371836662292, + "ronpo/residual_abs": 0.24404847621917725, + "ronpo/target": 0.0043561337515711784, + "ronpo/target_abs": 0.056930769234895706, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 395 + }, + { + "drift/chosen_abs": 0.3046533465385437, + "drift/rejected_abs": 0.11980541050434113, + "epoch": 0.12713547874453715, + "grad_norm": 92.5, + "learning_rate": 6.818966024765757e-08, + "logits/chosen": -1.7593889236450195, + "logits/rejected": -1.7481002807617188, + "logps/chosen": -0.3030829429626465, + "logps/rejected": -0.29993414878845215, + "loss": 0.5101957321166992, + "loss/core": 0.48600858449935913, + "loss/preference_sft": 0.29708653688430786, + "loss/reference_anchor": 0.4540334641933441, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.0456247329711914, + "rewards/margins": 1.8476349115371704, + "rewards/rejected": 1.1979897022247314, + "ronpo/logit": 0.18476352095603943, + "ronpo/residual": 0.19976118206977844, + "ronpo/residual_abs": 0.23570862412452698, + "ronpo/target": -0.014997666701674461, + "ronpo/target_abs": 0.07206665724515915, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 400 + }, + { + "drift/chosen_abs": 0.2052384316921234, + "drift/rejected_abs": 0.11188337951898575, + "epoch": 0.12872467222884387, + "grad_norm": 21.75, + "learning_rate": 6.728310842118683e-08, + "logits/chosen": -2.14799427986145, + "logits/rejected": -2.0824573040008545, + "logps/chosen": -0.286243736743927, + "logps/rejected": -0.27005332708358765, + "loss": 0.06200665980577469, + "loss/core": 0.05436018109321594, + "loss/preference_sft": 0.27878865599632263, + "loss/reference_anchor": 0.1250506341457367, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.052384614944458, + "rewards/margins": 0.9343948364257812, + "rewards/rejected": 1.1179895401000977, + "ronpo/logit": 0.0934394896030426, + "ronpo/residual": 0.10573802143335342, + "ronpo/residual_abs": 0.15494385361671448, + "ronpo/target": -0.012298518791794777, + "ronpo/target_abs": 0.09242352098226547, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 405 + }, + { + "drift/chosen_abs": 0.35525864362716675, + "drift/rejected_abs": 0.17476989328861237, + "epoch": 0.13031386571315057, + "grad_norm": 18.125, + "learning_rate": 6.637005712446621e-08, + "logits/chosen": -1.7883316278457642, + "logits/rejected": -1.7869304418563843, + "logps/chosen": -0.25297173857688904, + "logps/rejected": -0.2491494119167328, + "loss": 0.45122259855270386, + "loss/core": 0.42534032464027405, + "loss/preference_sft": 0.25509124994277954, + "loss/reference_anchor": 0.4921363890171051, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.551823377609253, + "rewards/margins": 1.8160899877548218, + "rewards/rejected": 1.7357333898544312, + "ronpo/logit": 0.18160900473594666, + "ronpo/residual": 0.1837209165096283, + "ronpo/residual_abs": 0.25326791405677795, + "ronpo/target": -0.002111931797116995, + "ronpo/target_abs": 0.06821735203266144, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 410 + }, + { + "drift/chosen_abs": 0.25623852014541626, + "drift/rejected_abs": 0.1311134397983551, + "epoch": 0.1319030591974573, + "grad_norm": 45.5, + "learning_rate": 6.545084971874738e-08, + "logits/chosen": -1.7289890050888062, + "logits/rejected": -1.7614386081695557, + "logps/chosen": -0.2858942747116089, + "logps/rejected": -0.2846941351890564, + "loss": 0.18270006775856018, + "loss/core": 0.1690942347049713, + "loss/preference_sft": 0.2852400839328766, + "loss/reference_anchor": 0.24359269440174103, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.5602011680603027, + "rewards/margins": 1.2526819705963135, + "rewards/rejected": 1.307519555091858, + "ronpo/logit": 0.12526817619800568, + "ronpo/residual": 0.1053306832909584, + "ronpo/residual_abs": 0.18119731545448303, + "ronpo/target": 0.01993749849498272, + "ronpo/target_abs": 0.07559109479188919, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 415 + }, + { + "drift/chosen_abs": 0.1505512297153473, + "drift/rejected_abs": 0.08253350853919983, + "epoch": 0.133492252681764, + "grad_norm": 12.125, + "learning_rate": 6.452583188034274e-08, + "logits/chosen": -1.7700755596160889, + "logits/rejected": -1.791124701499939, + "logps/chosen": -0.2845797538757324, + "logps/rejected": -0.2848334014415741, + "loss": 0.03450245410203934, + "loss/core": 0.030069872736930847, + "loss/preference_sft": 0.2898898720741272, + "loss/reference_anchor": 0.05966266989707947, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 1.5050227642059326, + "rewards/margins": 0.6809515953063965, + "rewards/rejected": 0.8240710496902466, + "ronpo/logit": 0.06809516996145248, + "ronpo/residual": 0.06789997965097427, + "ronpo/residual_abs": 0.11400087177753448, + "ronpo/target": 0.0001951886370079592, + "ronpo/target_abs": 0.07576659321784973, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 420 + }, + { + "drift/chosen_abs": 0.2125011682510376, + "drift/rejected_abs": 0.09349431842565536, + "epoch": 0.1350814461660707, + "grad_norm": 2.640625, + "learning_rate": 6.35953514706309e-08, + "logits/chosen": -1.9663327932357788, + "logits/rejected": -1.9798023700714111, + "logps/chosen": -0.3189515769481659, + "logps/rejected": -0.3072357177734375, + "loss": 0.19334067404270172, + "loss/core": 0.1795506328344345, + "loss/preference_sft": 0.3102140724658966, + "loss/reference_anchor": 0.24477925896644592, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.124784231185913, + "rewards/margins": 1.1916717290878296, + "rewards/rejected": 0.933112621307373, + "ronpo/logit": 0.11916716396808624, + "ronpo/residual": 0.0928797796368599, + "ronpo/residual_abs": 0.16781887412071228, + "ronpo/target": 0.026287391781806946, + "ronpo/target_abs": 0.08866839855909348, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 425 + }, + { + "drift/chosen_abs": 0.3195541203022003, + "drift/rejected_abs": 0.15836499631404877, + "epoch": 0.13667063965037743, + "grad_norm": 2.296875, + "learning_rate": 6.26597584052401e-08, + "logits/chosen": -1.5612452030181885, + "logits/rejected": -1.5029213428497314, + "logps/chosen": -0.285647451877594, + "logps/rejected": -0.29933539032936096, + "loss": 0.45964616537094116, + "loss/core": 0.4238843321800232, + "loss/preference_sft": 0.29544249176979065, + "loss/reference_anchor": 0.6856938004493713, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.19382905960083, + "rewards/margins": 1.6618378162384033, + "rewards/rejected": 1.5319918394088745, + "ronpo/logit": 0.166183739900589, + "ronpo/residual": 0.16860680282115936, + "ronpo/residual_abs": 0.22841337323188782, + "ronpo/target": -0.0024230708368122578, + "ronpo/target_abs": 0.08164582401514053, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 430 + }, + { + "drift/chosen_abs": 0.5011860132217407, + "drift/rejected_abs": 0.2851537764072418, + "epoch": 0.13825983313468415, + "grad_norm": 1.859375, + "learning_rate": 6.171940452245923e-08, + "logits/chosen": -1.9892600774765015, + "logits/rejected": -1.9781463146209717, + "logps/chosen": -0.2973264157772064, + "logps/rejected": -0.28821566700935364, + "loss": 0.9089182019233704, + "loss/core": 0.8096088171005249, + "loss/preference_sft": 0.29011955857276917, + "loss/reference_anchor": 1.9571746587753296, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 5.010129451751709, + "rewards/margins": 2.1612067222595215, + "rewards/rejected": 2.8489227294921875, + "ronpo/logit": 0.2161206752061844, + "ronpo/residual": 0.20811843872070312, + "ronpo/residual_abs": 0.3104029595851898, + "ronpo/target": 0.008002226240932941, + "ronpo/target_abs": 0.0776105746626854, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 435 + }, + { + "drift/chosen_abs": 0.34103599190711975, + "drift/rejected_abs": 0.1386711746454239, + "epoch": 0.13984902661899087, + "grad_norm": 480.0, + "learning_rate": 6.0774643450926e-08, + "logits/chosen": -1.8233058452606201, + "logits/rejected": -1.8174107074737549, + "logps/chosen": -0.278222918510437, + "logps/rejected": -0.28023761510849, + "loss": 0.6035203337669373, + "loss/core": 0.5771490335464478, + "loss/preference_sft": 0.28037887811660767, + "loss/reference_anchor": 0.49938860535621643, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.4100043773651123, + "rewards/margins": 2.0244264602661133, + "rewards/rejected": 1.3855777978897095, + "ronpo/logit": 0.20244267582893372, + "ronpo/residual": 0.2062930315732956, + "ronpo/residual_abs": 0.2548952102661133, + "ronpo/target": -0.003850378794595599, + "ronpo/target_abs": 0.07118948549032211, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 440 + }, + { + "drift/chosen_abs": 0.15881577134132385, + "drift/rejected_abs": 0.07861863076686859, + "epoch": 0.14143822010329757, + "grad_norm": 0.486328125, + "learning_rate": 5.982583047664151e-08, + "logits/chosen": -1.8153941631317139, + "logits/rejected": -1.7571932077407837, + "logps/chosen": -0.29266512393951416, + "logps/rejected": -0.2830495536327362, + "loss": 0.06146319955587387, + "loss/core": 0.056963998824357986, + "loss/preference_sft": 0.2907244563102722, + "loss/reference_anchor": 0.06091168522834778, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.5873725414276123, + "rewards/margins": 0.804255485534668, + "rewards/rejected": 0.7831171154975891, + "ronpo/logit": 0.08042554557323456, + "ronpo/residual": 0.06820248067378998, + "ronpo/residual_abs": 0.13618895411491394, + "ronpo/target": 0.01222306489944458, + "ronpo/target_abs": 0.0686340257525444, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 445 + }, + { + "drift/chosen_abs": 0.27281489968299866, + "drift/rejected_abs": 0.15323896706104279, + "epoch": 0.1430274135876043, + "grad_norm": 5.6875, + "learning_rate": 5.8873322409361756e-08, + "logits/chosen": -1.9215360879898071, + "logits/rejected": -1.7960364818572998, + "logps/chosen": -0.2579748034477234, + "logps/rejected": -0.2565074563026428, + "loss": 0.11640655994415283, + "loss/core": 0.10340286791324615, + "loss/preference_sft": 0.2626219391822815, + "loss/reference_anchor": 0.2338119000196457, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.728148937225342, + "rewards/margins": 1.1970012187957764, + "rewards/rejected": 1.5311477184295654, + "ronpo/logit": 0.1197001114487648, + "ronpo/residual": 0.1095530167222023, + "ronpo/residual_abs": 0.1741444170475006, + "ronpo/target": 0.010147105902433395, + "ronpo/target_abs": 0.08469612896442413, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 450 + }, + { + "drift/chosen_abs": 0.4300481677055359, + "drift/rejected_abs": 0.09889129549264908, + "epoch": 0.144616607071911, + "grad_norm": 0.52734375, + "learning_rate": 5.7917477448416144e-08, + "logits/chosen": -1.7946687936782837, + "logits/rejected": -1.744091272354126, + "logps/chosen": -0.28207752108573914, + "logps/rejected": -0.28602561354637146, + "loss": 1.9361851215362549, + "loss/core": 1.8742101192474365, + "loss/preference_sft": 0.2911958396434784, + "loss/reference_anchor": 1.2103819847106934, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.30048131942749, + "rewards/margins": 3.311803102493286, + "rewards/rejected": 0.9886780977249146, + "ronpo/logit": 0.3311803340911865, + "ronpo/residual": 0.3193929195404053, + "ronpo/residual_abs": 0.3614575266838074, + "ronpo/target": 0.011787394993007183, + "ronpo/target_abs": 0.06393725425004959, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 455 + }, + { + "drift/chosen_abs": 0.2632841169834137, + "drift/rejected_abs": 0.06613259017467499, + "epoch": 0.1462058005562177, + "grad_norm": 26.625, + "learning_rate": 5.695865504800327e-08, + "logits/chosen": -1.7865066528320312, + "logits/rejected": -1.81333327293396, + "logps/chosen": -0.3158157467842102, + "logps/rejected": -0.3114406168460846, + "loss": 0.8839119076728821, + "loss/core": 0.8581841588020325, + "loss/preference_sft": 0.31136512756347656, + "loss/reference_anchor": 0.48341670632362366, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.6306302547454834, + "rewards/margins": 1.9706141948699951, + "rewards/rejected": 0.6600159406661987, + "ronpo/logit": 0.1970614641904831, + "ronpo/residual": 0.20879793167114258, + "ronpo/residual_abs": 0.2626114785671234, + "ronpo/target": -0.011736475862562656, + "ronpo/target_abs": 0.08205690234899521, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 460 + }, + { + "drift/chosen_abs": 0.18584024906158447, + "drift/rejected_abs": 0.1095375269651413, + "epoch": 0.14779499404052443, + "grad_norm": 3.09375, + "learning_rate": 5.5997215782014984e-08, + "logits/chosen": -1.861520767211914, + "logits/rejected": -1.8145278692245483, + "logps/chosen": -0.3011169731616974, + "logps/rejected": -0.30149930715560913, + "loss": 0.086545430123806, + "loss/core": 0.07661891728639603, + "loss/preference_sft": 0.3033149540424347, + "loss/reference_anchor": 0.1681988537311554, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8580764532089233, + "rewards/margins": 0.7628976106643677, + "rewards/rejected": 1.0951789617538452, + "ronpo/logit": 0.07628976553678513, + "ronpo/residual": 0.07566403597593307, + "ronpo/residual_abs": 0.14903251826763153, + "ronpo/target": 0.0006257258355617523, + "ronpo/target_abs": 0.08669845014810562, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 465 + }, + { + "drift/chosen_abs": 0.2513282001018524, + "drift/rejected_abs": 0.14881765842437744, + "epoch": 0.14938418752483115, + "grad_norm": 1.421875, + "learning_rate": 5.503352120843923e-08, + "logits/chosen": -1.9602091312408447, + "logits/rejected": -1.9950196743011475, + "logps/chosen": -0.32254284620285034, + "logps/rejected": -0.3045181632041931, + "loss": 0.2901255190372467, + "loss/core": 0.25516438484191895, + "loss/preference_sft": 0.31007856130599976, + "loss/reference_anchor": 0.6682136654853821, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.511671304702759, + "rewards/margins": 1.0251829624176025, + "rewards/rejected": 1.4864884614944458, + "ronpo/logit": 0.10251828283071518, + "ronpo/residual": 0.10790930688381195, + "ronpo/residual_abs": 0.15721961855888367, + "ronpo/target": -0.0053910380229353905, + "ronpo/target_abs": 0.06892824172973633, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 470 + }, + { + "drift/chosen_abs": 0.2127540111541748, + "drift/rejected_abs": 0.11244404315948486, + "epoch": 0.15097338100913787, + "grad_norm": 9.4375, + "learning_rate": 5.406793373339291e-08, + "logits/chosen": -1.941626787185669, + "logits/rejected": -1.9039710760116577, + "logps/chosen": -0.271634042263031, + "logps/rejected": -0.25962454080581665, + "loss": 0.07151190936565399, + "loss/core": 0.06459919363260269, + "loss/preference_sft": 0.267228901386261, + "loss/reference_anchor": 0.11153163760900497, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.12636137008667, + "rewards/margins": 1.0033315420150757, + "rewards/rejected": 1.1230299472808838, + "ronpo/logit": 0.10033313184976578, + "ronpo/residual": 0.10426007211208344, + "ronpo/residual_abs": 0.15038594603538513, + "ronpo/target": -0.003926927223801613, + "ronpo/target_abs": 0.06808660924434662, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 475 + }, + { + "drift/chosen_abs": 0.14452792704105377, + "drift/rejected_abs": 0.05248863250017166, + "epoch": 0.15256257449344457, + "grad_norm": 3.234375, + "learning_rate": 5.3100816474835766e-08, + "logits/chosen": -1.8934004306793213, + "logits/rejected": -1.9814293384552002, + "logps/chosen": -0.316048800945282, + "logps/rejected": -0.3190820813179016, + "loss": 0.05717045068740845, + "loss/core": 0.05321327596902847, + "loss/preference_sft": 0.3178454637527466, + "loss/reference_anchor": 0.04735885187983513, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 1.4452793598175049, + "rewards/margins": 0.9231868982315063, + "rewards/rejected": 0.5220924019813538, + "ronpo/logit": 0.09231869131326675, + "ronpo/residual": 0.07715944945812225, + "ronpo/residual_abs": 0.14198298752307892, + "ronpo/target": 0.015159247443079948, + "ronpo/target_abs": 0.09165415912866592, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 480 + }, + { + "drift/chosen_abs": 0.2202528715133667, + "drift/rejected_abs": 0.1520129144191742, + "epoch": 0.1541517679777513, + "grad_norm": 43.5, + "learning_rate": 5.2132533126016534e-08, + "logits/chosen": -1.76580810546875, + "logits/rejected": -1.7142798900604248, + "logps/chosen": -0.28421875834465027, + "logps/rejected": -0.2967900335788727, + "loss": 0.08789822459220886, + "loss/core": 0.06753416359424591, + "loss/preference_sft": 0.30561891198158264, + "loss/reference_anchor": 0.376719206571579, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.201965808868408, + "rewards/margins": 0.6839362978935242, + "rewards/rejected": 1.5180293321609497, + "ronpo/logit": 0.06839364767074585, + "ronpo/residual": 0.0788511410355568, + "ronpo/residual_abs": 0.14319899678230286, + "ronpo/target": -0.01045750267803669, + "ronpo/target_abs": 0.07381303608417511, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 485 + }, + { + "drift/chosen_abs": 0.3343612253665924, + "drift/rejected_abs": 0.13808700442314148, + "epoch": 0.15574096146205801, + "grad_norm": 24.25, + "learning_rate": 5.116344781870281e-08, + "logits/chosen": -1.6695568561553955, + "logits/rejected": -1.6447023153305054, + "logps/chosen": -0.24842624366283417, + "logps/rejected": -0.24762670695781708, + "loss": 0.2929847836494446, + "loss/core": 0.2731058895587921, + "loss/preference_sft": 0.2550978660583496, + "loss/reference_anchor": 0.37206801772117615, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.3418400287628174, + "rewards/margins": 1.9618446826934814, + "rewards/rejected": 1.3799952268600464, + "ronpo/logit": 0.19618447124958038, + "ronpo/residual": 0.1800888329744339, + "ronpo/residual_abs": 0.23794646561145782, + "ronpo/target": 0.016095636412501335, + "ronpo/target_abs": 0.07803904265165329, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 490 + }, + { + "drift/chosen_abs": 0.08378258347511292, + "drift/rejected_abs": 0.03460133820772171, + "epoch": 0.1573301549463647, + "grad_norm": 21.625, + "learning_rate": 5.0193924986246017e-08, + "logits/chosen": -1.8778587579727173, + "logits/rejected": -1.8371680974960327, + "logps/chosen": -0.30237627029418945, + "logps/rejected": -0.3027265667915344, + "loss": 0.032751940190792084, + "loss/core": 0.03037204220890999, + "loss/preference_sft": 0.30147770047187805, + "loss/reference_anchor": 0.017450321465730667, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.8357017636299133, + "rewards/margins": 0.496019184589386, + "rewards/rejected": 0.3396826386451721, + "ronpo/logit": 0.04960191622376442, + "ronpo/residual": 0.06800566613674164, + "ronpo/residual_abs": 0.10949333012104034, + "ronpo/target": -0.01840374991297722, + "ronpo/target_abs": 0.07336892932653427, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 495 + }, + { + "drift/chosen_abs": 0.3098334074020386, + "drift/rejected_abs": 0.20627370476722717, + "epoch": 0.15891934843067143, + "grad_norm": 1.2578125, + "learning_rate": 4.9224329226532834e-08, + "logits/chosen": -1.7887645959854126, + "logits/rejected": -1.8265126943588257, + "logps/chosen": -0.30098414421081543, + "logps/rejected": -0.28809601068496704, + "loss": 0.17970222234725952, + "loss/core": 0.1251877248287201, + "loss/preference_sft": 0.29191064834594727, + "loss/reference_anchor": 1.0610988140106201, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.0972111225128174, + "rewards/margins": 1.035546898841858, + "rewards/rejected": 2.06166410446167, + "ronpo/logit": 0.10355468839406967, + "ronpo/residual": 0.11979351192712784, + "ronpo/residual_abs": 0.17769846320152283, + "ronpo/target": -0.01623881608247757, + "ronpo/target_abs": 0.08119514584541321, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 500 + }, + { + "drift/chosen_abs": 0.4369746744632721, + "drift/rejected_abs": 0.14313063025474548, + "epoch": 0.16050854191497815, + "grad_norm": 0.73046875, + "learning_rate": 4.8255025164874966e-08, + "logits/chosen": -1.9329090118408203, + "logits/rejected": -1.8906736373901367, + "logps/chosen": -0.2954661250114441, + "logps/rejected": -0.2762017846107483, + "loss": 1.6327224969863892, + "loss/core": 1.5709314346313477, + "loss/preference_sft": 0.2684187591075897, + "loss/reference_anchor": 1.2089756727218628, + "rewards/accuracies": 0.875, + "rewards/chosen": 4.366913795471191, + "rewards/margins": 2.9390738010406494, + "rewards/rejected": 1.427840232849121, + "ronpo/logit": 0.29390740394592285, + "ronpo/residual": 0.29641279578208923, + "ronpo/residual_abs": 0.33722934126853943, + "ronpo/target": -0.0025054272264242172, + "ronpo/target_abs": 0.06659742444753647, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 505 + }, + { + "drift/chosen_abs": 0.19818530976772308, + "drift/rejected_abs": 0.09616676717996597, + "epoch": 0.16209773539928488, + "grad_norm": 1.2421875, + "learning_rate": 4.7286377316888315e-08, + "logits/chosen": -1.8848209381103516, + "logits/rejected": -1.9005283117294312, + "logps/chosen": -0.2789894938468933, + "logps/rejected": -0.29110604524612427, + "loss": 0.12926748394966125, + "loss/core": 0.1197998970746994, + "loss/preference_sft": 0.27562230825424194, + "loss/reference_anchor": 0.16178929805755615, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.980821967124939, + "rewards/margins": 1.0209791660308838, + "rewards/rejected": 0.9598425626754761, + "ronpo/logit": 0.10209791362285614, + "ronpo/residual": 0.10504169762134552, + "ronpo/residual_abs": 0.1421099603176117, + "ronpo/target": -0.002943767700344324, + "ronpo/target_abs": 0.0699705258011818, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 510 + }, + { + "drift/chosen_abs": 0.33460456132888794, + "drift/rejected_abs": 0.1129327192902565, + "epoch": 0.16368692888359157, + "grad_norm": 6.21875, + "learning_rate": 4.631874995141376e-08, + "logits/chosen": -1.8654136657714844, + "logits/rejected": -1.799949288368225, + "logps/chosen": -0.2905055284500122, + "logps/rejected": -0.328683078289032, + "loss": 0.9674808382987976, + "loss/core": 0.9366046190261841, + "loss/preference_sft": 0.32833415269851685, + "loss/reference_anchor": 0.5846928954124451, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.344911575317383, + "rewards/margins": 2.230219602584839, + "rewards/rejected": 1.1146920919418335, + "ronpo/logit": 0.2230219542980194, + "ronpo/residual": 0.25300055742263794, + "ronpo/residual_abs": 0.2966272830963135, + "ronpo/target": -0.02997860684990883, + "ronpo/target_abs": 0.08517836779356003, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 515 + }, + { + "drift/chosen_abs": 0.28033939003944397, + "drift/rejected_abs": 0.10343112051486969, + "epoch": 0.1652761223678983, + "grad_norm": 1.4375, + "learning_rate": 4.535250695353039e-08, + "logits/chosen": -1.6896569728851318, + "logits/rejected": -1.6575727462768555, + "logps/chosen": -0.3145434856414795, + "logps/rejected": -0.30273348093032837, + "loss": 0.3629056215286255, + "loss/core": 0.34335726499557495, + "loss/preference_sft": 0.30437609553337097, + "loss/reference_anchor": 0.3605298101902008, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.802736759185791, + "rewards/margins": 1.7711127996444702, + "rewards/rejected": 1.0316243171691895, + "ronpo/logit": 0.17711128294467926, + "ronpo/residual": 0.19178500771522522, + "ronpo/residual_abs": 0.23747721314430237, + "ronpo/target": -0.014673741534352303, + "ronpo/target_abs": 0.07631038129329681, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 520 + }, + { + "drift/chosen_abs": 0.23427505791187286, + "drift/rejected_abs": 0.19274473190307617, + "epoch": 0.16686531585220502, + "grad_norm": 0.458984375, + "learning_rate": 4.438801168771327e-08, + "logits/chosen": -1.811059594154358, + "logits/rejected": -1.7457149028778076, + "logps/chosen": -0.2789601981639862, + "logps/rejected": -0.2822405695915222, + "loss": 0.6040679216384888, + "loss/core": 0.5695381760597229, + "loss/preference_sft": 0.28496214747428894, + "loss/reference_anchor": 0.6620967984199524, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.342750310897827, + "rewards/margins": 0.4186655580997467, + "rewards/rejected": 1.9240849018096924, + "ronpo/logit": 0.04186655953526497, + "ronpo/residual": 0.058896519243717194, + "ronpo/residual_abs": 0.2368089258670807, + "ronpo/target": -0.017029952257871628, + "ronpo/target_abs": 0.065000519156456, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 525 + }, + { + "drift/chosen_abs": 0.27734851837158203, + "drift/rejected_abs": 0.15080618858337402, + "epoch": 0.1684545093365117, + "grad_norm": 18.25, + "learning_rate": 4.342562686118687e-08, + "logits/chosen": -1.7826459407806396, + "logits/rejected": -1.716897964477539, + "logps/chosen": -0.2615172564983368, + "logps/rejected": -0.2668178975582123, + "loss": 0.23560409247875214, + "loss/core": 0.22138634324073792, + "loss/preference_sft": 0.2622404992580414, + "loss/reference_anchor": 0.25813111662864685, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.771690845489502, + "rewards/margins": 1.2636291980743408, + "rewards/rejected": 1.5080616474151611, + "ronpo/logit": 0.12636291980743408, + "ronpo/residual": 0.13701412081718445, + "ronpo/residual_abs": 0.22391745448112488, + "ronpo/target": -0.010651206597685814, + "ronpo/target_abs": 0.08514300733804703, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 530 + }, + { + "drift/chosen_abs": 0.3595048785209656, + "drift/rejected_abs": 0.1292239874601364, + "epoch": 0.17004370282081843, + "grad_norm": 450.0, + "learning_rate": 4.2465714387525845e-08, + "logits/chosen": -1.6385431289672852, + "logits/rejected": -1.6657865047454834, + "logps/chosen": -0.2958716154098511, + "logps/rejected": -0.3102319836616516, + "loss": 1.2631784677505493, + "loss/core": 1.2100608348846436, + "loss/preference_sft": 0.29885178804397583, + "loss/reference_anchor": 1.0324665307998657, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.5940539836883545, + "rewards/margins": 2.3138844966888428, + "rewards/rejected": 1.2801692485809326, + "ronpo/logit": 0.23138847947120667, + "ronpo/residual": 0.22915032505989075, + "ronpo/residual_abs": 0.2899910509586334, + "ronpo/target": 0.002238139044493437, + "ronpo/target_abs": 0.07926853001117706, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 535 + }, + { + "drift/chosen_abs": 0.3161032199859619, + "drift/rejected_abs": 0.10872749239206314, + "epoch": 0.17163289630512515, + "grad_norm": 1.296875, + "learning_rate": 4.1508635250553965e-08, + "logits/chosen": -1.6535122394561768, + "logits/rejected": -1.7463783025741577, + "logps/chosen": -0.27848881483078003, + "logps/rejected": -0.271538108587265, + "loss": 0.9267969131469727, + "loss/core": 0.8974257707595825, + "loss/preference_sft": 0.27611082792282104, + "loss/reference_anchor": 0.5598097443580627, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.160801410675049, + "rewards/margins": 2.073526620864868, + "rewards/rejected": 1.0872750282287598, + "ronpo/logit": 0.20735260844230652, + "ronpo/residual": 0.19190707802772522, + "ronpo/residual_abs": 0.25170207023620605, + "ronpo/target": 0.01544556487351656, + "ronpo/target_abs": 0.08709506690502167, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 540 + }, + { + "drift/chosen_abs": 0.3257554769515991, + "drift/rejected_abs": 0.13702979683876038, + "epoch": 0.17322208978943188, + "grad_norm": 32.25, + "learning_rate": 4.0554749368592995e-08, + "logits/chosen": -1.6410595178604126, + "logits/rejected": -1.6608667373657227, + "logps/chosen": -0.3060685992240906, + "logps/rejected": -0.29526740312576294, + "loss": 0.48791199922561646, + "loss/core": 0.46756306290626526, + "loss/preference_sft": 0.30378422141075134, + "loss/reference_anchor": 0.3766012191772461, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.256690263748169, + "rewards/margins": 1.887966513633728, + "rewards/rejected": 1.368723750114441, + "ronpo/logit": 0.18879660964012146, + "ronpo/residual": 0.17176969349384308, + "ronpo/residual_abs": 0.2844948470592499, + "ronpo/target": 0.017026938498020172, + "ronpo/target_abs": 0.08832515776157379, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 545 + }, + { + "drift/chosen_abs": 0.22399315237998962, + "drift/rejected_abs": 0.10357632488012314, + "epoch": 0.17481128327373857, + "grad_norm": 22.0, + "learning_rate": 3.960441545911204e-08, + "logits/chosen": -1.787977933883667, + "logits/rejected": -1.812804937362671, + "logps/chosen": -0.2800469994544983, + "logps/rejected": -0.27289098501205444, + "loss": 0.11282701790332794, + "loss/core": 0.10225673019886017, + "loss/preference_sft": 0.28042763471603394, + "loss/reference_anchor": 0.18336261808872223, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.2391538619995117, + "rewards/margins": 1.2041218280792236, + "rewards/rejected": 1.035032033920288, + "ronpo/logit": 0.12041217088699341, + "ronpo/residual": 0.10191146284341812, + "ronpo/residual_abs": 0.15801844000816345, + "ronpo/target": 0.018500715494155884, + "ronpo/target_abs": 0.08216994255781174, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 550 + }, + { + "drift/chosen_abs": 0.22719137370586395, + "drift/rejected_abs": 0.0889752134680748, + "epoch": 0.1764004767580453, + "grad_norm": 7.78125, + "learning_rate": 3.8657990903828655e-08, + "logits/chosen": -1.6774036884307861, + "logits/rejected": -1.6927173137664795, + "logps/chosen": -0.28917351365089417, + "logps/rejected": -0.28399351239204407, + "loss": 0.5557877421379089, + "loss/core": 0.531394898891449, + "loss/preference_sft": 0.2850340008735657, + "loss/reference_anchor": 0.45935481786727905, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.271148204803467, + "rewards/margins": 1.386685848236084, + "rewards/rejected": 0.8844623565673828, + "ronpo/logit": 0.13866856694221497, + "ronpo/residual": 0.12557737529277802, + "ronpo/residual_abs": 0.18068963289260864, + "ronpo/target": 0.01309121958911419, + "ronpo/target_abs": 0.08298710733652115, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 555 + }, + { + "drift/chosen_abs": 0.32667970657348633, + "drift/rejected_abs": 0.13467362523078918, + "epoch": 0.17798967024235202, + "grad_norm": 23.125, + "learning_rate": 3.7715831614312174e-08, + "logits/chosen": -1.8457578420639038, + "logits/rejected": -1.8797744512557983, + "logps/chosen": -0.2537483274936676, + "logps/rejected": -0.25457605719566345, + "loss": 0.4204334616661072, + "loss/core": 0.3994172215461731, + "loss/preference_sft": 0.2603244185447693, + "loss/reference_anchor": 0.3942931294441223, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.265937328338623, + "rewards/margins": 1.921112298965454, + "rewards/rejected": 1.3448251485824585, + "ronpo/logit": 0.19211125373840332, + "ronpo/residual": 0.20365771651268005, + "ronpo/residual_abs": 0.2606772780418396, + "ronpo/target": -0.011546471156179905, + "ronpo/target_abs": 0.08113471418619156, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 560 + }, + { + "drift/chosen_abs": 0.18193916976451874, + "drift/rejected_abs": 0.08163003623485565, + "epoch": 0.1795788637266587, + "grad_norm": 2.234375, + "learning_rate": 3.67782918981399e-08, + "logits/chosen": -1.9215097427368164, + "logits/rejected": -1.9562021493911743, + "logps/chosen": -0.2751021981239319, + "logps/rejected": -0.27950841188430786, + "loss": 0.14423240721225739, + "loss/core": 0.13524220883846283, + "loss/preference_sft": 0.27696993947029114, + "loss/reference_anchor": 0.15210723876953125, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 1.818294882774353, + "rewards/margins": 1.004157304763794, + "rewards/rejected": 0.8141376376152039, + "ronpo/logit": 0.10041572898626328, + "ronpo/residual": 0.11555007845163345, + "ronpo/residual_abs": 0.1486656218767166, + "ronpo/target": -0.01513433177024126, + "ronpo/target_abs": 0.07270049303770065, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 565 + }, + { + "drift/chosen_abs": 0.30289047956466675, + "drift/rejected_abs": 0.1840471625328064, + "epoch": 0.18116805721096543, + "grad_norm": 6.96875, + "learning_rate": 3.584572432565648e-08, + "logits/chosen": -1.8963152170181274, + "logits/rejected": -1.8974689245224, + "logps/chosen": -0.2739769220352173, + "logps/rejected": -0.26712292432785034, + "loss": 0.3248220384120941, + "loss/core": 0.3049502372741699, + "loss/preference_sft": 0.2694453001022339, + "loss/reference_anchor": 0.3704921007156372, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.028883934020996, + "rewards/margins": 1.1917301416397095, + "rewards/rejected": 1.837153673171997, + "ronpo/logit": 0.11917302757501602, + "ronpo/residual": 0.1130208745598793, + "ronpo/residual_abs": 0.2363136261701584, + "ronpo/target": 0.00615213718265295, + "ronpo/target_abs": 0.08978766947984695, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 570 + }, + { + "drift/chosen_abs": 0.24188709259033203, + "drift/rejected_abs": 0.09579716622829437, + "epoch": 0.18275725069527216, + "grad_norm": 81.5, + "learning_rate": 3.491847959738672e-08, + "logits/chosen": -1.8621025085449219, + "logits/rejected": -1.7978849411010742, + "logps/chosen": -0.2750481963157654, + "logps/rejected": -0.27491137385368347, + "loss": 0.15762899816036224, + "loss/core": 0.14700618386268616, + "loss/preference_sft": 0.2757367789745331, + "loss/reference_anchor": 0.18488280475139618, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.4188714027404785, + "rewards/margins": 1.4699872732162476, + "rewards/rejected": 0.9488840103149414, + "ronpo/logit": 0.14699873328208923, + "ronpo/residual": 0.12900659441947937, + "ronpo/residual_abs": 0.19406011700630188, + "ronpo/target": 0.01799212209880352, + "ronpo/target_abs": 0.0778588205575943, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 575 + }, + { + "drift/chosen_abs": 0.42244258522987366, + "drift/rejected_abs": 0.14777334034442902, + "epoch": 0.18434644417957885, + "grad_norm": 85.5, + "learning_rate": 3.3996906412151415e-08, + "logits/chosen": -1.7508312463760376, + "logits/rejected": -1.7235362529754639, + "logps/chosen": -0.2758569121360779, + "logps/rejected": -0.2773497700691223, + "loss": 0.6645770072937012, + "loss/core": 0.6348112225532532, + "loss/preference_sft": 0.2726094722747803, + "loss/reference_anchor": 0.5680543780326843, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.22373104095459, + "rewards/margins": 2.7611031532287598, + "rewards/rejected": 1.4626275300979614, + "ronpo/logit": 0.27611032128334045, + "ronpo/residual": 0.27662983536720276, + "ronpo/residual_abs": 0.3160479664802551, + "ronpo/target": -0.0005195157718844712, + "ronpo/target_abs": 0.0704670175909996, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 580 + }, + { + "drift/chosen_abs": 0.2685272991657257, + "drift/rejected_abs": 0.12745855748653412, + "epoch": 0.18593563766388557, + "grad_norm": 8.25, + "learning_rate": 3.3081351335935947e-08, + "logits/chosen": -1.864542007446289, + "logits/rejected": -1.882070541381836, + "logps/chosen": -0.2766827344894409, + "logps/rejected": -0.2663271725177765, + "loss": 0.3190165162086487, + "loss/core": 0.29371339082717896, + "loss/preference_sft": 0.27629172801971436, + "loss/reference_anchor": 0.47843486070632935, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.683490753173828, + "rewards/margins": 1.414543867111206, + "rewards/rejected": 1.268946886062622, + "ronpo/logit": 0.14145438373088837, + "ronpo/residual": 0.11811937391757965, + "ronpo/residual_abs": 0.18924079835414886, + "ronpo/target": 0.02333502098917961, + "ronpo/target_abs": 0.07983358204364777, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 585 + }, + { + "drift/chosen_abs": 0.28528568148612976, + "drift/rejected_abs": 0.1476709544658661, + "epoch": 0.1875248311481923, + "grad_norm": 2.765625, + "learning_rate": 3.2172158671561e-08, + "logits/chosen": -1.8741655349731445, + "logits/rejected": -1.9201557636260986, + "logps/chosen": -0.29076990485191345, + "logps/rejected": -0.28063392639160156, + "loss": 0.3805859088897705, + "loss/core": 0.36150652170181274, + "loss/preference_sft": 0.2858158349990845, + "loss/reference_anchor": 0.3530064523220062, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.8528571128845215, + "rewards/margins": 1.3815311193466187, + "rewards/rejected": 1.4713258743286133, + "ronpo/logit": 0.1381531059741974, + "ronpo/residual": 0.14053115248680115, + "ronpo/residual_abs": 0.2531353533267975, + "ronpo/target": -0.002378053730353713, + "ronpo/target_abs": 0.07857644557952881, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 590 + }, + { + "drift/chosen_abs": 0.1880657821893692, + "drift/rejected_abs": 0.08449472486972809, + "epoch": 0.18911402463249902, + "grad_norm": 5.9375, + "learning_rate": 3.126967032920439e-08, + "logits/chosen": -1.7578903436660767, + "logits/rejected": -1.783585786819458, + "logps/chosen": -0.28262588381767273, + "logps/rejected": -0.2848055958747864, + "loss": 0.07938707619905472, + "loss/core": 0.07369506359100342, + "loss/preference_sft": 0.28935831785202026, + "loss/reference_anchor": 0.08490435779094696, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.8778693675994873, + "rewards/margins": 1.0337949991226196, + "rewards/rejected": 0.8440744280815125, + "ronpo/logit": 0.1033795103430748, + "ronpo/residual": 0.09837718307971954, + "ronpo/residual_abs": 0.14326299726963043, + "ronpo/target": 0.005002323072403669, + "ronpo/target_abs": 0.06280878931283951, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 595 + }, + { + "drift/chosen_abs": 0.38983526825904846, + "drift/rejected_abs": 0.1542014479637146, + "epoch": 0.1907032181168057, + "grad_norm": 6.0625, + "learning_rate": 3.037422569782264e-08, + "logits/chosen": -1.7056328058242798, + "logits/rejected": -1.6530392169952393, + "logps/chosen": -0.2509880065917969, + "logps/rejected": -0.2619604170322418, + "loss": 0.701583206653595, + "loss/core": 0.663438081741333, + "loss/preference_sft": 0.25877368450164795, + "loss/reference_anchor": 0.7370229959487915, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.898181200027466, + "rewards/margins": 2.3588569164276123, + "rewards/rejected": 1.5393238067626953, + "ronpo/logit": 0.23588573932647705, + "ronpo/residual": 0.23551622033119202, + "ronpo/residual_abs": 0.29292362928390503, + "ronpo/target": 0.0003694974002428353, + "ronpo/target_abs": 0.08346446603536606, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 600 + }, + { + "drift/chosen_abs": 0.38783174753189087, + "drift/rejected_abs": 0.19440807402133942, + "epoch": 0.19229241160111243, + "grad_norm": 944.0, + "learning_rate": 2.9486161517520596e-08, + "logits/chosen": -1.8185880184173584, + "logits/rejected": -1.771431565284729, + "logps/chosen": -0.28466832637786865, + "logps/rejected": -0.28105106949806213, + "loss": 0.8451844453811646, + "loss/core": 0.7918490171432495, + "loss/preference_sft": 0.2889874577522278, + "loss/reference_anchor": 1.0378073453903198, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.8768718242645264, + "rewards/margins": 1.943351149559021, + "rewards/rejected": 1.9335205554962158, + "ronpo/logit": 0.19433513283729553, + "ronpo/residual": 0.17206700146198273, + "ronpo/residual_abs": 0.29958412051200867, + "ronpo/target": 0.02226812019944191, + "ronpo/target_abs": 0.08312074840068817, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 605 + }, + { + "drift/chosen_abs": 0.3009849190711975, + "drift/rejected_abs": 0.11645524203777313, + "epoch": 0.19388160508541916, + "grad_norm": 177.0, + "learning_rate": 2.8605811752917298e-08, + "logits/chosen": -1.9999271631240845, + "logits/rejected": -2.000976085662842, + "logps/chosen": -0.27779993414878845, + "logps/rejected": -0.277484655380249, + "loss": 0.6784787178039551, + "loss/core": 0.6391760110855103, + "loss/preference_sft": 0.2794664800167084, + "loss/reference_anchor": 0.7581073641777039, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.006049394607544, + "rewards/margins": 1.8458268642425537, + "rewards/rejected": 1.1602224111557007, + "ronpo/logit": 0.1845826953649521, + "ronpo/residual": 0.18958279490470886, + "ronpo/residual_abs": 0.23554134368896484, + "ronpo/target": -0.005000096745789051, + "ronpo/target_abs": 0.07321993261575699, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 610 + }, + { + "drift/chosen_abs": 0.27367550134658813, + "drift/rejected_abs": 0.08648597449064255, + "epoch": 0.19547079856972585, + "grad_norm": 1.4296875, + "learning_rate": 2.7733507467555527e-08, + "logits/chosen": -1.92385733127594, + "logits/rejected": -1.9347591400146484, + "logps/chosen": -0.294109970331192, + "logps/rejected": -0.27960658073425293, + "loss": 0.4980928897857666, + "loss/core": 0.48153653740882874, + "loss/preference_sft": 0.2906952500343323, + "loss/reference_anchor": 0.30205675959587097, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.735163688659668, + "rewards/margins": 1.8733736276626587, + "rewards/rejected": 0.8617900609970093, + "ronpo/logit": 0.18733736872673035, + "ronpo/residual": 0.18723754584789276, + "ronpo/residual_abs": 0.24290697276592255, + "ronpo/target": 9.980481263482943e-05, + "ronpo/target_abs": 0.07993032038211823, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 615 + }, + { + "drift/chosen_abs": 0.11056061089038849, + "drift/rejected_abs": 0.0920964851975441, + "epoch": 0.19705999205403257, + "grad_norm": 197.0, + "learning_rate": 2.686957669940242e-08, + "logits/chosen": -1.8443567752838135, + "logits/rejected": -1.8728549480438232, + "logps/chosen": -0.3104780912399292, + "logps/rejected": -0.30508318543434143, + "loss": 0.1757979691028595, + "loss/core": 0.16883358359336853, + "loss/preference_sft": 0.303710013628006, + "loss/reference_anchor": 0.108916737139225, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.1045570373535156, + "rewards/margins": 0.18372417986392975, + "rewards/rejected": 0.9208329319953918, + "ronpo/logit": 0.018372418358922005, + "ronpo/residual": 0.047217488288879395, + "ronpo/residual_abs": 0.1612224578857422, + "ronpo/target": -0.02884506806731224, + "ronpo/target_abs": 0.08857301622629166, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 620 + }, + { + "drift/chosen_abs": 0.24403142929077148, + "drift/rejected_abs": 0.1381409615278244, + "epoch": 0.1986491855383393, + "grad_norm": 3.203125, + "learning_rate": 2.6014344337487702e-08, + "logits/chosen": -1.6158983707427979, + "logits/rejected": -1.693192720413208, + "logps/chosen": -0.29910343885421753, + "logps/rejected": -0.2826142907142639, + "loss": 0.07637186348438263, + "loss/core": 0.06501691043376923, + "loss/preference_sft": 0.29273831844329834, + "loss/reference_anchor": 0.1978251039981842, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.4398252964019775, + "rewards/margins": 1.059569001197815, + "rewards/rejected": 1.3802564144134521, + "ronpo/logit": 0.10595688968896866, + "ronpo/residual": 0.10343287140130997, + "ronpo/residual_abs": 0.15932440757751465, + "ronpo/target": 0.0025240257382392883, + "ronpo/target_abs": 0.08598555624485016, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 625 + }, + { + "drift/chosen_abs": 0.263141006231308, + "drift/rejected_abs": 0.11765114963054657, + "epoch": 0.20023837902264602, + "grad_norm": 6.4375, + "learning_rate": 2.51681319997262e-08, + "logits/chosen": -2.0076048374176025, + "logits/rejected": -1.9558988809585571, + "logps/chosen": -0.27460214495658875, + "logps/rejected": -0.26767414808273315, + "loss": 0.26412975788116455, + "loss/core": 0.24851731956005096, + "loss/preference_sft": 0.2709651291370392, + "loss/reference_anchor": 0.2851526141166687, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.6304500102996826, + "rewards/margins": 1.4610058069229126, + "rewards/rejected": 1.1694444417953491, + "ronpo/logit": 0.14610058069229126, + "ronpo/residual": 0.13935787975788116, + "ronpo/residual_abs": 0.1994233876466751, + "ronpo/target": 0.006742709316313267, + "ronpo/target_abs": 0.0730559304356575, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 630 + }, + { + "drift/chosen_abs": 0.16716927289962769, + "drift/rejected_abs": 0.07392685115337372, + "epoch": 0.2018275725069527, + "grad_norm": 1.1015625, + "learning_rate": 2.4331257911970626e-08, + "logits/chosen": -1.8764339685440063, + "logits/rejected": -1.900081992149353, + "logps/chosen": -0.3134706914424896, + "logps/rejected": -0.3003923296928406, + "loss": 0.12685158848762512, + "loss/core": 0.12000010162591934, + "loss/preference_sft": 0.312440425157547, + "loss/reference_anchor": 0.10578552633523941, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.670975923538208, + "rewards/margins": 0.9369431734085083, + "rewards/rejected": 0.7340327501296997, + "ronpo/logit": 0.09369431436061859, + "ronpo/residual": 0.08684712648391724, + "ronpo/residual_abs": 0.1398160457611084, + "ronpo/target": 0.0068471841514110565, + "ronpo/target_abs": 0.0784897580742836, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 635 + }, + { + "drift/chosen_abs": 0.34248116612434387, + "drift/rejected_abs": 0.08185670524835587, + "epoch": 0.20341676599125944, + "grad_norm": 1.1796875, + "learning_rate": 2.350403678833976e-08, + "logits/chosen": -1.7811704874038696, + "logits/rejected": -1.7476812601089478, + "logps/chosen": -0.316486656665802, + "logps/rejected": -0.3071419894695282, + "loss": 1.2131757736206055, + "loss/core": 1.1782453060150146, + "loss/preference_sft": 0.30921900272369385, + "loss/reference_anchor": 0.667685866355896, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.4235236644744873, + "rewards/margins": 2.605436086654663, + "rewards/rejected": 0.8180872797966003, + "ronpo/logit": 0.2605436146259308, + "ronpo/residual": 0.2597568929195404, + "ronpo/residual_abs": 0.3174801170825958, + "ronpo/target": 0.0007867075619287789, + "ronpo/target_abs": 0.0725146010518074, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 640 + }, + { + "drift/chosen_abs": 0.3487434983253479, + "drift/rejected_abs": 0.1786487102508545, + "epoch": 0.20500595947556616, + "grad_norm": 31.875, + "learning_rate": 2.2686779712867315e-08, + "logits/chosen": -1.9565203189849854, + "logits/rejected": -1.954850435256958, + "logps/chosen": -0.29713040590286255, + "logps/rejected": -0.29292789101600647, + "loss": 0.609022319316864, + "loss/core": 0.548900842666626, + "loss/preference_sft": 0.2938844561576843, + "loss/reference_anchor": 1.1730408668518066, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 3.4874351024627686, + "rewards/margins": 1.7022377252578735, + "rewards/rejected": 1.7851974964141846, + "ronpo/logit": 0.17022375762462616, + "ronpo/residual": 0.17084741592407227, + "ronpo/residual_abs": 0.2177102118730545, + "ronpo/target": -0.0006236616754904389, + "ronpo/target_abs": 0.0669097825884819, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 645 + }, + { + "drift/chosen_abs": 0.2442498654127121, + "drift/rejected_abs": 0.09864034503698349, + "epoch": 0.20659515295987285, + "grad_norm": 35.5, + "learning_rate": 2.1879794022516e-08, + "logits/chosen": -1.7124214172363281, + "logits/rejected": -1.7760320901870728, + "logps/chosen": -0.2731096148490906, + "logps/rejected": -0.28077295422554016, + "loss": 0.12899014353752136, + "loss/core": 0.11936305463314056, + "loss/preference_sft": 0.2820122241973877, + "loss/reference_anchor": 0.16434051096439362, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.4420547485351562, + "rewards/margins": 1.4615581035614014, + "rewards/rejected": 0.9804967641830444, + "ronpo/logit": 0.14615580439567566, + "ronpo/residual": 0.1428360939025879, + "ronpo/residual_abs": 0.18947355449199677, + "ronpo/target": 0.003319724230095744, + "ronpo/target_abs": 0.0828058049082756, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 650 + }, + { + "drift/chosen_abs": 0.18947014212608337, + "drift/rejected_abs": 0.0865272730588913, + "epoch": 0.20818434644417957, + "grad_norm": 48.25, + "learning_rate": 2.1083383191600673e-08, + "logits/chosen": -1.6375888586044312, + "logits/rejected": -1.6597034931182861, + "logps/chosen": -0.2796572744846344, + "logps/rejected": -0.2848132252693176, + "loss": 0.2575990557670593, + "loss/core": 0.2444666624069214, + "loss/preference_sft": 0.28200775384902954, + "loss/reference_anchor": 0.23444685339927673, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.8931496143341064, + "rewards/margins": 1.028186321258545, + "rewards/rejected": 0.8649630546569824, + "ronpo/logit": 0.10281863063573837, + "ronpo/residual": 0.09756249189376831, + "ronpo/residual_abs": 0.17437264323234558, + "ronpo/target": 0.005256150849163532, + "ronpo/target_abs": 0.0813509151339531, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 655 + }, + { + "drift/chosen_abs": 0.2058045119047165, + "drift/rejected_abs": 0.0914444550871849, + "epoch": 0.2097735399284863, + "grad_norm": 192.0, + "learning_rate": 2.029784671766404e-08, + "logits/chosen": -1.7744375467300415, + "logits/rejected": -1.7810022830963135, + "logps/chosen": -0.2940472662448883, + "logps/rejected": -0.29379576444625854, + "loss": 0.24170386791229248, + "loss/core": 0.23180270195007324, + "loss/preference_sft": 0.2991836667060852, + "loss/reference_anchor": 0.1681046038866043, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.056238889694214, + "rewards/margins": 1.14398193359375, + "rewards/rejected": 0.9122568368911743, + "ronpo/logit": 0.11439820379018784, + "ronpo/residual": 0.10298512876033783, + "ronpo/residual_abs": 0.164739191532135, + "ronpo/target": 0.01141307596117258, + "ronpo/target_abs": 0.07934392988681793, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 660 + }, + { + "drift/chosen_abs": 0.38957634568214417, + "drift/rejected_abs": 0.11693358421325684, + "epoch": 0.21136273341279302, + "grad_norm": 5.28125, + "learning_rate": 1.9523480008847854e-08, + "logits/chosen": -1.8623239994049072, + "logits/rejected": -1.796321153640747, + "logps/chosen": -0.27734166383743286, + "logps/rejected": -0.2643659710884094, + "loss": 0.7925857305526733, + "loss/core": 0.7652046084403992, + "loss/preference_sft": 0.2695058584213257, + "loss/reference_anchor": 0.5206728577613831, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 3.895763397216797, + "rewards/margins": 2.7316036224365234, + "rewards/rejected": 1.1641600131988525, + "ronpo/logit": 0.2731603682041168, + "ronpo/residual": 0.28218016028404236, + "ronpo/residual_abs": 0.3127972483634949, + "ronpo/target": -0.009019814431667328, + "ronpo/target_abs": 0.07987822592258453, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 665 + }, + { + "drift/chosen_abs": 0.14632907509803772, + "drift/rejected_abs": 0.0827428549528122, + "epoch": 0.2129519268970997, + "grad_norm": 6.8125, + "learning_rate": 1.8760574272802e-08, + "logits/chosen": -1.893527626991272, + "logits/rejected": -1.864617943763733, + "logps/chosen": -0.2893826961517334, + "logps/rejected": -0.28469884395599365, + "loss": 0.048207398504018784, + "loss/core": 0.04376941919326782, + "loss/preference_sft": 0.29032936692237854, + "loss/reference_anchor": 0.05972672253847122, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.463290810585022, + "rewards/margins": 0.6385823488235474, + "rewards/rejected": 0.8247084617614746, + "ronpo/logit": 0.06385823339223862, + "ronpo/residual": 0.04385846480727196, + "ronpo/residual_abs": 0.13213634490966797, + "ronpo/target": 0.019999774172902107, + "ronpo/target_abs": 0.09107968956232071, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 670 + }, + { + "drift/chosen_abs": 0.1413107067346573, + "drift/rejected_abs": 0.06459660083055496, + "epoch": 0.21454112038140644, + "grad_norm": 3.078125, + "learning_rate": 1.8009416407173257e-08, + "logits/chosen": -1.669947624206543, + "logits/rejected": -1.8324556350708008, + "logps/chosen": -0.31188154220581055, + "logps/rejected": -0.3008914887905121, + "loss": 0.0633026510477066, + "loss/core": 0.05928937345743179, + "loss/preference_sft": 0.30907323956489563, + "loss/reference_anchor": 0.04935837537050247, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.4124871492385864, + "rewards/margins": 0.767764687538147, + "rewards/rejected": 0.6447224020957947, + "ronpo/logit": 0.07677648216485977, + "ronpo/residual": 0.06589711457490921, + "ronpo/residual_abs": 0.1341782808303833, + "ronpo/target": 0.010879358276724815, + "ronpo/target_abs": 0.07572633028030396, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 675 + }, + { + "drift/chosen_abs": 0.25216197967529297, + "drift/rejected_abs": 0.13002386689186096, + "epoch": 0.21613031386571316, + "grad_norm": 9.875, + "learning_rate": 1.7270288891714813e-08, + "logits/chosen": -1.821359634399414, + "logits/rejected": -1.7911430597305298, + "logps/chosen": -0.2525402307510376, + "logps/rejected": -0.26482558250427246, + "loss": 0.3033674359321594, + "loss/core": 0.2915551960468292, + "loss/preference_sft": 0.2592369616031647, + "loss/reference_anchor": 0.2103213369846344, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.5206844806671143, + "rewards/margins": 1.2219209671020508, + "rewards/rejected": 1.2987635135650635, + "ronpo/logit": 0.12219210714101791, + "ronpo/residual": 0.10521011054515839, + "ronpo/residual_abs": 0.25379353761672974, + "ronpo/target": 0.016981983557343483, + "ronpo/target_abs": 0.08764125406742096, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 680 + }, + { + "drift/chosen_abs": 0.2599492371082306, + "drift/rejected_abs": 0.15403486788272858, + "epoch": 0.21771950735001985, + "grad_norm": 7.5, + "learning_rate": 1.6543469682057103e-08, + "logits/chosen": -1.9823789596557617, + "logits/rejected": -1.9565290212631226, + "logps/chosen": -0.267892062664032, + "logps/rejected": -0.2650378346443176, + "loss": 0.14402912557125092, + "loss/core": 0.1269291490316391, + "loss/preference_sft": 0.26532235741615295, + "loss/reference_anchor": 0.3154672384262085, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.5976312160491943, + "rewards/margins": 1.058511734008789, + "rewards/rejected": 1.5391194820404053, + "ronpo/logit": 0.1058511734008789, + "ronpo/residual": 0.12076152861118317, + "ronpo/residual_abs": 0.1801002025604248, + "ronpo/target": -0.014910360798239708, + "ronpo/target_abs": 0.08301638811826706, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 685 + }, + { + "drift/chosen_abs": 0.18989941477775574, + "drift/rejected_abs": 0.08928252756595612, + "epoch": 0.21930870083432658, + "grad_norm": 1.1328125, + "learning_rate": 1.582923210518014e-08, + "logits/chosen": -1.8569204807281494, + "logits/rejected": -1.8808708190917969, + "logps/chosen": -0.2738683819770813, + "logps/rejected": -0.277884304523468, + "loss": 0.1312539279460907, + "loss/core": 0.12408691644668579, + "loss/preference_sft": 0.2749662697315216, + "loss/reference_anchor": 0.11584333330392838, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 1.898453950881958, + "rewards/margins": 1.0071128606796265, + "rewards/rejected": 0.8913409113883972, + "ronpo/logit": 0.10071128606796265, + "ronpo/residual": 0.09246021509170532, + "ronpo/residual_abs": 0.14917293190956116, + "ronpo/target": 0.0082510681822896, + "ronpo/target_abs": 0.07735707610845566, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 690 + }, + { + "drift/chosen_abs": 0.2127593457698822, + "drift/rejected_abs": 0.1324995905160904, + "epoch": 0.2208978943186333, + "grad_norm": 1.7734375, + "learning_rate": 1.5127844756626436e-08, + "logits/chosen": -1.8886617422103882, + "logits/rejected": -1.866326093673706, + "logps/chosen": -0.30845409631729126, + "logps/rejected": -0.311231791973114, + "loss": 0.07940120995044708, + "loss/core": 0.06841110438108444, + "loss/preference_sft": 0.3118094503879547, + "loss/reference_anchor": 0.18862099945545197, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.124119281768799, + "rewards/margins": 0.8049348592758179, + "rewards/rejected": 1.31918466091156, + "ronpo/logit": 0.0804934874176979, + "ronpo/residual": 0.08477647602558136, + "ronpo/residual_abs": 0.15998438000679016, + "ronpo/target": -0.004282998386770487, + "ronpo/target_abs": 0.08868735283613205, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 695 + }, + { + "drift/chosen_abs": 0.2849193811416626, + "drift/rejected_abs": 0.15217965841293335, + "epoch": 0.22248708780294002, + "grad_norm": 3.828125, + "learning_rate": 1.4439571399493145e-08, + "logits/chosen": -1.804674506187439, + "logits/rejected": -1.758469581604004, + "logps/chosen": -0.268559992313385, + "logps/rejected": -0.25594258308410645, + "loss": 0.3818897306919098, + "loss/core": 0.36068207025527954, + "loss/preference_sft": 0.27696743607521057, + "loss/reference_anchor": 0.3964560031890869, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.849193811416626, + "rewards/margins": 1.3282945156097412, + "rewards/rejected": 1.5208990573883057, + "ronpo/logit": 0.132829487323761, + "ronpo/residual": 0.14115296304225922, + "ronpo/residual_abs": 0.24943992495536804, + "ronpo/target": -0.008323474787175655, + "ronpo/target_abs": 0.07197760045528412, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 700 + }, + { + "drift/chosen_abs": 0.32905009388923645, + "drift/rejected_abs": 0.09508834779262543, + "epoch": 0.22407628128724671, + "grad_norm": 25.125, + "learning_rate": 1.3764670865241557e-08, + "logits/chosen": -1.7177156209945679, + "logits/rejected": -1.6808580160140991, + "logps/chosen": -0.26480594277381897, + "logps/rejected": -0.27780020236968994, + "loss": 1.043757677078247, + "loss/core": 1.009283423423767, + "loss/preference_sft": 0.2664128839969635, + "loss/reference_anchor": 0.6628433465957642, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.290501117706299, + "rewards/margins": 2.3396174907684326, + "rewards/rejected": 0.9508835673332214, + "ronpo/logit": 0.2339617758989334, + "ronpo/residual": 0.24024538695812225, + "ronpo/residual_abs": 0.28473344445228577, + "ronpo/target": -0.00628363573923707, + "ronpo/target_abs": 0.07632976025342941, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 705 + }, + { + "drift/chosen_abs": 0.4384216368198395, + "drift/rejected_abs": 0.1377977877855301, + "epoch": 0.22566547477155344, + "grad_norm": 41.5, + "learning_rate": 1.310339695636118e-08, + "logits/chosen": -1.8576635122299194, + "logits/rejected": -1.8414652347564697, + "logps/chosen": -0.30472978949546814, + "logps/rejected": -0.3077087104320526, + "loss": 1.5751392841339111, + "loss/core": 1.5171544551849365, + "loss/preference_sft": 0.3141063153743744, + "loss/reference_anchor": 1.1282850503921509, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.384188652038574, + "rewards/margins": 3.014103651046753, + "rewards/rejected": 1.37008535861969, + "ronpo/logit": 0.30141040682792664, + "ronpo/residual": 0.3095789849758148, + "ronpo/residual_abs": 0.35598164796829224, + "ronpo/target": -0.008168593980371952, + "ronpo/target_abs": 0.08209376782178879, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 710 + }, + { + "drift/chosen_abs": 0.2829729914665222, + "drift/rejected_abs": 0.10287122428417206, + "epoch": 0.22725466825586016, + "grad_norm": 3.40625, + "learning_rate": 1.245599835092504e-08, + "logits/chosen": -1.7704976797103882, + "logits/rejected": -1.8326141834259033, + "logps/chosen": -0.29713183641433716, + "logps/rejected": -0.3228703439235687, + "loss": 0.8810839653015137, + "loss/core": 0.8518657684326172, + "loss/preference_sft": 0.30574506521224976, + "loss/reference_anchor": 0.5537921786308289, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.8288185596466064, + "rewards/margins": 1.8033968210220337, + "rewards/rejected": 1.0254218578338623, + "ronpo/logit": 0.18033969402313232, + "ronpo/residual": 0.15812399983406067, + "ronpo/residual_abs": 0.22622855007648468, + "ronpo/target": 0.02221568673849106, + "ronpo/target_abs": 0.07745952159166336, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 715 + }, + { + "drift/chosen_abs": 0.4834979474544525, + "drift/rejected_abs": 0.14388954639434814, + "epoch": 0.22884386174016685, + "grad_norm": 10.0625, + "learning_rate": 1.1822718509071988e-08, + "logits/chosen": -1.8110580444335938, + "logits/rejected": -1.8221015930175781, + "logps/chosen": -0.2708401083946228, + "logps/rejected": -0.2723672091960907, + "loss": 2.1321358680725098, + "loss/core": 2.061873435974121, + "loss/preference_sft": 0.2761738896369934, + "loss/reference_anchor": 1.3776265382766724, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.834713459014893, + "rewards/margins": 3.3967089653015137, + "rewards/rejected": 1.4380043745040894, + "ronpo/logit": 0.33967092633247375, + "ronpo/residual": 0.3569709062576294, + "ronpo/residual_abs": 0.4112846255302429, + "ronpo/target": -0.017299985513091087, + "ronpo/target_abs": 0.08841768652200699, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 720 + }, + { + "drift/chosen_abs": 0.24089869856834412, + "drift/rejected_abs": 0.11333000659942627, + "epoch": 0.23043305522447358, + "grad_norm": 9.3125, + "learning_rate": 1.1203795581451287e-08, + "logits/chosen": -1.7944285869598389, + "logits/rejected": -1.7804477214813232, + "logps/chosen": -0.3111926019191742, + "logps/rejected": -0.3086337447166443, + "loss": 0.23198828101158142, + "loss/core": 0.2124747484922409, + "loss/preference_sft": 0.31827235221862793, + "loss/reference_anchor": 0.35844409465789795, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.3976962566375732, + "rewards/margins": 1.275817632675171, + "rewards/rejected": 1.121878743171692, + "ronpo/logit": 0.12758177518844604, + "ronpo/residual": 0.1312047243118286, + "ronpo/residual_abs": 0.1830698698759079, + "ronpo/target": -0.0036229542456567287, + "ronpo/target_abs": 0.07441584765911102, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 725 + }, + { + "drift/chosen_abs": 0.18596777319908142, + "drift/rejected_abs": 0.09318926930427551, + "epoch": 0.2320222487087803, + "grad_norm": 91.5, + "learning_rate": 1.0599462319663904e-08, + "logits/chosen": -1.8789104223251343, + "logits/rejected": -1.7996902465820312, + "logps/chosen": -0.28050804138183594, + "logps/rejected": -0.28717318177223206, + "loss": 0.09919708967208862, + "loss/core": 0.08663090318441391, + "loss/preference_sft": 0.28598660230636597, + "loss/reference_anchor": 0.22272494435310364, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.859379768371582, + "rewards/margins": 0.9287394285202026, + "rewards/rejected": 0.9306403398513794, + "ronpo/logit": 0.0928739532828331, + "ronpo/residual": 0.10799574851989746, + "ronpo/residual_abs": 0.14936821162700653, + "ronpo/target": -0.015121802687644958, + "ronpo/target_abs": 0.07535012811422348, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 730 + }, + { + "drift/chosen_abs": 0.18436908721923828, + "drift/rejected_abs": 0.08182942867279053, + "epoch": 0.23361144219308702, + "grad_norm": 2.96875, + "learning_rate": 1.0009945988734203e-08, + "logits/chosen": -1.8670095205307007, + "logits/rejected": -1.9019635915756226, + "logps/chosen": -0.2799551486968994, + "logps/rejected": -0.27838417887687683, + "loss": 0.13352563977241516, + "loss/core": 0.12648555636405945, + "loss/preference_sft": 0.2852904796600342, + "loss/reference_anchor": 0.11227299273014069, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.843457818031311, + "rewards/margins": 1.0267233848571777, + "rewards/rejected": 0.8167344331741333, + "ronpo/logit": 0.10267233848571777, + "ronpo/residual": 0.10309547185897827, + "ronpo/residual_abs": 0.14217627048492432, + "ronpo/target": -0.00042313747690059245, + "ronpo/target_abs": 0.07198002189397812, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 735 + }, + { + "drift/chosen_abs": 0.18157462775707245, + "drift/rejected_abs": 0.05101984739303589, + "epoch": 0.23520063567739372, + "grad_norm": 3.703125, + "learning_rate": 9.435468281644798e-09, + "logits/chosen": -1.789648413658142, + "logits/rejected": -1.8189513683319092, + "logps/chosen": -0.2933686375617981, + "logps/rejected": -0.2890208959579468, + "loss": 0.39400720596313477, + "loss/core": 0.38300806283950806, + "loss/preference_sft": 0.2970552444458008, + "loss/reference_anchor": 0.19027724862098694, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.814532995223999, + "rewards/margins": 1.3092584609985352, + "rewards/rejected": 0.5052744150161743, + "ronpo/logit": 0.13092584908008575, + "ronpo/residual": 0.12319651991128922, + "ronpo/residual_abs": 0.16983821988105774, + "ronpo/target": 0.0077293342910707, + "ronpo/target_abs": 0.06624205410480499, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 740 + }, + { + "drift/chosen_abs": 0.3139849007129669, + "drift/rejected_abs": 0.12468956410884857, + "epoch": 0.23678982916170044, + "grad_norm": 7.1875, + "learning_rate": 8.876245235966883e-09, + "logits/chosen": -1.5578440427780151, + "logits/rejected": -1.514190673828125, + "logps/chosen": -0.2828544080257416, + "logps/rejected": -0.29736337065696716, + "loss": 0.2802901864051819, + "loss/core": 0.265177845954895, + "loss/preference_sft": 0.29184380173683167, + "loss/reference_anchor": 0.27306291460990906, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.1378657817840576, + "rewards/margins": 1.8915119171142578, + "rewards/rejected": 1.2463538646697998, + "ronpo/logit": 0.18915119767189026, + "ronpo/residual": 0.19307444989681244, + "ronpo/residual_abs": 0.24125421047210693, + "ronpo/target": -0.0039232466369867325, + "ronpo/target_abs": 0.08785430341959, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 745 + }, + { + "drift/chosen_abs": 0.3995332419872284, + "drift/rejected_abs": 0.20359914004802704, + "epoch": 0.23837902264600716, + "grad_norm": 12.625, + "learning_rate": 8.332487152617423e-09, + "logits/chosen": -1.831875205039978, + "logits/rejected": -1.8069343566894531, + "logps/chosen": -0.29953733086586, + "logps/rejected": -0.282090961933136, + "loss": 1.5924561023712158, + "loss/core": 1.5354974269866943, + "loss/preference_sft": 0.28296080231666565, + "loss/reference_anchor": 1.1108777523040771, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.993727207183838, + "rewards/margins": 1.9586591720581055, + "rewards/rejected": 2.035067558288574, + "ronpo/logit": 0.19586597383022308, + "ronpo/residual": 0.18231366574764252, + "ronpo/residual_abs": 0.3800063729286194, + "ronpo/target": 0.013552306219935417, + "ronpo/target_abs": 0.06323491036891937, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 750 + }, + { + "drift/chosen_abs": 0.4130985140800476, + "drift/rejected_abs": 0.10734666883945465, + "epoch": 0.23996821613031386, + "grad_norm": 37.25, + "learning_rate": 7.804398516773464e-09, + "logits/chosen": -1.844090461730957, + "logits/rejected": -1.9883720874786377, + "logps/chosen": -0.298724889755249, + "logps/rejected": -0.29675233364105225, + "loss": 1.6366679668426514, + "loss/core": 1.5810120105743408, + "loss/preference_sft": 0.3001899719238281, + "loss/reference_anchor": 1.083099365234375, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.13097620010376, + "rewards/margins": 3.073918342590332, + "rewards/rejected": 1.0570578575134277, + "ronpo/logit": 0.30739185214042664, + "ronpo/residual": 0.29710453748703003, + "ronpo/residual_abs": 0.3430529236793518, + "ronpo/target": 0.01028729323297739, + "ronpo/target_abs": 0.06950847059488297, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 755 + }, + { + "drift/chosen_abs": 0.3958708643913269, + "drift/rejected_abs": 0.11953407526016235, + "epoch": 0.24155740961462058, + "grad_norm": 17.75, + "learning_rate": 7.292177920973724e-09, + "logits/chosen": -1.7642419338226318, + "logits/rejected": -1.7521655559539795, + "logps/chosen": -0.2835713028907776, + "logps/rejected": -0.2796153426170349, + "loss": 1.3200641870498657, + "loss/core": 1.2784984111785889, + "loss/preference_sft": 0.2840017080307007, + "loss/reference_anchor": 0.8029199838638306, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.9578804969787598, + "rewards/margins": 2.7641587257385254, + "rewards/rejected": 1.1937220096588135, + "ronpo/logit": 0.2764158248901367, + "ronpo/residual": 0.26121872663497925, + "ronpo/residual_abs": 0.325788676738739, + "ronpo/target": 0.015197129920125008, + "ronpo/target_abs": 0.07956832647323608, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 760 + }, + { + "drift/chosen_abs": 0.2987801730632782, + "drift/rejected_abs": 0.1019306555390358, + "epoch": 0.2431466030989273, + "grad_norm": 1.65625, + "learning_rate": 6.796017990435976e-09, + "logits/chosen": -1.7064396142959595, + "logits/rejected": -1.7161552906036377, + "logps/chosen": -0.27210524678230286, + "logps/rejected": -0.26053616404533386, + "loss": 0.3539239466190338, + "loss/core": 0.3408985733985901, + "loss/preference_sft": 0.26584452390670776, + "loss/reference_anchor": 0.23392312228679657, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.987363338470459, + "rewards/margins": 1.9795618057250977, + "rewards/rejected": 1.0078017711639404, + "ronpo/logit": 0.1979561746120453, + "ronpo/residual": 0.19465027749538422, + "ronpo/residual_abs": 0.2738838493824005, + "ronpo/target": 0.003305905032902956, + "ronpo/target_abs": 0.06634985655546188, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 765 + }, + { + "drift/chosen_abs": 0.2944992780685425, + "drift/rejected_abs": 0.08568915724754333, + "epoch": 0.24473579658323402, + "grad_norm": 1.5546875, + "learning_rate": 6.316105310618664e-09, + "logits/chosen": -1.618821144104004, + "logits/rejected": -1.687339186668396, + "logps/chosen": -0.2868710160255432, + "logps/rejected": -0.28085970878601074, + "loss": 0.5253114700317383, + "loss/core": 0.5071266889572144, + "loss/preference_sft": 0.28473153710365295, + "loss/reference_anchor": 0.33522337675094604, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.94403076171875, + "rewards/margins": 2.089409351348877, + "rewards/rejected": 0.8546212315559387, + "ronpo/logit": 0.20894095301628113, + "ronpo/residual": 0.21501250565052032, + "ronpo/residual_abs": 0.251351535320282, + "ronpo/target": -0.006071551237255335, + "ronpo/target_abs": 0.0737655907869339, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 770 + }, + { + "drift/chosen_abs": 0.39996713399887085, + "drift/rejected_abs": 0.18100953102111816, + "epoch": 0.24632499006754072, + "grad_norm": 4.5625, + "learning_rate": 5.8526203570536504e-09, + "logits/chosen": -1.798459768295288, + "logits/rejected": -1.7976763248443604, + "logps/chosen": -0.26244863867759705, + "logps/rejected": -0.2599298357963562, + "loss": 0.3279760181903839, + "loss/core": 0.30524200201034546, + "loss/preference_sft": 0.26279181241989136, + "loss/reference_anchor": 0.4284016191959381, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.99951434135437, + "rewards/margins": 2.1924331188201904, + "rewards/rejected": 1.807080864906311, + "ronpo/logit": 0.21924333274364471, + "ronpo/residual": 0.22038576006889343, + "ronpo/residual_abs": 0.2682816684246063, + "ronpo/target": -0.0011424478143453598, + "ronpo/target_abs": 0.0722413957118988, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 775 + }, + { + "drift/chosen_abs": 0.1589912623167038, + "drift/rejected_abs": 0.09040515124797821, + "epoch": 0.24791418355184744, + "grad_norm": 1.2578125, + "learning_rate": 5.405737427476853e-09, + "logits/chosen": -1.9486103057861328, + "logits/rejected": -1.923584222793579, + "logps/chosen": -0.2780265510082245, + "logps/rejected": -0.27566081285476685, + "loss": 0.03590798377990723, + "loss/core": 0.030683238059282303, + "loss/preference_sft": 0.27917927503585815, + "loss/reference_anchor": 0.07657701522111893, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.58791184425354, + "rewards/margins": 0.684416651725769, + "rewards/rejected": 0.9034950137138367, + "ronpo/logit": 0.06844166666269302, + "ronpo/residual": 0.07022113353013992, + "ronpo/residual_abs": 0.12148270756006241, + "ronpo/target": -0.0017794653540477157, + "ronpo/target_abs": 0.07558373361825943, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 780 + }, + { + "drift/chosen_abs": 0.21656222641468048, + "drift/rejected_abs": 0.09968177229166031, + "epoch": 0.24950337703615416, + "grad_norm": 5.84375, + "learning_rate": 4.975624576281889e-09, + "logits/chosen": -1.7334492206573486, + "logits/rejected": -1.745598554611206, + "logps/chosen": -0.2670922577381134, + "logps/rejected": -0.28659647703170776, + "loss": 0.13521121442317963, + "loss/core": 0.12704432010650635, + "loss/preference_sft": 0.2725936770439148, + "loss/reference_anchor": 0.1360786259174347, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.1648879051208496, + "rewards/margins": 1.1697208881378174, + "rewards/rejected": 0.9951672554016113, + "ronpo/logit": 0.11697210371494293, + "ronpo/residual": 0.13287900388240814, + "ronpo/residual_abs": 0.17512091994285583, + "ronpo/target": -0.015906909480690956, + "ronpo/target_abs": 0.08019174635410309, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 785 + }, + { + "drift/chosen_abs": 0.27706044912338257, + "drift/rejected_abs": 0.08440014719963074, + "epoch": 0.25109257052046086, + "grad_norm": 145.0, + "learning_rate": 4.562443551321787e-09, + "logits/chosen": -1.738996148109436, + "logits/rejected": -1.7702322006225586, + "logps/chosen": -0.2960914671421051, + "logps/rejected": -0.2777342200279236, + "loss": 0.7593822479248047, + "loss/core": 0.7336170077323914, + "loss/preference_sft": 0.3006787896156311, + "loss/reference_anchor": 0.48523634672164917, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.7688331604003906, + "rewards/margins": 1.9260284900665283, + "rewards/rejected": 0.8428047895431519, + "ronpo/logit": 0.19260284304618835, + "ronpo/residual": 0.18357378244400024, + "ronpo/residual_abs": 0.22910535335540771, + "ronpo/target": 0.009029053151607513, + "ronpo/target_abs": 0.0732058733701706, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 790 + }, + { + "drift/chosen_abs": 0.23308022320270538, + "drift/rejected_abs": 0.11013714224100113, + "epoch": 0.2526817640047676, + "grad_norm": 3.625, + "learning_rate": 4.166349733082153e-09, + "logits/chosen": -1.8755409717559814, + "logits/rejected": -1.8500206470489502, + "logps/chosen": -0.2593986392021179, + "logps/rejected": -0.2735549807548523, + "loss": 0.09857200086116791, + "loss/core": 0.09173043817281723, + "loss/preference_sft": 0.2679574191570282, + "loss/reference_anchor": 0.11003559827804565, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.330761671066284, + "rewards/margins": 1.2306572198867798, + "rewards/rejected": 1.1001043319702148, + "ronpo/logit": 0.12306573241949081, + "ronpo/residual": 0.12092306464910507, + "ronpo/residual_abs": 0.1728290170431137, + "ronpo/target": 0.002142664510756731, + "ronpo/target_abs": 0.08366601169109344, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 795 + }, + { + "drift/chosen_abs": 0.30347901582717896, + "drift/rejected_abs": 0.14543643593788147, + "epoch": 0.2542709574890743, + "grad_norm": 3.921875, + "learning_rate": 3.7874920762489935e-09, + "logits/chosen": -1.8720595836639404, + "logits/rejected": -1.7962238788604736, + "logps/chosen": -0.28576621413230896, + "logps/rejected": -0.2803238034248352, + "loss": 0.264794260263443, + "loss/core": 0.2447708547115326, + "loss/preference_sft": 0.28448182344436646, + "loss/reference_anchor": 0.37201887369155884, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.034790515899658, + "rewards/margins": 1.5887119770050049, + "rewards/rejected": 1.4460780620574951, + "ronpo/logit": 0.15887121856212616, + "ronpo/residual": 0.1484646499156952, + "ronpo/residual_abs": 0.22516998648643494, + "ronpo/target": 0.010406563989818096, + "ronpo/target_abs": 0.07933439314365387, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 800 + }, + { + "drift/chosen_abs": 0.2623865604400635, + "drift/rejected_abs": 0.13297392427921295, + "epoch": 0.255860150973381, + "grad_norm": 6.15625, + "learning_rate": 3.426013053692878e-09, + "logits/chosen": -1.765254020690918, + "logits/rejected": -1.6951442956924438, + "logps/chosen": -0.25918513536453247, + "logps/rejected": -0.2669827938079834, + "loss": 0.13117735087871552, + "loss/core": 0.1169406920671463, + "loss/preference_sft": 0.26121383905410767, + "loss/reference_anchor": 0.25861161947250366, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.6238656044006348, + "rewards/margins": 1.2945048809051514, + "rewards/rejected": 1.3293604850769043, + "ronpo/logit": 0.1294504851102829, + "ronpo/residual": 0.1414770781993866, + "ronpo/residual_abs": 0.17369873821735382, + "ronpo/target": -0.012026568874716759, + "ronpo/target_abs": 0.07064296305179596, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 805 + }, + { + "drift/chosen_abs": 0.4997061789035797, + "drift/rejected_abs": 0.19101104140281677, + "epoch": 0.25744934445768775, + "grad_norm": 832.0, + "learning_rate": 3.0820486028908077e-09, + "logits/chosen": -1.7124788761138916, + "logits/rejected": -1.7720760107040405, + "logps/chosen": -0.25494056940078735, + "logps/rejected": -0.24831128120422363, + "loss": 0.960421085357666, + "loss/core": 0.9222633242607117, + "loss/preference_sft": 0.26125359535217285, + "loss/reference_anchor": 0.737030029296875, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 4.997061729431152, + "rewards/margins": 3.0881500244140625, + "rewards/rejected": 1.9089114665985107, + "ronpo/logit": 0.30881503224372864, + "ronpo/residual": 0.30170923471450806, + "ronpo/residual_abs": 0.3625412583351135, + "ronpo/target": 0.007105761673301458, + "ronpo/target_abs": 0.06471297889947891, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 810 + }, + { + "drift/chosen_abs": 0.3279181122779846, + "drift/rejected_abs": 0.13609564304351807, + "epoch": 0.25903853794199444, + "grad_norm": 1504.0, + "learning_rate": 2.7557280748055966e-09, + "logits/chosen": -1.8881208896636963, + "logits/rejected": -1.9525415897369385, + "logps/chosen": -0.2766019403934479, + "logps/rejected": -0.2734753489494324, + "loss": 0.871407687664032, + "loss/core": 0.8373653292655945, + "loss/preference_sft": 0.2795865535736084, + "loss/reference_anchor": 0.6528871655464172, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.26275634765625, + "rewards/margins": 1.9969438314437866, + "rewards/rejected": 1.2658125162124634, + "ronpo/logit": 0.19969438016414642, + "ronpo/residual": 0.20564095675945282, + "ronpo/residual_abs": 0.27914053201675415, + "ronpo/target": -0.005946573801338673, + "ronpo/target_abs": 0.07861017435789108, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 815 + }, + { + "drift/chosen_abs": 0.12528498470783234, + "drift/rejected_abs": 0.08170704543590546, + "epoch": 0.26062773142630113, + "grad_norm": 2.03125, + "learning_rate": 2.4471741852423233e-09, + "logits/chosen": -2.0222911834716797, + "logits/rejected": -2.054588556289673, + "logps/chosen": -0.2888307273387909, + "logps/rejected": -0.2907455861568451, + "loss": 0.06904357671737671, + "loss/core": 0.06414641439914703, + "loss/preference_sft": 0.2814570367336273, + "loss/reference_anchor": 0.06979776173830032, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.252849817276001, + "rewards/margins": 0.436107337474823, + "rewards/rejected": 0.8167423009872437, + "ronpo/logit": 0.04361073672771454, + "ronpo/residual": 0.04019293561577797, + "ronpo/residual_abs": 0.13318975269794464, + "ronpo/target": 0.0034177973866462708, + "ronpo/target_abs": 0.07740087062120438, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 820 + }, + { + "drift/chosen_abs": 0.1749141365289688, + "drift/rejected_abs": 0.09744749963283539, + "epoch": 0.2622169249106079, + "grad_norm": 4.125, + "learning_rate": 2.1565029686999305e-09, + "logits/chosen": -2.02028226852417, + "logits/rejected": -1.9485108852386475, + "logps/chosen": -0.29039934277534485, + "logps/rejected": -0.30477070808410645, + "loss": 0.08298453688621521, + "loss/core": 0.07695572078227997, + "loss/preference_sft": 0.30129221081733704, + "loss/reference_anchor": 0.09044675529003143, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7491413354873657, + "rewards/margins": 0.7976366281509399, + "rewards/rejected": 0.9515047073364258, + "ronpo/logit": 0.07976364344358444, + "ronpo/residual": 0.09047956764698029, + "ronpo/residual_abs": 0.13659612834453583, + "ronpo/target": -0.010715911164879799, + "ronpo/target_abs": 0.06951193511486053, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 825 + }, + { + "drift/chosen_abs": 0.39292028546333313, + "drift/rejected_abs": 0.15564313530921936, + "epoch": 0.2638061183949146, + "grad_norm": 51.75, + "learning_rate": 1.8838237347353845e-09, + "logits/chosen": -1.829711675643921, + "logits/rejected": -1.8036315441131592, + "logps/chosen": -0.28607892990112305, + "logps/rejected": -0.2774202823638916, + "loss": 1.1372624635696411, + "loss/core": 1.0995932817459106, + "loss/preference_sft": 0.2805403172969818, + "loss/reference_anchor": 0.7253289222717285, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.927274703979492, + "rewards/margins": 2.3715097904205322, + "rewards/rejected": 1.5557647943496704, + "ronpo/logit": 0.23715098202228546, + "ronpo/residual": 0.2534126341342926, + "ronpo/residual_abs": 0.40109115839004517, + "ronpo/target": -0.01626167818903923, + "ronpo/target_abs": 0.08010418713092804, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 830 + }, + { + "drift/chosen_abs": 0.11015255749225616, + "drift/rejected_abs": 0.08351143449544907, + "epoch": 0.2653953118792213, + "grad_norm": 3.609375, + "learning_rate": 1.6292390268568101e-09, + "logits/chosen": -1.9036916494369507, + "logits/rejected": -1.8151781558990479, + "logps/chosen": -0.388921856880188, + "logps/rejected": -0.2949334681034088, + "loss": 0.0698942169547081, + "loss/core": 0.0649130567908287, + "loss/preference_sft": 0.30027008056640625, + "loss/reference_anchor": 0.06959625333547592, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.099797010421753, + "rewards/margins": 0.264737993478775, + "rewards/rejected": 0.8350589871406555, + "ronpo/logit": 0.02647380903363228, + "ronpo/residual": 0.028390701860189438, + "ronpo/residual_abs": 0.12346197664737701, + "ronpo/target": -0.0019168935250490904, + "ronpo/target_abs": 0.07617279142141342, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 835 + }, + { + "drift/chosen_abs": 0.2376348078250885, + "drift/rejected_abs": 0.11902894079685211, + "epoch": 0.266984505363528, + "grad_norm": 260.0, + "learning_rate": 1.392844583961078e-09, + "logits/chosen": -1.9641727209091187, + "logits/rejected": -1.9484821557998657, + "logps/chosen": -0.29593294858932495, + "logps/rejected": -0.29610633850097656, + "loss": 0.33072429895401, + "loss/core": 0.30467796325683594, + "loss/preference_sft": 0.291830450296402, + "loss/reference_anchor": 0.49174419045448303, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.3760135173797607, + "rewards/margins": 1.187029480934143, + "rewards/rejected": 1.1889841556549072, + "ronpo/logit": 0.1187029629945755, + "ronpo/residual": 0.14209163188934326, + "ronpo/residual_abs": 0.18299010396003723, + "ronpo/target": -0.023388683795928955, + "ronpo/target_abs": 0.0768231675028801, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 840 + }, + { + "drift/chosen_abs": 0.1240852028131485, + "drift/rejected_abs": 0.0827094167470932, + "epoch": 0.2685736988478347, + "grad_norm": 1.6953125, + "learning_rate": 1.1747293043303518e-09, + "logits/chosen": -1.8987678289413452, + "logits/rejected": -1.8802201747894287, + "logps/chosen": -0.30965709686279297, + "logps/rejected": -0.2994539141654968, + "loss": 0.02463197335600853, + "loss/core": 0.02076641097664833, + "loss/preference_sft": 0.3030204176902771, + "loss/reference_anchor": 0.04700922966003418, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.2399028539657593, + "rewards/margins": 0.4139002859592438, + "rewards/rejected": 0.8260027170181274, + "ronpo/logit": 0.041390031576156616, + "ronpo/residual": 0.043523531407117844, + "ronpo/residual_abs": 0.10440732538700104, + "ronpo/target": -0.002133502857759595, + "ronpo/target_abs": 0.072882741689682, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 845 + }, + { + "drift/chosen_abs": 0.2770349085330963, + "drift/rejected_abs": 0.1225806251168251, + "epoch": 0.2701628923321414, + "grad_norm": 59.0, + "learning_rate": 9.749752122010346e-10, + "logits/chosen": -1.6999680995941162, + "logits/rejected": -1.6244916915893555, + "logps/chosen": -0.28973299264907837, + "logps/rejected": -0.3002259433269501, + "loss": 0.36320897936820984, + "loss/core": 0.34727710485458374, + "loss/preference_sft": 0.2909989058971405, + "loss/reference_anchor": 0.2895377278327942, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.769200086593628, + "rewards/margins": 1.6400721073150635, + "rewards/rejected": 1.1291282176971436, + "ronpo/logit": 0.16400720179080963, + "ronpo/residual": 0.15951409935951233, + "ronpo/residual_abs": 0.21055129170417786, + "ronpo/target": 0.004493103362619877, + "ronpo/target_abs": 0.07943262159824371, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 850 + }, + { + "drift/chosen_abs": 0.32434701919555664, + "drift/rejected_abs": 0.10581743717193604, + "epoch": 0.27175208581644816, + "grad_norm": 74.5, + "learning_rate": 7.936574269178375e-10, + "logits/chosen": -1.7718331813812256, + "logits/rejected": -1.7748457193374634, + "logps/chosen": -0.3256663680076599, + "logps/rejected": -0.32961875200271606, + "loss": 0.8090022206306458, + "loss/core": 0.7803620100021362, + "loss/preference_sft": 0.33879441022872925, + "loss/reference_anchor": 0.538924515247345, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.241380214691162, + "rewards/margins": 2.1839098930358887, + "rewards/rejected": 1.0574705600738525, + "ronpo/logit": 0.21839098632335663, + "ronpo/residual": 0.23008199036121368, + "ronpo/residual_abs": 0.28760650753974915, + "ronpo/target": -0.011690998449921608, + "ronpo/target_abs": 0.09034421294927597, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 855 + }, + { + "drift/chosen_abs": 0.3449726402759552, + "drift/rejected_abs": 0.1498272866010666, + "epoch": 0.27334127930075486, + "grad_norm": 190.0, + "learning_rate": 6.308441346844384e-10, + "logits/chosen": -1.9813814163208008, + "logits/rejected": -1.9928051233291626, + "logps/chosen": -0.28107166290283203, + "logps/rejected": -0.2793976366519928, + "loss": 0.5188241004943848, + "loss/core": 0.4884120523929596, + "loss/preference_sft": 0.28123125433921814, + "loss/reference_anchor": 0.5801182985305786, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.4484634399414062, + "rewards/margins": 1.9525508880615234, + "rewards/rejected": 1.4959125518798828, + "ronpo/logit": 0.1952550858259201, + "ronpo/residual": 0.2171076238155365, + "ronpo/residual_abs": 0.2710626721382141, + "ronpo/target": -0.02185254916548729, + "ronpo/target_abs": 0.0946681946516037, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 860 + }, + { + "drift/chosen_abs": 0.34945374727249146, + "drift/rejected_abs": 0.13005147874355316, + "epoch": 0.2749304727850616, + "grad_norm": 68.5, + "learning_rate": 4.865965629214819e-10, + "logits/chosen": -1.6880762577056885, + "logits/rejected": -1.679848074913025, + "logps/chosen": -0.28330764174461365, + "logps/rejected": -0.28511470556259155, + "loss": 0.8294771909713745, + "loss/core": 0.7987702488899231, + "loss/preference_sft": 0.28860875964164734, + "loss/reference_anchor": 0.5852810740470886, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.494537353515625, + "rewards/margins": 2.194039821624756, + "rewards/rejected": 1.3004978895187378, + "ronpo/logit": 0.2194039523601532, + "ronpo/residual": 0.21074934303760529, + "ronpo/residual_abs": 0.2737117409706116, + "ronpo/target": 0.008654648438096046, + "ronpo/target_abs": 0.09077721834182739, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 865 + }, + { + "drift/chosen_abs": 0.2341894656419754, + "drift/rejected_abs": 0.08986641466617584, + "epoch": 0.2765196662693683, + "grad_norm": 10.375, + "learning_rate": 3.609689572414143e-10, + "logits/chosen": -1.6340396404266357, + "logits/rejected": -1.6042693853378296, + "logps/chosen": -0.3055315911769867, + "logps/rejected": -0.30593135952949524, + "loss": 0.21940772235393524, + "loss/core": 0.20771245658397675, + "loss/preference_sft": 0.3087262213230133, + "loss/reference_anchor": 0.20303305983543396, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.340512990951538, + "rewards/margins": 1.4420377016067505, + "rewards/rejected": 0.8984752893447876, + "ronpo/logit": 0.14420375227928162, + "ronpo/residual": 0.12460936605930328, + "ronpo/residual_abs": 0.19094470143318176, + "ronpo/target": 0.019594397395849228, + "ronpo/target_abs": 0.06982195377349854, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 870 + }, + { + "drift/chosen_abs": 0.3996649384498596, + "drift/rejected_abs": 0.1653774082660675, + "epoch": 0.278108859753675, + "grad_norm": 3.75, + "learning_rate": 2.5400856104894063e-10, + "logits/chosen": -1.6849534511566162, + "logits/rejected": -1.678696870803833, + "logps/chosen": -0.2519184947013855, + "logps/rejected": -0.2652212083339691, + "loss": 1.347104787826538, + "loss/core": 1.2939437627792358, + "loss/preference_sft": 0.254986047744751, + "loss/reference_anchor": 1.0377271175384521, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.9966492652893066, + "rewards/margins": 2.35422420501709, + "rewards/rejected": 1.6424251794815063, + "ronpo/logit": 0.23542241752147675, + "ronpo/residual": 0.24257414042949677, + "ronpo/residual_abs": 0.3515390455722809, + "ronpo/target": -0.0071517289616167545, + "ronpo/target_abs": 0.07295659929513931, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 875 + }, + { + "drift/chosen_abs": 0.2877359986305237, + "drift/rejected_abs": 0.21151041984558105, + "epoch": 0.27969805323798175, + "grad_norm": 52.0, + "learning_rate": 1.6575559777469716e-10, + "logits/chosen": -1.8132593631744385, + "logits/rejected": -1.8345884084701538, + "logps/chosen": -0.27794936299324036, + "logps/rejected": -0.29504913091659546, + "loss": 0.7968089580535889, + "loss/core": 0.7706011533737183, + "loss/preference_sft": 0.2765906751155853, + "loss/reference_anchor": 0.49649912118911743, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.8772411346435547, + "rewards/margins": 0.7623518109321594, + "rewards/rejected": 2.11488938331604, + "ronpo/logit": 0.07623519748449326, + "ronpo/residual": 0.08102894574403763, + "ronpo/residual_abs": 0.29983019828796387, + "ronpo/target": -0.004793758969753981, + "ronpo/target_abs": 0.08000515401363373, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 880 + }, + { + "drift/chosen_abs": 0.3133094906806946, + "drift/rejected_abs": 0.1380651891231537, + "epoch": 0.28128724672228844, + "grad_norm": 4.8125, + "learning_rate": 9.624325574890125e-11, + "logits/chosen": -1.596097707748413, + "logits/rejected": -1.5967873334884644, + "logps/chosen": -0.3078146278858185, + "logps/rejected": -0.31876230239868164, + "loss": 0.26423799991607666, + "loss/core": 0.2500300407409668, + "loss/preference_sft": 0.311987042427063, + "loss/reference_anchor": 0.2529599070549011, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.131915330886841, + "rewards/margins": 1.7639577388763428, + "rewards/rejected": 1.367957353591919, + "ronpo/logit": 0.17639580368995667, + "ronpo/residual": 0.17563918232917786, + "ronpo/residual_abs": 0.22658184170722961, + "ronpo/target": 0.0007566086715087295, + "ronpo/target_abs": 0.0707857608795166, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 885 + }, + { + "drift/chosen_abs": 0.10474882274866104, + "drift/rejected_abs": 0.07787434756755829, + "epoch": 0.28287644020659514, + "grad_norm": 0.73828125, + "learning_rate": 4.5497675720540527e-11, + "logits/chosen": -1.9180591106414795, + "logits/rejected": -1.8720309734344482, + "logps/chosen": -0.32056906819343567, + "logps/rejected": -0.31805619597435, + "loss": 0.020853275433182716, + "loss/core": 0.01702974922955036, + "loss/preference_sft": 0.31593579053878784, + "loss/reference_anchor": 0.04487697035074234, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.0474880933761597, + "rewards/margins": 0.2687447667121887, + "rewards/rejected": 0.7787435054779053, + "ronpo/logit": 0.026874477043747902, + "ronpo/residual": 0.041672371327877045, + "ronpo/residual_abs": 0.09372963011264801, + "ronpo/target": -0.014797890558838844, + "ronpo/target_abs": 0.07123903930187225, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 890 + }, + { + "drift/chosen_abs": 0.4191473424434662, + "drift/rejected_abs": 0.1661919206380844, + "epoch": 0.2844656336909019, + "grad_norm": 61.25, + "learning_rate": 1.3537941026914302e-11, + "logits/chosen": -1.8258793354034424, + "logits/rejected": -1.7830054759979248, + "logps/chosen": -0.27327242493629456, + "logps/rejected": -0.28399670124053955, + "loss": 0.7437120079994202, + "loss/core": 0.7049421668052673, + "loss/preference_sft": 0.27699244022369385, + "loss/reference_anchor": 0.7476966381072998, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 4.191155433654785, + "rewards/margins": 2.578888177871704, + "rewards/rejected": 1.6122677326202393, + "ronpo/logit": 0.2578887939453125, + "ronpo/residual": 0.2486531287431717, + "ronpo/residual_abs": 0.30036187171936035, + "ronpo/target": 0.009235644713044167, + "ronpo/target_abs": 0.07194842398166656, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 895 + }, + { + "drift/chosen_abs": 0.18127210438251495, + "drift/rejected_abs": 0.08103598654270172, + "epoch": 0.2860548271752086, + "grad_norm": 23.25, + "learning_rate": 3.7607041719622813e-13, + "logits/chosen": -1.8266232013702393, + "logits/rejected": -1.9079567193984985, + "logps/chosen": -0.2764757573604584, + "logps/rejected": -0.2799293100833893, + "loss": 0.06943485885858536, + "loss/core": 0.06367450952529907, + "loss/preference_sft": 0.28113025426864624, + "loss/reference_anchor": 0.08709382265806198, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.8119440078735352, + "rewards/margins": 1.002147912979126, + "rewards/rejected": 0.8097962141036987, + "ronpo/logit": 0.10021479427814484, + "ronpo/residual": 0.08235163241624832, + "ronpo/residual_abs": 0.14560458064079285, + "ronpo/target": 0.017863161861896515, + "ronpo/target_abs": 0.07489818334579468, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 900 + }, + { + "epoch": 0.2860548271752086, + "step": 900, + "total_flos": 0.0, + "train_loss": 0.48261189058423043, + "train_runtime": 8509.445, + "train_samples_per_second": 1.692, + "train_steps_per_second": 0.106 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..07514f8 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64a55dd3f02e27240f2eb131eec514778f8ff0418ca1c52da6b44830d5ddb084 +size 7057