From ec51c3f261704ac3debef1b68b38eff8ce80e2ba Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sun, 26 Jul 2026 05:04:09 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: promotion/qwen3-8b-aaai27-flagship-ronpo-full-expect-s44 Source: Original Platform --- .gitattributes | 36 + README.md | 24 + all_results.json | 9 + chat_template.jinja | 89 + config.json | 71 + config.yaml | 59 + generation_config.json | 12 + job_status.json | 23 + model.safetensors | 3 + objective_protocol.json | 69 + pair_manifest.json | 34 + provenance.json | 12 + run_status.json | 13 + split_manifest.json | 23 + stability_gate.json | 38 + tokenizer.json | 3 + tokenizer_config.json | 30 + train_results.json | 9 + trainer_state.json | 4903 +++++++++++++++++++++++++++++++++++++++ training_args.bin | 3 + 20 files changed, 5463 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 config.yaml create mode 100644 generation_config.json create mode 100644 job_status.json create mode 100644 model.safetensors create mode 100644 objective_protocol.json create mode 100644 pair_manifest.json create mode 100644 provenance.json create mode 100644 run_status.json create mode 100644 split_manifest.json create mode 100644 stability_gate.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..17878c7 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +--- +library_name: transformers +base_model: Qwen/Qwen3-8B +tags: +- ronpo +- mnpo +- preference-optimization +--- + +# qwen3-8b-aaai27-flagship-ronpo-full-expect-s44 + +Research checkpoint for the RONPO AAAI revision experiments. + +- Method: ronpo_full_expect +- Base model: `Qwen/Qwen3-8B` +- Seed: 44 +- Local source at upload time: `/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/ronpo_full_expect/seed44/attempt1` +- Uploaded at UTC: 2026-07-12T17:19:00Z +- Run status metadata: `{"attempt": 1, "effective_batch_size": 16, "method": "ronpo_full_expect", "objective_protocol": "objective_protocol.json", "optimizer_steps": 900, "seed": 44, "split_manifest": "split_manifest.json", "stability_gate": "stability_gate.json", "status": "completed", "wandb_run_id": "e7dcaa92b396", "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/e7dcaa92b396"}` + +AAAI-27 P1 matched budget: 900 optimizer steps, effective batch 16; passed non-thinking and collapse stability gates; sealed-test metrics are not used for checkpoint selection. + +Intended use: reproducibility and evaluation for the RONPO paper. This +checkpoint is not intended as a production assistant. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..6e871d2 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.2860548271752086, + "total_flos": 0.0, + "train_loss": 0.48311558212671013, + "train_runtime": 8488.8129, + "train_samples": 50340, + "train_samples_per_second": 1.696, + "train_steps_per_second": 0.106 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..c2965a7 --- /dev/null +++ b/config.yaml @@ -0,0 +1,59 @@ +model_name_or_path: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218 +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + ? /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/ronpo_full_expect + : 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: ronpo +eta: 0.0075 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.05 +preference_sft_weight: 0.005 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 1.0e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 44 +gradient_accumulation_steps: 8 +gradient_checkpointing: false +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 2 +per_device_eval_batch_size: 2 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/ronpo_full_expect/seed44/attempt1 +run_name: aaai27-flagship-full-ronpo_full_expect-s44-a1 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/job_status.json b/job_status.json new file mode 100644 index 0000000..6657233 --- /dev/null +++ b/job_status.json @@ -0,0 +1,23 @@ +{ + "status": "completed", + "stage": "full", + "method": "ronpo_full_expect", + "seed": 44, + "attempt": 1, + "gpu": 2, + "gpus": [ + 2 + ], + "returncode": 0, + "finite_metrics": true, + "oom": false, + "traceback": false, + "reason": "passed", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "e7dcaa92b396", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/e7dcaa92b396", + "output_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/ronpo_full_expect/seed44/attempt1", + "log": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/logs/full_ronpo_full_expect_s44_a1.log", + "completed_at": "2026-07-12T17:12:08Z" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..5bcaf7c --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b392f3d3d84cebb436a3edc2d2e895a12201bccf234a22592f7a918f8ae2c77a +size 16381517208 diff --git a/objective_protocol.json b/objective_protocol.json new file mode 100644 index 0000000..35b3b44 --- /dev/null +++ b/objective_protocol.json @@ -0,0 +1,69 @@ +{ + "schema_version": 1, + "frozen_at_kst": "2026-07-12T21:40:00+09:00", + "base_model": "Qwen/Qwen3-8B", + "primary_triple": { + "helpfulness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "ultrafeedback-helpfulness", + "transform": "identity" + }, + "safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "conciseness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "helpsteer-verbosity", + "transform": "negate" + } + }, + "gate_thresholds": { + "raw_score_std_strictly_greater_than": 0.01, + "aggregate_prompt_pair_spearman_median_strictly_less_than": 0.0, + "aggregate_pairwise_top1_mismatch_at_least": 0.5 + }, + "gate_pool": "source-train Qwen3-8B response pool split into deterministic train/validation; sealed source-test excluded", + "training_seeds": [42, 43, 44], + "optimizer_steps": 900, + "effective_batch_size": 16, + "decode": { + "enable_thinking": false, + "temperature": 0.7, + "top_p": 0.9, + "max_new_tokens": 2048, + "decode_seed": 42 + }, + "methods": [ + "ronpo_full_expect", + "ronpo_k_only", + "dpo", + "ipo", + "simpo", + "kto", + "sppo_avg", + "inpo_avg", + "ht_mnpo_helpfulness", + "ht_mnpo_safety", + "ht_mnpo_conciseness" + ], + "selection_policy": "No sealed-test selection. One preregistered configuration per method; failures use only the declared stability retry recipe, at most three attempts.", + "stability_gate": { + "validation_prompts": 128, + "min_candidate_base_mean_word_ratio": 0.33, + "max_candidate_base_mean_word_ratio": 2.0, + "max_consecutive_identical_word_run": 20, + "think_tag_count": 0, + "empty_response_count": 0, + "retry_recipe": "attempt 2 halves learning rate, enables gradient checkpointing, and adds reference=0.02/SFT=0.002 anchoring to unanchored unified losses; attempt 3 quarters learning rate and uses reference=0.05/SFT=0.005; RONPO uses reference=0.05/SFT=0.005 from attempt 1" + }, + "primary_metric": "mean prompt-level minimum of per-prompt min-max-normalized helpfulness, safety, conciseness", + "bootstrap": { + "unit": "prompt", + "paired": true, + "resamples": 2000, + "interval": "percentile_95", + "seed": 42 + } +} diff --git a/pair_manifest.json b/pair_manifest.json new file mode 100644 index 0000000..be034b0 --- /dev/null +++ b/pair_manifest.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "objectives": [ + "helpfulness", + "safety", + "conciseness" + ], + "normalization": "per_prompt_minmax", + "adversary": { + "steps": 25, + "alpha": 1.0, + "kappa": 0.05, + "preference_scale": 8.0 + }, + "ronpo_estimator": "Rao-Blackwellized full sigma expectation with three common response pairs per prompt", + "counts": { + "train": { + "avg": 16746, + "ronpo_full_expect": 50340, + "ronpo_k_only": 50340, + "ht_mnpo_helpfulness": 16743, + "ht_mnpo_safety": 16764, + "ht_mnpo_conciseness": 16755 + }, + "validation": { + "avg": 1859, + "ronpo_full_expect": 5592, + "ronpo_k_only": 5592, + "ht_mnpo_helpfulness": 1864, + "ht_mnpo_safety": 1862, + "ht_mnpo_conciseness": 1858 + } + } +} diff --git a/provenance.json b/provenance.json new file mode 100644 index 0000000..801c907 --- /dev/null +++ b/provenance.json @@ -0,0 +1,12 @@ +{ + "method": "ronpo_full_expect", + "seed": 44, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "e7dcaa92b396", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/e7dcaa92b396", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "stability_gate": "stability_gate.json" +} diff --git a/run_status.json b/run_status.json new file mode 100644 index 0000000..61c2cfd --- /dev/null +++ b/run_status.json @@ -0,0 +1,13 @@ +{ + "method": "ronpo_full_expect", + "seed": 44, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "e7dcaa92b396", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/e7dcaa92b396", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "stability_gate": "stability_gate.json", + "status": "completed" +} diff --git a/split_manifest.json b/split_manifest.json new file mode 100644 index 0000000..4fb96bd --- /dev/null +++ b/split_manifest.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "split_rule": "sort source-train prompts by sha256('flagship-v1||'+prompt_id); first round(10%) validation", + "validation_fraction": 0.1, + "source_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/avg_oracle", + "base_model": "Qwen/Qwen3-8B", + "counts": { + "train": 16780, + "validation": 1864, + "sealed_test": 604 + }, + "prompt_id_sha256": { + "train": "fcf6f2ea5a0b7bcd41179e6f5288f4d850b2d04bdb729da91907bd18bf733a69", + "validation": "a87ba6f519192164daa487b109a6547dd8cbde46a27a373d6da201ca2d061b8f", + "sealed_test": "b398ced6af6cf73d731e1a5f32a0631d8060323d51eda1e375695566bf5f1867" + }, + "overlaps": { + "train_validation": 0, + "train_sealed_test": 0, + "validation_sealed_test": 0 + }, + "sealed_test_policy": "prompts only until S4a; no objective scores or model generations used in S0-S3" +} diff --git a/stability_gate.json b/stability_gate.json new file mode 100644 index 0000000..dc691e4 --- /dev/null +++ b/stability_gate.json @@ -0,0 +1,38 @@ +{ + "status": "passed", + "passed": true, + "base": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 388.1484375, + "max_words": 1379, + "max_repeat_run": 4 + }, + "candidate": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 391.1171875, + "max_words": 1299, + "max_repeat_run": 3 + }, + "candidate_base_mean_word_ratio": 1.0076484914357025, + "thresholds": { + "min_length_ratio": 0.33, + "max_length_ratio": 2.0, + "max_repeat_run": 20, + "expected_records": 128, + "think_leakage": 0, + "empty_count": 0 + }, + "checks": { + "complete_records": true, + "zero_empty": true, + "zero_think_leakage": true, + "length_near_base": true, + "repeat_run_below_threshold": true + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ffd19d2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..6e871d2 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.2860548271752086, + "total_flos": 0.0, + "train_loss": 0.48311558212671013, + "train_runtime": 8488.8129, + "train_samples": 50340, + "train_samples_per_second": 1.696, + "train_steps_per_second": 0.106 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..1a292c4 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,4903 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.2860548271752086, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "drift/chosen_abs": 0.26923736929893494, + "drift/rejected_abs": 0.12245669215917587, + "epoch": 0.0015891934843067143, + "grad_norm": 6.15625, + "learning_rate": 4.444444444444444e-09, + "logits/chosen": -1.8292624950408936, + "logits/rejected": -1.8292913436889648, + "logps/chosen": -0.25974613428115845, + "logps/rejected": -0.2631612718105316, + "loss": 0.3697437345981598, + "loss/core": 0.35558179020881653, + "loss/preference_sft": 0.25867244601249695, + "loss/reference_anchor": 0.25737133622169495, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.6913039684295654, + "rewards/margins": 1.4922035932540894, + "rewards/rejected": 1.1991007328033447, + "ronpo/logit": 0.14922034740447998, + "ronpo/residual": 0.14157937467098236, + "ronpo/residual_abs": 0.20015816390514374, + "ronpo/target": 0.007640989031642675, + "ronpo/target_abs": 0.07512292265892029, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 5 + }, + { + "drift/chosen_abs": 0.3896196186542511, + "drift/rejected_abs": 0.18290038406848907, + "epoch": 0.0031783869686134287, + "grad_norm": 0.49609375, + "learning_rate": 1e-08, + "logits/chosen": -1.8097797632217407, + "logits/rejected": -1.7882839441299438, + "logps/chosen": -0.2950737774372101, + "logps/rejected": -0.3086322844028473, + "loss": 0.6527813673019409, + "loss/core": 0.6042948365211487, + "loss/preference_sft": 0.2980811893939972, + "loss/reference_anchor": 0.9399248361587524, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 3.8948261737823486, + "rewards/margins": 2.0671069622039795, + "rewards/rejected": 1.8277194499969482, + "ronpo/logit": 0.20671074092388153, + "ronpo/residual": 0.18510529398918152, + "ronpo/residual_abs": 0.2420695573091507, + "ronpo/target": 0.021605418995022774, + "ronpo/target_abs": 0.07878313958644867, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 10 + }, + { + "drift/chosen_abs": 0.3951796889305115, + "drift/rejected_abs": 0.09188535064458847, + "epoch": 0.004767580452920143, + "grad_norm": 0.96875, + "learning_rate": 1.5555555555555554e-08, + "logits/chosen": -1.7505464553833008, + "logits/rejected": -1.7822763919830322, + "logps/chosen": -0.29122844338417053, + "logps/rejected": -0.2962791323661804, + "loss": 1.0267797708511353, + "loss/core": 0.9949584007263184, + "loss/preference_sft": 0.28717607259750366, + "loss/reference_anchor": 0.6077091097831726, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.9513306617736816, + "rewards/margins": 3.043854236602783, + "rewards/rejected": 0.9074764251708984, + "ronpo/logit": 0.30438539385795593, + "ronpo/residual": 0.29969078302383423, + "ronpo/residual_abs": 0.35535189509391785, + "ronpo/target": 0.0046946206130087376, + "ronpo/target_abs": 0.07615546137094498, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 15 + }, + { + "drift/chosen_abs": 0.2791958451271057, + "drift/rejected_abs": 0.13730254769325256, + "epoch": 0.006356773937226857, + "grad_norm": 1.046875, + "learning_rate": 2.111111111111111e-08, + "logits/chosen": -1.9703651666641235, + "logits/rejected": -2.017350196838379, + "logps/chosen": -0.28874635696411133, + "logps/rejected": -0.2909610867500305, + "loss": 0.41066980361938477, + "loss/core": 0.3952859938144684, + "loss/preference_sft": 0.3018035292625427, + "loss/reference_anchor": 0.27749544382095337, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.7919583320617676, + "rewards/margins": 1.4557034969329834, + "rewards/rejected": 1.3362548351287842, + "ronpo/logit": 0.14557036757469177, + "ronpo/residual": 0.14364346861839294, + "ronpo/residual_abs": 0.21150994300842285, + "ronpo/target": 0.001926881494000554, + "ronpo/target_abs": 0.0835525169968605, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 20 + }, + { + "drift/chosen_abs": 0.32657527923583984, + "drift/rejected_abs": 0.14058224856853485, + "epoch": 0.007945967421533572, + "grad_norm": 5.71875, + "learning_rate": 2.6666666666666664e-08, + "logits/chosen": -1.6839513778686523, + "logits/rejected": -1.7249832153320312, + "logps/chosen": -0.2955150306224823, + "logps/rejected": -0.28480157256126404, + "loss": 0.48044338822364807, + "loss/core": 0.44645896553993225, + "loss/preference_sft": 0.3023730218410492, + "loss/reference_anchor": 0.6494489312171936, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.2632546424865723, + "rewards/margins": 1.8599933385849, + "rewards/rejected": 1.4032608270645142, + "ronpo/logit": 0.18599936366081238, + "ronpo/residual": 0.18265409767627716, + "ronpo/residual_abs": 0.22560951113700867, + "ronpo/target": 0.0033452813513576984, + "ronpo/target_abs": 0.07430481910705566, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 25 + }, + { + "drift/chosen_abs": 0.37191981077194214, + "drift/rejected_abs": 0.11496677249670029, + "epoch": 0.009535160905840286, + "grad_norm": 408.0, + "learning_rate": 3.2222222222222224e-08, + "logits/chosen": -1.6651420593261719, + "logits/rejected": -1.615891695022583, + "logps/chosen": -0.2600121796131134, + "logps/rejected": -0.25948435068130493, + "loss": 1.2412470579147339, + "loss/core": 1.2034351825714111, + "loss/preference_sft": 0.2637467384338379, + "loss/reference_anchor": 0.7298651933670044, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.719198226928711, + "rewards/margins": 2.570418119430542, + "rewards/rejected": 1.148780107498169, + "ronpo/logit": 0.2570417821407318, + "ronpo/residual": 0.2595512270927429, + "ronpo/residual_abs": 0.322681725025177, + "ronpo/target": -0.0025094160810112953, + "ronpo/target_abs": 0.0828036218881607, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 30 + }, + { + "drift/chosen_abs": 0.2374390810728073, + "drift/rejected_abs": 0.06636892259120941, + "epoch": 0.011124354390147001, + "grad_norm": 498.0, + "learning_rate": 3.7777777777777774e-08, + "logits/chosen": -1.8363662958145142, + "logits/rejected": -1.8149330615997314, + "logps/chosen": -0.322407066822052, + "logps/rejected": -0.31593409180641174, + "loss": 1.0479040145874023, + "loss/core": 1.019943118095398, + "loss/preference_sft": 0.3177246153354645, + "loss/reference_anchor": 0.5274423360824585, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.3728532791137695, + "rewards/margins": 1.7133121490478516, + "rewards/rejected": 0.6595410108566284, + "ronpo/logit": 0.17133121192455292, + "ronpo/residual": 0.18483777344226837, + "ronpo/residual_abs": 0.2308831661939621, + "ronpo/target": -0.0135065708309412, + "ronpo/target_abs": 0.08307793736457825, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 35 + }, + { + "drift/chosen_abs": 0.31959348917007446, + "drift/rejected_abs": 0.12583120167255402, + "epoch": 0.012713547874453715, + "grad_norm": 0.640625, + "learning_rate": 4.333333333333333e-08, + "logits/chosen": -1.7055801153182983, + "logits/rejected": -1.7161920070648193, + "logps/chosen": -0.2816811800003052, + "logps/rejected": -0.285296767950058, + "loss": 0.585159182548523, + "loss/core": 0.5509465932846069, + "loss/preference_sft": 0.2811506390571594, + "loss/reference_anchor": 0.6561352014541626, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.1953861713409424, + "rewards/margins": 1.939779281616211, + "rewards/rejected": 1.2556068897247314, + "ronpo/logit": 0.19397792220115662, + "ronpo/residual": 0.183498352766037, + "ronpo/residual_abs": 0.23382291197776794, + "ronpo/target": 0.010479571297764778, + "ronpo/target_abs": 0.08148641884326935, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 40 + }, + { + "drift/chosen_abs": 0.342174232006073, + "drift/rejected_abs": 0.17132499814033508, + "epoch": 0.014302741358760428, + "grad_norm": 9.1875, + "learning_rate": 4.888888888888889e-08, + "logits/chosen": -1.8917038440704346, + "logits/rejected": -1.9098457098007202, + "logps/chosen": -0.26641327142715454, + "logps/rejected": -0.25877052545547485, + "loss": 0.20708735287189484, + "loss/core": 0.16268716752529144, + "loss/preference_sft": 0.26177817583084106, + "loss/reference_anchor": 0.8618253469467163, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.419778347015381, + "rewards/margins": 1.7149407863616943, + "rewards/rejected": 1.7048375606536865, + "ronpo/logit": 0.17149408161640167, + "ronpo/residual": 0.1707213819026947, + "ronpo/residual_abs": 0.22870823740959167, + "ronpo/target": 0.0007726888870820403, + "ronpo/target_abs": 0.08850932121276855, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 45 + }, + { + "drift/chosen_abs": 0.2557240426540375, + "drift/rejected_abs": 0.1332291066646576, + "epoch": 0.015891934843067144, + "grad_norm": 10.0625, + "learning_rate": 5.444444444444444e-08, + "logits/chosen": -1.7528432607650757, + "logits/rejected": -1.7131255865097046, + "logps/chosen": -0.25860145688056946, + "logps/rejected": -0.25767049193382263, + "loss": 0.14077365398406982, + "loss/core": 0.12432624399662018, + "loss/preference_sft": 0.2571946680545807, + "loss/reference_anchor": 0.30322855710983276, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.5567574501037598, + "rewards/margins": 1.230201005935669, + "rewards/rejected": 1.3265564441680908, + "ronpo/logit": 0.12302011251449585, + "ronpo/residual": 0.13287095725536346, + "ronpo/residual_abs": 0.17298848927021027, + "ronpo/target": -0.009850854054093361, + "ronpo/target_abs": 0.06360660493373871, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 50 + }, + { + "drift/chosen_abs": 0.186763733625412, + "drift/rejected_abs": 0.119536854326725, + "epoch": 0.01748112832737386, + "grad_norm": 2.65625, + "learning_rate": 6e-08, + "logits/chosen": -1.6005624532699585, + "logits/rejected": -1.5535829067230225, + "logps/chosen": -0.32774192094802856, + "logps/rejected": -0.3251161575317383, + "loss": 0.04407442361116409, + "loss/core": 0.034535981714725494, + "loss/preference_sft": 0.331603467464447, + "loss/reference_anchor": 0.15760840475559235, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.8662309646606445, + "rewards/margins": 0.6730316877365112, + "rewards/rejected": 1.1931991577148438, + "ronpo/logit": 0.06730317324399948, + "ronpo/residual": 0.05887698009610176, + "ronpo/residual_abs": 0.1300887018442154, + "ronpo/target": 0.008426194079220295, + "ronpo/target_abs": 0.0830519050359726, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 55 + }, + { + "drift/chosen_abs": 0.09217182546854019, + "drift/rejected_abs": 0.05582476407289505, + "epoch": 0.01907032181168057, + "grad_norm": 3.953125, + "learning_rate": 6.555555555555555e-08, + "logits/chosen": -2.019383668899536, + "logits/rejected": -2.0197396278381348, + "logps/chosen": -0.3089592456817627, + "logps/rejected": -0.30434226989746094, + "loss": 0.0220238845795393, + "loss/core": 0.019543960690498352, + "loss/preference_sft": 0.3074526786804199, + "loss/reference_anchor": 0.01885320618748665, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 0.9209924936294556, + "rewards/margins": 0.3633802831172943, + "rewards/rejected": 0.5576121211051941, + "ronpo/logit": 0.03633803129196167, + "ronpo/residual": 0.05237559229135513, + "ronpo/residual_abs": 0.1066572442650795, + "ronpo/target": -0.016037559136748314, + "ronpo/target_abs": 0.08774705976247787, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 60 + }, + { + "drift/chosen_abs": 0.38381367921829224, + "drift/rejected_abs": 0.1924135982990265, + "epoch": 0.020659515295987287, + "grad_norm": 241.0, + "learning_rate": 7.111111111111111e-08, + "logits/chosen": -1.7329849004745483, + "logits/rejected": -1.6644783020019531, + "logps/chosen": -0.2624332010746002, + "logps/rejected": -0.28014859557151794, + "loss": 0.7392578125, + "loss/core": 0.7066189646720886, + "loss/preference_sft": 0.26991426944732666, + "loss/reference_anchor": 0.6257854104042053, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.8338208198547363, + "rewards/margins": 1.931775689125061, + "rewards/rejected": 1.9020452499389648, + "ronpo/logit": 0.19317759573459625, + "ronpo/residual": 0.17849010229110718, + "ronpo/residual_abs": 0.30854010581970215, + "ronpo/target": 0.01468747854232788, + "ronpo/target_abs": 0.07874740660190582, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 65 + }, + { + "drift/chosen_abs": 0.3801648020744324, + "drift/rejected_abs": 0.17235013842582703, + "epoch": 0.022248708780294002, + "grad_norm": 3.765625, + "learning_rate": 7.666666666666666e-08, + "logits/chosen": -1.6759541034698486, + "logits/rejected": -1.7048391103744507, + "logps/chosen": -0.28707024455070496, + "logps/rejected": -0.29981210827827454, + "loss": 1.3195468187332153, + "loss/core": 1.2741187810897827, + "loss/preference_sft": 0.30114036798477173, + "loss/reference_anchor": 0.878444492816925, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.8014285564422607, + "rewards/margins": 2.0898962020874023, + "rewards/rejected": 1.7115325927734375, + "ronpo/logit": 0.20898965001106262, + "ronpo/residual": 0.22612443566322327, + "ronpo/residual_abs": 0.3519369661808014, + "ronpo/target": -0.017134791240096092, + "ronpo/target_abs": 0.07949066162109375, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 70 + }, + { + "drift/chosen_abs": 0.20290859043598175, + "drift/rejected_abs": 0.06069299578666687, + "epoch": 0.023837902264600714, + "grad_norm": 5.5, + "learning_rate": 8.222222222222221e-08, + "logits/chosen": -1.853276252746582, + "logits/rejected": -1.8313668966293335, + "logps/chosen": -0.2681577205657959, + "logps/rejected": -0.2761048674583435, + "loss": 0.13597606122493744, + "loss/core": 0.129965141415596, + "loss/preference_sft": 0.27459055185317993, + "loss/reference_anchor": 0.09275973588228226, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.0285112857818604, + "rewards/margins": 1.4334367513656616, + "rewards/rejected": 0.5950745344161987, + "ronpo/logit": 0.14334365725517273, + "ronpo/residual": 0.14809009432792664, + "ronpo/residual_abs": 0.19457709789276123, + "ronpo/target": -0.004746442195028067, + "ronpo/target_abs": 0.08164028823375702, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 75 + }, + { + "drift/chosen_abs": 0.19127903878688812, + "drift/rejected_abs": 0.10886410623788834, + "epoch": 0.02542709574890743, + "grad_norm": 13.8125, + "learning_rate": 8.777777777777778e-08, + "logits/chosen": -1.827038049697876, + "logits/rejected": -1.7601505517959595, + "logps/chosen": -0.302360475063324, + "logps/rejected": -0.29604336619377136, + "loss": 0.05623495578765869, + "loss/core": 0.049041394144296646, + "loss/preference_sft": 0.30537527799606323, + "loss/reference_anchor": 0.11333372443914413, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.9121793508529663, + "rewards/margins": 0.8244398236274719, + "rewards/rejected": 1.0877394676208496, + "ronpo/logit": 0.08244398236274719, + "ronpo/residual": 0.0879630595445633, + "ronpo/residual_abs": 0.13678863644599915, + "ronpo/target": -0.005519069265574217, + "ronpo/target_abs": 0.07422248274087906, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 80 + }, + { + "drift/chosen_abs": 0.337031751871109, + "drift/rejected_abs": 0.1565742790699005, + "epoch": 0.027016289233214145, + "grad_norm": 22.25, + "learning_rate": 9.333333333333334e-08, + "logits/chosen": -1.9177894592285156, + "logits/rejected": -1.9806658029556274, + "logps/chosen": -0.27497977018356323, + "logps/rejected": -0.27717286348342896, + "loss": 0.6743783354759216, + "loss/core": 0.6322578191757202, + "loss/preference_sft": 0.27909091114997864, + "loss/reference_anchor": 0.8145014047622681, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.369861125946045, + "rewards/margins": 1.80503249168396, + "rewards/rejected": 1.5648287534713745, + "ronpo/logit": 0.1805032342672348, + "ronpo/residual": 0.1641511768102646, + "ronpo/residual_abs": 0.22668591141700745, + "ronpo/target": 0.016352057456970215, + "ronpo/target_abs": 0.07268329709768295, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 85 + }, + { + "drift/chosen_abs": 0.22597479820251465, + "drift/rejected_abs": 0.10148023068904877, + "epoch": 0.028605482717520857, + "grad_norm": 4.1875, + "learning_rate": 9.888888888888889e-08, + "logits/chosen": -1.8360424041748047, + "logits/rejected": -1.9166383743286133, + "logps/chosen": -0.2628440856933594, + "logps/rejected": -0.2537669837474823, + "loss": 0.08646391332149506, + "loss/core": 0.08045277744531631, + "loss/preference_sft": 0.26596134901046753, + "loss/reference_anchor": 0.09362667798995972, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.257885456085205, + "rewards/margins": 1.2502182722091675, + "rewards/rejected": 1.0076671838760376, + "ronpo/logit": 0.125021830201149, + "ronpo/residual": 0.11178363859653473, + "ronpo/residual_abs": 0.1606568545103073, + "ronpo/target": 0.013238191604614258, + "ronpo/target_abs": 0.08457295596599579, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 90 + }, + { + "drift/chosen_abs": 0.22616279125213623, + "drift/rejected_abs": 0.08887867629528046, + "epoch": 0.030194676201827572, + "grad_norm": 0.3984375, + "learning_rate": 9.999398298646737e-08, + "logits/chosen": -1.8040069341659546, + "logits/rejected": -1.828992486000061, + "logps/chosen": -0.2888103127479553, + "logps/rejected": -0.28947386145591736, + "loss": 0.18194636702537537, + "loss/core": 0.17117659747600555, + "loss/preference_sft": 0.2951456904411316, + "loss/reference_anchor": 0.18588101863861084, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.2616281509399414, + "rewards/margins": 1.3749189376831055, + "rewards/rejected": 0.8867092132568359, + "ronpo/logit": 0.1374918818473816, + "ronpo/residual": 0.12481904029846191, + "ronpo/residual_abs": 0.1675949990749359, + "ronpo/target": 0.012672854587435722, + "ronpo/target_abs": 0.07328122109174728, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 95 + }, + { + "drift/chosen_abs": 0.3307506740093231, + "drift/rejected_abs": 0.16090306639671326, + "epoch": 0.03178386968613429, + "grad_norm": 92.5, + "learning_rate": 9.996954135095479e-08, + "logits/chosen": -1.7753002643585205, + "logits/rejected": -1.8347423076629639, + "logps/chosen": -0.2850727438926697, + "logps/rejected": -0.2890182435512543, + "loss": 0.7790960073471069, + "loss/core": 0.7361105680465698, + "loss/preference_sft": 0.2861769199371338, + "loss/reference_anchor": 0.8310915231704712, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.3066513538360596, + "rewards/margins": 1.7068488597869873, + "rewards/rejected": 1.5998027324676514, + "ronpo/logit": 0.17068488895893097, + "ronpo/residual": 0.18404145538806915, + "ronpo/residual_abs": 0.2643963396549225, + "ronpo/target": -0.013356572017073631, + "ronpo/target_abs": 0.06924636662006378, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 100 + }, + { + "drift/chosen_abs": 0.23268786072731018, + "drift/rejected_abs": 0.0795300155878067, + "epoch": 0.033373063170441, + "grad_norm": 12.375, + "learning_rate": 9.992630821454457e-08, + "logits/chosen": -1.9496657848358154, + "logits/rejected": -1.9096084833145142, + "logps/chosen": -0.2818007469177246, + "logps/rejected": -0.28615885972976685, + "loss": 0.29838305711746216, + "loss/core": 0.28360605239868164, + "loss/preference_sft": 0.28804826736450195, + "loss/reference_anchor": 0.2667350769042969, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.32478404045105, + "rewards/margins": 1.5362763404846191, + "rewards/rejected": 0.7885079383850098, + "ronpo/logit": 0.15362763404846191, + "ronpo/residual": 0.15497443079948425, + "ronpo/residual_abs": 0.20809760689735413, + "ronpo/target": -0.0013467819662764668, + "ronpo/target_abs": 0.06745518743991852, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 105 + }, + { + "drift/chosen_abs": 0.46602663397789, + "drift/rejected_abs": 0.2611501216888428, + "epoch": 0.03496225665474772, + "grad_norm": 936.0, + "learning_rate": 9.986429983545125e-08, + "logits/chosen": -1.918752670288086, + "logits/rejected": -1.8298696279525757, + "logps/chosen": -0.2680893540382385, + "logps/rejected": -0.3012104034423828, + "loss": 0.8778338432312012, + "loss/core": 0.8211013674736023, + "loss/preference_sft": 0.28689926862716675, + "loss/reference_anchor": 1.1059573888778687, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 4.65931510925293, + "rewards/margins": 2.0492444038391113, + "rewards/rejected": 2.6100707054138184, + "ronpo/logit": 0.20492446422576904, + "ronpo/residual": 0.19235225021839142, + "ronpo/residual_abs": 0.3394165635108948, + "ronpo/target": 0.01257222332060337, + "ronpo/target_abs": 0.0747305378317833, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 110 + }, + { + "drift/chosen_abs": 0.20914717018604279, + "drift/rejected_abs": 0.10763482004404068, + "epoch": 0.03655145013905443, + "grad_norm": 7.125, + "learning_rate": 9.978353953249021e-08, + "logits/chosen": -1.7422930002212524, + "logits/rejected": -1.7559515237808228, + "logps/chosen": -0.2856474220752716, + "logps/rejected": -0.288565993309021, + "loss": 0.07564763724803925, + "loss/core": 0.06711847335100174, + "loss/preference_sft": 0.2876007556915283, + "loss/reference_anchor": 0.14182327687740326, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.089381456375122, + "rewards/margins": 1.014510154724121, + "rewards/rejected": 1.074871301651001, + "ronpo/logit": 0.10145101696252823, + "ronpo/residual": 0.0855717882514, + "ronpo/residual_abs": 0.15696951746940613, + "ronpo/target": 0.015879224985837936, + "ronpo/target_abs": 0.08091199398040771, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 115 + }, + { + "drift/chosen_abs": 0.38106095790863037, + "drift/rejected_abs": 0.1474742889404297, + "epoch": 0.03814064362336114, + "grad_norm": 1.0546875, + "learning_rate": 9.968405767630856e-08, + "logits/chosen": -1.8890774250030518, + "logits/rejected": -1.897849440574646, + "logps/chosen": -0.27462321519851685, + "logps/rejected": -0.287124902009964, + "loss": 0.8557225465774536, + "loss/core": 0.8139694333076477, + "loss/preference_sft": 0.28747329115867615, + "loss/reference_anchor": 0.8063166737556458, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.8102378845214844, + "rewards/margins": 2.3369901180267334, + "rewards/rejected": 1.4732472896575928, + "ronpo/logit": 0.2336990088224411, + "ronpo/residual": 0.22560825943946838, + "ronpo/residual_abs": 0.2692892253398895, + "ronpo/target": 0.008090745657682419, + "ronpo/target_abs": 0.07093735784292221, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 120 + }, + { + "drift/chosen_abs": 0.23315230011940002, + "drift/rejected_abs": 0.05996812507510185, + "epoch": 0.03972983710766786, + "grad_norm": 12.25, + "learning_rate": 9.956589167796391e-08, + "logits/chosen": -1.9130750894546509, + "logits/rejected": -1.9361803531646729, + "logps/chosen": -0.2950516641139984, + "logps/rejected": -0.268522173166275, + "loss": 0.841772735118866, + "loss/core": 0.8178065419197083, + "loss/preference_sft": 0.2870258688926697, + "loss/reference_anchor": 0.45062151551246643, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.331399440765381, + "rewards/margins": 1.7358144521713257, + "rewards/rejected": 0.5955848097801208, + "ronpo/logit": 0.17358146607875824, + "ronpo/residual": 0.18443605303764343, + "ronpo/residual_abs": 0.2259272038936615, + "ronpo/target": -0.010854599997401237, + "ronpo/target_abs": 0.08973653614521027, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 125 + }, + { + "drift/chosen_abs": 0.30845123529434204, + "drift/rejected_abs": 0.11910972744226456, + "epoch": 0.04131903059197457, + "grad_norm": 11.0625, + "learning_rate": 9.942908597485558e-08, + "logits/chosen": -1.8599214553833008, + "logits/rejected": -1.7708088159561157, + "logps/chosen": -0.25000277161598206, + "logps/rejected": -0.2528410851955414, + "loss": 0.5783834457397461, + "loss/core": 0.5539836883544922, + "loss/preference_sft": 0.25629931688308716, + "loss/reference_anchor": 0.46236711740493774, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.0844438076019287, + "rewards/margins": 1.8948835134506226, + "rewards/rejected": 1.1895604133605957, + "ronpo/logit": 0.18948833644390106, + "ronpo/residual": 0.17439742386341095, + "ronpo/residual_abs": 0.2648027837276459, + "ronpo/target": 0.015090915374457836, + "ronpo/target_abs": 0.07385419309139252, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 130 + }, + { + "drift/chosen_abs": 0.14000125229358673, + "drift/rejected_abs": 0.10091972351074219, + "epoch": 0.04290822407628129, + "grad_norm": 12.8125, + "learning_rate": 9.927369201401357e-08, + "logits/chosen": -1.9906437397003174, + "logits/rejected": -1.9516689777374268, + "logps/chosen": -0.2750958502292633, + "logps/rejected": -0.284817099571228, + "loss": 0.043532054871320724, + "loss/core": 0.03747930750250816, + "loss/preference_sft": 0.2810477614402771, + "loss/reference_anchor": 0.09295015782117844, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.398533821105957, + "rewards/margins": 0.38980793952941895, + "rewards/rejected": 1.0087257623672485, + "ronpo/logit": 0.038980789482593536, + "ronpo/residual": 0.042172592133283615, + "ronpo/residual_abs": 0.11995859444141388, + "ronpo/target": -0.0031918026506900787, + "ronpo/target_abs": 0.0830756425857544, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 135 + }, + { + "drift/chosen_abs": 0.24642232060432434, + "drift/rejected_abs": 0.15418444573879242, + "epoch": 0.044497417560588004, + "grad_norm": 5.9375, + "learning_rate": 9.909976823275142e-08, + "logits/chosen": -1.6394424438476562, + "logits/rejected": -1.6369479894638062, + "logps/chosen": -0.26613327860832214, + "logps/rejected": -0.25486260652542114, + "loss": 0.07675577700138092, + "loss/core": 0.06679292768239975, + "loss/preference_sft": 0.26119264960289, + "loss/reference_anchor": 0.17313750088214874, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.4627883434295654, + "rewards/margins": 0.9230669140815735, + "rewards/rejected": 1.5397213697433472, + "ronpo/logit": 0.09230668842792511, + "ronpo/residual": 0.0767742171883583, + "ronpo/residual_abs": 0.14399012923240662, + "ronpo/target": 0.01553247682750225, + "ronpo/target_abs": 0.0734403133392334, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 140 + }, + { + "drift/chosen_abs": 0.194285050034523, + "drift/rejected_abs": 0.10672137886285782, + "epoch": 0.04608661104489471, + "grad_norm": 272.0, + "learning_rate": 9.890738003669028e-08, + "logits/chosen": -2.0392231941223145, + "logits/rejected": -2.050145387649536, + "logps/chosen": -0.3017435669898987, + "logps/rejected": -0.3074425160884857, + "loss": 0.5806225538253784, + "loss/core": 0.5637769103050232, + "loss/preference_sft": 0.30560463666915894, + "loss/reference_anchor": 0.30635276436805725, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.942411184310913, + "rewards/margins": 0.8751977682113647, + "rewards/rejected": 1.067213773727417, + "ronpo/logit": 0.08751976490020752, + "ronpo/residual": 0.09972776472568512, + "ronpo/residual_abs": 0.25175055861473083, + "ronpo/target": -0.012207982130348682, + "ronpo/target_abs": 0.0755801871418953, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 145 + }, + { + "drift/chosen_abs": 0.31181323528289795, + "drift/rejected_abs": 0.08847779780626297, + "epoch": 0.04767580452920143, + "grad_norm": 24.875, + "learning_rate": 9.86965997751626e-08, + "logits/chosen": -1.8684451580047607, + "logits/rejected": -1.8871656656265259, + "logps/chosen": -0.2658239006996155, + "logps/rejected": -0.2555927336215973, + "loss": 0.5834201574325562, + "loss/core": 0.5655571222305298, + "loss/preference_sft": 0.2657192051410675, + "loss/reference_anchor": 0.33068618178367615, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.1168699264526367, + "rewards/margins": 2.2592427730560303, + "rewards/rejected": 0.8576269149780273, + "ronpo/logit": 0.2259242832660675, + "ronpo/residual": 0.22903215885162354, + "ronpo/residual_abs": 0.27631235122680664, + "ronpo/target": -0.003107901895418763, + "ronpo/target_abs": 0.08685997128486633, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 150 + }, + { + "drift/chosen_abs": 0.12096400558948517, + "drift/rejected_abs": 0.07475753873586655, + "epoch": 0.04926499801350814, + "grad_norm": 1.0703125, + "learning_rate": 9.846750671400445e-08, + "logits/chosen": -1.976851463317871, + "logits/rejected": -1.939896583557129, + "logps/chosen": -0.2994900345802307, + "logps/rejected": -0.30892014503479004, + "loss": 0.03421011567115784, + "loss/core": 0.030272841453552246, + "loss/preference_sft": 0.31494635343551636, + "loss/reference_anchor": 0.04725093021988869, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.2083873748779297, + "rewards/margins": 0.46204623579978943, + "rewards/rejected": 0.7463411092758179, + "ronpo/logit": 0.04620462656021118, + "ronpo/residual": 0.03318546339869499, + "ronpo/residual_abs": 0.11656346172094345, + "ronpo/target": 0.013019162230193615, + "ronpo/target_abs": 0.08384847640991211, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 155 + }, + { + "drift/chosen_abs": 0.16452792286872864, + "drift/rejected_abs": 0.1353653371334076, + "epoch": 0.05085419149781486, + "grad_norm": 8.4375, + "learning_rate": 9.822018700574695e-08, + "logits/chosen": -1.954433798789978, + "logits/rejected": -1.929897665977478, + "logps/chosen": -0.28900691866874695, + "logps/rejected": -0.291254460811615, + "loss": 0.2159826010465622, + "loss/core": 0.20637917518615723, + "loss/preference_sft": 0.2918491065502167, + "loss/reference_anchor": 0.16288307309150696, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.644606351852417, + "rewards/margins": 0.29627370834350586, + "rewards/rejected": 1.3483326435089111, + "ronpo/logit": 0.029627371579408646, + "ronpo/residual": 0.037656258791685104, + "ronpo/residual_abs": 0.1743960678577423, + "ronpo/target": -0.008028890937566757, + "ronpo/target_abs": 0.073066346347332, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 160 + }, + { + "drift/chosen_abs": 0.3098263144493103, + "drift/rejected_abs": 0.1317211091518402, + "epoch": 0.052443384982121574, + "grad_norm": 5.40625, + "learning_rate": 9.79547336572177e-08, + "logits/chosen": -1.7146660089492798, + "logits/rejected": -1.6996208429336548, + "logps/chosen": -0.2845102846622467, + "logps/rejected": -0.2821315824985504, + "loss": 0.39138349890708923, + "loss/core": 0.3647650480270386, + "loss/preference_sft": 0.2856329381465912, + "loss/reference_anchor": 0.5038057565689087, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.0979020595550537, + "rewards/margins": 1.7806909084320068, + "rewards/rejected": 1.3172109127044678, + "ronpo/logit": 0.1780690848827362, + "ronpo/residual": 0.1646689921617508, + "ronpo/residual_abs": 0.223629429936409, + "ronpo/target": 0.013400102965533733, + "ronpo/target_abs": 0.06691430509090424, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 165 + }, + { + "drift/chosen_abs": 0.31989067792892456, + "drift/rejected_abs": 0.12814299762248993, + "epoch": 0.05403257846642829, + "grad_norm": 2.625, + "learning_rate": 9.767124649456482e-08, + "logits/chosen": -1.8619706630706787, + "logits/rejected": -1.8246967792510986, + "logps/chosen": -0.28764164447784424, + "logps/rejected": -0.2885397970676422, + "loss": 0.5198912024497986, + "loss/core": 0.4936496615409851, + "loss/preference_sft": 0.2862817645072937, + "loss/reference_anchor": 0.49620193243026733, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.1981494426727295, + "rewards/margins": 1.9242050647735596, + "rewards/rejected": 1.273944616317749, + "ronpo/logit": 0.19242048263549805, + "ronpo/residual": 0.18473686277866364, + "ronpo/residual_abs": 0.24073760211467743, + "ronpo/target": 0.007683630101382732, + "ronpo/target_abs": 0.0752769261598587, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 170 + }, + { + "drift/chosen_abs": 0.22625796496868134, + "drift/rejected_abs": 0.08741482347249985, + "epoch": 0.055621771950735005, + "grad_norm": 38.25, + "learning_rate": 9.736983212571646e-08, + "logits/chosen": -1.825382947921753, + "logits/rejected": -1.8280212879180908, + "logps/chosen": -0.2802065908908844, + "logps/rejected": -0.2865963578224182, + "loss": 0.21462078392505646, + "loss/core": 0.2051103413105011, + "loss/preference_sft": 0.2825978994369507, + "loss/reference_anchor": 0.1619492471218109, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.261361598968506, + "rewards/margins": 1.3890936374664307, + "rewards/rejected": 0.8722679018974304, + "ronpo/logit": 0.13890936970710754, + "ronpo/residual": 0.14540444314479828, + "ronpo/residual_abs": 0.1883070468902588, + "ronpo/target": -0.006495094392448664, + "ronpo/target_abs": 0.07979901880025864, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 175 + }, + { + "drift/chosen_abs": 0.31887388229370117, + "drift/rejected_abs": 0.14860941469669342, + "epoch": 0.057210965435041714, + "grad_norm": 137.0, + "learning_rate": 9.705060390028978e-08, + "logits/chosen": -1.8157069683074951, + "logits/rejected": -1.7937179803848267, + "logps/chosen": -0.2820594310760498, + "logps/rejected": -0.2725641131401062, + "loss": 0.30303138494491577, + "loss/core": 0.28699272871017456, + "loss/preference_sft": 0.26962658762931824, + "loss/reference_anchor": 0.2938117980957031, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.188739061355591, + "rewards/margins": 1.7026450634002686, + "rewards/rejected": 1.4860939979553223, + "ronpo/logit": 0.17026451230049133, + "ronpo/residual": 0.16811729967594147, + "ronpo/residual_abs": 0.21821363270282745, + "ronpo/target": 0.0021472182124853134, + "ronpo/target_abs": 0.07455535978078842, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 180 + }, + { + "drift/chosen_abs": 0.3943818509578705, + "drift/rejected_abs": 0.16938166320323944, + "epoch": 0.05880015891934843, + "grad_norm": 13.9375, + "learning_rate": 9.671368186696487e-08, + "logits/chosen": -1.682560682296753, + "logits/rejected": -1.7234325408935547, + "logps/chosen": -0.29317352175712585, + "logps/rejected": -0.3039776384830475, + "loss": 0.7356561422348022, + "loss/core": 0.6865347623825073, + "loss/preference_sft": 0.3006138801574707, + "loss/reference_anchor": 0.9523669481277466, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.942030429840088, + "rewards/margins": 2.3751327991485596, + "rewards/rejected": 1.5668972730636597, + "ronpo/logit": 0.23751330375671387, + "ronpo/residual": 0.23533913493156433, + "ronpo/residual_abs": 0.28016549348831177, + "ronpo/target": 0.0021741650998592377, + "ronpo/target_abs": 0.07683144509792328, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 185 + }, + { + "drift/chosen_abs": 0.21583101153373718, + "drift/rejected_abs": 0.14082249999046326, + "epoch": 0.060389352403655144, + "grad_norm": 1.84375, + "learning_rate": 9.635919272833936e-08, + "logits/chosen": -1.7084299325942993, + "logits/rejected": -1.7109653949737549, + "logps/chosen": -0.28803473711013794, + "logps/rejected": -0.29996567964553833, + "loss": 0.10773409903049469, + "loss/core": 0.09961052983999252, + "loss/preference_sft": 0.2932335436344147, + "loss/reference_anchor": 0.1331479549407959, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.157505750656128, + "rewards/margins": 0.7513991594314575, + "rewards/rejected": 1.4061065912246704, + "ronpo/logit": 0.07513991743326187, + "ronpo/residual": 0.061122387647628784, + "ronpo/residual_abs": 0.1610337495803833, + "ronpo/target": 0.014017535373568535, + "ronpo/target_abs": 0.08196841925382614, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 190 + }, + { + "drift/chosen_abs": 0.18434445559978485, + "drift/rejected_abs": 0.11586540937423706, + "epoch": 0.06197854588796186, + "grad_norm": 1.765625, + "learning_rate": 9.598726979328077e-08, + "logits/chosen": -1.86000657081604, + "logits/rejected": -1.8386214971542358, + "logps/chosen": -0.3027092218399048, + "logps/rejected": -0.2923242449760437, + "loss": 0.20319950580596924, + "loss/core": 0.1940327286720276, + "loss/preference_sft": 0.2979516088962555, + "loss/reference_anchor": 0.15354016423225403, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.8425309658050537, + "rewards/margins": 0.6865968704223633, + "rewards/rejected": 1.15593421459198, + "ronpo/logit": 0.06865968555212021, + "ronpo/residual": 0.06499691307544708, + "ronpo/residual_abs": 0.19421474635601044, + "ronpo/target": 0.0036627836525440216, + "ronpo/target_abs": 0.07378067076206207, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 195 + }, + { + "drift/chosen_abs": 0.3092130422592163, + "drift/rejected_abs": 0.1502634435892105, + "epoch": 0.06356773937226858, + "grad_norm": 12.6875, + "learning_rate": 9.559805292679444e-08, + "logits/chosen": -1.878720998764038, + "logits/rejected": -1.853334665298462, + "logps/chosen": -0.28743189573287964, + "logps/rejected": -0.3008131980895996, + "loss": 0.38157254457473755, + "loss/core": 0.3494413197040558, + "loss/preference_sft": 0.2995978593826294, + "loss/reference_anchor": 0.6126652359962463, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.0919899940490723, + "rewards/margins": 1.5904356241226196, + "rewards/rejected": 1.5015544891357422, + "ronpo/logit": 0.1590435802936554, + "ronpo/residual": 0.16887164115905762, + "ronpo/residual_abs": 0.21642287075519562, + "ronpo/target": -0.009828063659369946, + "ronpo/target_abs": 0.09070595353841782, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 200 + }, + { + "drift/chosen_abs": 0.2549411654472351, + "drift/rejected_abs": 0.10770858824253082, + "epoch": 0.06515693285657528, + "grad_norm": 9.0625, + "learning_rate": 9.519168849742603e-08, + "logits/chosen": -1.8176409006118774, + "logits/rejected": -1.7602897882461548, + "logps/chosen": -0.28294557332992554, + "logps/rejected": -0.2822961211204529, + "loss": 0.13457834720611572, + "loss/core": 0.1233421191573143, + "loss/preference_sft": 0.27934110164642334, + "loss/reference_anchor": 0.19679050147533417, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.5482711791992188, + "rewards/margins": 1.4736816883087158, + "rewards/rejected": 1.074589490890503, + "ronpo/logit": 0.1473681777715683, + "ronpo/residual": 0.15736904740333557, + "ronpo/residual_abs": 0.1994774490594864, + "ronpo/target": -0.010000874288380146, + "ronpo/target_abs": 0.07206423580646515, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 205 + }, + { + "drift/chosen_abs": 0.3109588325023651, + "drift/rejected_abs": 0.12311758100986481, + "epoch": 0.066746126340882, + "grad_norm": 19.75, + "learning_rate": 9.476832932221834e-08, + "logits/chosen": -1.8160145282745361, + "logits/rejected": -1.8787750005722046, + "logps/chosen": -0.2810691297054291, + "logps/rejected": -0.28845909237861633, + "loss": 0.5719379186630249, + "loss/core": 0.5511919260025024, + "loss/preference_sft": 0.2875823974609375, + "loss/reference_anchor": 0.38616234064102173, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.108250141143799, + "rewards/margins": 1.8811763525009155, + "rewards/rejected": 1.2270740270614624, + "ronpo/logit": 0.1881176233291626, + "ronpo/residual": 0.18612520396709442, + "ronpo/residual_abs": 0.2601304054260254, + "ronpo/target": 0.0019924379885196686, + "ronpo/target_abs": 0.07251577079296112, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 210 + }, + { + "drift/chosen_abs": 0.173118457198143, + "drift/rejected_abs": 0.11045239865779877, + "epoch": 0.06833531982518871, + "grad_norm": 4.3125, + "learning_rate": 9.432813460924306e-08, + "logits/chosen": -1.8905178308486938, + "logits/rejected": -1.8230892419815063, + "logps/chosen": -0.2746286988258362, + "logps/rejected": -0.28819310665130615, + "loss": 0.03434374928474426, + "loss/core": 0.02805573306977749, + "loss/preference_sft": 0.28005850315093994, + "loss/reference_anchor": 0.09775449335575104, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.731080412864685, + "rewards/margins": 0.6269391179084778, + "rewards/rejected": 1.1041414737701416, + "ronpo/logit": 0.06269390881061554, + "ronpo/residual": 0.05062420293688774, + "ronpo/residual_abs": 0.11113785207271576, + "ronpo/target": 0.012069707736372948, + "ronpo/target_abs": 0.07993054389953613, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 215 + }, + { + "drift/chosen_abs": 0.3507832884788513, + "drift/rejected_abs": 0.13538460433483124, + "epoch": 0.06992451330949544, + "grad_norm": 1.5390625, + "learning_rate": 9.387126989772909e-08, + "logits/chosen": -1.713751196861267, + "logits/rejected": -1.7000278234481812, + "logps/chosen": -0.2693570852279663, + "logps/rejected": -0.288845419883728, + "loss": 1.1489449739456177, + "loss/core": 1.1118707656860352, + "loss/preference_sft": 0.28148311376571655, + "loss/reference_anchor": 0.713333010673523, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.5078330039978027, + "rewards/margins": 2.1564221382141113, + "rewards/rejected": 1.3514111042022705, + "ronpo/logit": 0.21564224362373352, + "ronpo/residual": 0.20636311173439026, + "ronpo/residual_abs": 0.3267207145690918, + "ronpo/target": 0.009279133751988411, + "ronpo/target_abs": 0.08725794404745102, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 220 + }, + { + "drift/chosen_abs": 0.33888542652130127, + "drift/rejected_abs": 0.08225339651107788, + "epoch": 0.07151370679380215, + "grad_norm": 1.046875, + "learning_rate": 9.339790699581003e-08, + "logits/chosen": -1.9155328273773193, + "logits/rejected": -1.8650407791137695, + "logps/chosen": -0.3122982084751129, + "logps/rejected": -0.30397704243659973, + "loss": 1.3468017578125, + "loss/core": 1.3081927299499512, + "loss/preference_sft": 0.3082147538661957, + "loss/reference_anchor": 0.741364061832428, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.388200044631958, + "rewards/margins": 2.5675928592681885, + "rewards/rejected": 0.8206073641777039, + "ronpo/logit": 0.25675931572914124, + "ronpo/residual": 0.24346081912517548, + "ronpo/residual_abs": 0.3151950538158417, + "ronpo/target": 0.01329845655709505, + "ronpo/target_abs": 0.08093789964914322, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 225 + }, + { + "drift/chosen_abs": 0.2742893099784851, + "drift/rejected_abs": 0.09671930223703384, + "epoch": 0.07310290027810885, + "grad_norm": 1.9921875, + "learning_rate": 9.290822391591417e-08, + "logits/chosen": -1.6402956247329712, + "logits/rejected": -1.6450611352920532, + "logps/chosen": -0.2720198631286621, + "logps/rejected": -0.2679344117641449, + "loss": 0.3185911476612091, + "loss/core": 0.30630046129226685, + "loss/preference_sft": 0.26990702748298645, + "loss/reference_anchor": 0.21882298588752747, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.742389440536499, + "rewards/margins": 1.7752317190170288, + "rewards/rejected": 0.9671577215194702, + "ronpo/logit": 0.1775231808423996, + "ronpo/residual": 0.17394277453422546, + "ronpo/residual_abs": 0.22346839308738708, + "ronpo/target": 0.003580413293093443, + "ronpo/target_abs": 0.09092529118061066, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 230 + }, + { + "drift/chosen_abs": 0.3355063498020172, + "drift/rejected_abs": 0.14276744425296783, + "epoch": 0.07469209376241558, + "grad_norm": 3.609375, + "learning_rate": 9.240240480782128e-08, + "logits/chosen": -1.8078244924545288, + "logits/rejected": -1.8107181787490845, + "logps/chosen": -0.2891586422920227, + "logps/rejected": -0.2673870921134949, + "loss": 0.7774661183357239, + "loss/core": 0.75029456615448, + "loss/preference_sft": 0.29623985290527344, + "loss/reference_anchor": 0.5138068795204163, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.3549816608428955, + "rewards/margins": 1.9295164346694946, + "rewards/rejected": 1.4254653453826904, + "ronpo/logit": 0.19295167922973633, + "ronpo/residual": 0.196090430021286, + "ronpo/residual_abs": 0.23866817355155945, + "ronpo/target": -0.0031387873459607363, + "ronpo/target_abs": 0.06865248829126358, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 235 + }, + { + "drift/chosen_abs": 0.3238152265548706, + "drift/rejected_abs": 0.14746034145355225, + "epoch": 0.07628128724672228, + "grad_norm": 1.3515625, + "learning_rate": 9.188063988941146e-08, + "logits/chosen": -1.9462316036224365, + "logits/rejected": -1.9106372594833374, + "logps/chosen": -0.25194069743156433, + "logps/rejected": -0.26762017607688904, + "loss": 0.428089439868927, + "loss/core": 0.4055059850215912, + "loss/preference_sft": 0.25560957193374634, + "loss/reference_anchor": 0.4261084198951721, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.237283706665039, + "rewards/margins": 1.8325433731079102, + "rewards/rejected": 1.4047404527664185, + "ronpo/logit": 0.18325433135032654, + "ronpo/residual": 0.18264977633953094, + "ronpo/residual_abs": 0.21224486827850342, + "ronpo/target": 0.0006045631016604602, + "ronpo/target_abs": 0.055509041994810104, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 240 + }, + { + "drift/chosen_abs": 0.24143171310424805, + "drift/rejected_abs": 0.09511907398700714, + "epoch": 0.07787048073102901, + "grad_norm": 4.53125, + "learning_rate": 9.134312537513186e-08, + "logits/chosen": -1.8824867010116577, + "logits/rejected": -1.8663095235824585, + "logps/chosen": -0.27780038118362427, + "logps/rejected": -0.27743133902549744, + "loss": 0.7090773582458496, + "loss/core": 0.6847109794616699, + "loss/preference_sft": 0.27157658338546753, + "loss/reference_anchor": 0.4601725935935974, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.4126458168029785, + "rewards/margins": 1.462439775466919, + "rewards/rejected": 0.9502058029174805, + "ronpo/logit": 0.14624398946762085, + "ronpo/residual": 0.16018269956111908, + "ronpo/residual_abs": 0.220293328166008, + "ronpo/target": -0.013938702642917633, + "ronpo/target_abs": 0.08255701512098312, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 245 + }, + { + "drift/chosen_abs": 0.23546819388866425, + "drift/rejected_abs": 0.12727732956409454, + "epoch": 0.07945967421533572, + "grad_norm": 3.34375, + "learning_rate": 9.07900634022086e-08, + "logits/chosen": -1.9230791330337524, + "logits/rejected": -1.8947263956069946, + "logps/chosen": -0.2548685669898987, + "logps/rejected": -0.25849586725234985, + "loss": 0.08169195055961609, + "loss/core": 0.06879390776157379, + "loss/preference_sft": 0.26240864396095276, + "loss/reference_anchor": 0.23172004520893097, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.35325288772583, + "rewards/margins": 1.0831009149551392, + "rewards/rejected": 1.2701518535614014, + "ronpo/logit": 0.10831008851528168, + "ronpo/residual": 0.09686598926782608, + "ronpo/residual_abs": 0.16113971173763275, + "ronpo/target": 0.01144410576671362, + "ronpo/target_abs": 0.07850413024425507, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 250 + }, + { + "drift/chosen_abs": 0.22031930088996887, + "drift/rejected_abs": 0.14361494779586792, + "epoch": 0.08104886769964244, + "grad_norm": 0.6328125, + "learning_rate": 9.02216619546311e-08, + "logits/chosen": -1.790488600730896, + "logits/rejected": -1.882672667503357, + "logps/chosen": -0.3244701027870178, + "logps/rejected": -0.32260042428970337, + "loss": 0.12643373012542725, + "loss/core": 0.08407403528690338, + "loss/preference_sft": 0.326153427362442, + "loss/reference_anchor": 0.8145785331726074, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.202606678009033, + "rewards/margins": 0.766952395439148, + "rewards/rejected": 1.4356540441513062, + "ronpo/logit": 0.07669521868228912, + "ronpo/residual": 0.0710417702794075, + "ronpo/residual_abs": 0.12641890347003937, + "ronpo/target": 0.005653454922139645, + "ronpo/target_abs": 0.07296518236398697, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 255 + }, + { + "drift/chosen_abs": 0.3037354350090027, + "drift/rejected_abs": 0.20296995341777802, + "epoch": 0.08263806118394915, + "grad_norm": 46.25, + "learning_rate": 8.963813478493788e-08, + "logits/chosen": -1.888965368270874, + "logits/rejected": -1.9428901672363281, + "logps/chosen": -0.27975988388061523, + "logps/rejected": -0.2818945348262787, + "loss": 0.22990703582763672, + "loss/core": 0.18613065779209137, + "loss/preference_sft": 0.27945324778556824, + "loss/reference_anchor": 0.8475823402404785, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.036375045776367, + "rewards/margins": 1.0084631443023682, + "rewards/rejected": 2.02791166305542, + "ronpo/logit": 0.1008463129401207, + "ronpo/residual": 0.10048683732748032, + "ronpo/residual_abs": 0.20321817696094513, + "ronpo/target": 0.00035947878495790064, + "ronpo/target_abs": 0.08794078230857849, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 260 + }, + { + "drift/chosen_abs": 0.24098190665245056, + "drift/rejected_abs": 0.1554281860589981, + "epoch": 0.08422725466825585, + "grad_norm": 119.0, + "learning_rate": 8.903970133383297e-08, + "logits/chosen": -1.8848758935928345, + "logits/rejected": -1.8094570636749268, + "logps/chosen": -0.26744991540908813, + "logps/rejected": -0.2860262989997864, + "loss": 0.37246307730674744, + "loss/core": 0.35889145731925964, + "loss/preference_sft": 0.2836495339870453, + "loss/reference_anchor": 0.243066668510437, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.4094414710998535, + "rewards/margins": 0.9786992073059082, + "rewards/rejected": 1.4307423830032349, + "ronpo/logit": 0.09786991775035858, + "ronpo/residual": 0.08945603668689728, + "ronpo/residual_abs": 0.22707316279411316, + "ronpo/target": 0.008413868956267834, + "ronpo/target_abs": 0.07881908118724823, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 265 + }, + { + "drift/chosen_abs": 0.3145125210285187, + "drift/rejected_abs": 0.13519734144210815, + "epoch": 0.08581644815256258, + "grad_norm": 55.0, + "learning_rate": 8.842658664766316e-08, + "logits/chosen": -1.8931232690811157, + "logits/rejected": -1.8972790241241455, + "logps/chosen": -0.291258305311203, + "logps/rejected": -0.2826591730117798, + "loss": 0.6564851999282837, + "loss/core": 0.6233766078948975, + "loss/preference_sft": 0.2801192104816437, + "loss/reference_anchor": 0.634162187576294, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.143418312072754, + "rewards/margins": 1.7918039560317993, + "rewards/rejected": 1.3516143560409546, + "ronpo/logit": 0.17918041348457336, + "ronpo/residual": 0.1889977604150772, + "ronpo/residual_abs": 0.24317440390586853, + "ronpo/target": -0.009817345067858696, + "ronpo/target_abs": 0.07919232547283173, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 270 + }, + { + "drift/chosen_abs": 0.14758369326591492, + "drift/rejected_abs": 0.06989634037017822, + "epoch": 0.08740564163686929, + "grad_norm": 13.3125, + "learning_rate": 8.779902129378738e-08, + "logits/chosen": -1.9660217761993408, + "logits/rejected": -1.9344068765640259, + "logps/chosen": -0.3128483295440674, + "logps/rejected": -0.3132597804069519, + "loss": 0.0763622298836708, + "loss/core": 0.07052332162857056, + "loss/preference_sft": 0.31526920199394226, + "loss/reference_anchor": 0.08525120466947556, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.475311517715454, + "rewards/margins": 0.7777661681175232, + "rewards/rejected": 0.6975454092025757, + "ronpo/logit": 0.07777661830186844, + "ronpo/residual": 0.07221326977014542, + "ronpo/residual_abs": 0.12839944660663605, + "ronpo/target": 0.005563343409448862, + "ronpo/target_abs": 0.06888459622859955, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 275 + }, + { + "drift/chosen_abs": 0.25686928629875183, + "drift/rejected_abs": 0.12125973403453827, + "epoch": 0.08899483512117601, + "grad_norm": 1.71875, + "learning_rate": 8.715724127386971e-08, + "logits/chosen": -1.8945934772491455, + "logits/rejected": -1.8945395946502686, + "logps/chosen": -0.28814825415611267, + "logps/rejected": -0.27774301171302795, + "loss": 0.25721555948257446, + "loss/core": 0.24336083233356476, + "loss/preference_sft": 0.2840600907802582, + "loss/reference_anchor": 0.24868884682655334, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.568566083908081, + "rewards/margins": 1.357149600982666, + "rewards/rejected": 1.2114163637161255, + "ronpo/logit": 0.13571497797966003, + "ronpo/residual": 0.13394533097743988, + "ronpo/residual_abs": 0.2140486240386963, + "ronpo/target": 0.0017696395516395569, + "ronpo/target_abs": 0.08350421488285065, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 280 + }, + { + "drift/chosen_abs": 0.22564442455768585, + "drift/rejected_abs": 0.10595756769180298, + "epoch": 0.09058402860548272, + "grad_norm": 1.359375, + "learning_rate": 8.650148793512872e-08, + "logits/chosen": -1.861046552658081, + "logits/rejected": -1.8655040264129639, + "logps/chosen": -0.2829212248325348, + "logps/rejected": -0.28727012872695923, + "loss": 0.11808571964502335, + "loss/core": 0.10924997180700302, + "loss/preference_sft": 0.28626394271850586, + "loss/reference_anchor": 0.14808858931064606, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.25600266456604, + "rewards/margins": 1.207207441329956, + "rewards/rejected": 1.0487949848175049, + "ronpo/logit": 0.1207207590341568, + "ronpo/residual": 0.14722566306591034, + "ronpo/residual_abs": 0.18361715972423553, + "ronpo/target": -0.026504909619688988, + "ronpo/target_abs": 0.08099932968616486, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 285 + }, + { + "drift/chosen_abs": 0.3155427873134613, + "drift/rejected_abs": 0.13171601295471191, + "epoch": 0.09217322208978942, + "grad_norm": 45.5, + "learning_rate": 8.58320078795768e-08, + "logits/chosen": -1.9795125722885132, + "logits/rejected": -1.9921321868896484, + "logps/chosen": -0.2706143856048584, + "logps/rejected": -0.266631156206131, + "loss": 0.5211764574050903, + "loss/core": 0.4964214861392975, + "loss/preference_sft": 0.2676003873348236, + "loss/reference_anchor": 0.46833816170692444, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.1550698280334473, + "rewards/margins": 1.890966773033142, + "rewards/rejected": 1.2641032934188843, + "ronpo/logit": 0.18909668922424316, + "ronpo/residual": 0.1774296909570694, + "ronpo/residual_abs": 0.23044650256633759, + "ronpo/target": 0.011666996404528618, + "ronpo/target_abs": 0.07283170521259308, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 290 + }, + { + "drift/chosen_abs": 0.3618084788322449, + "drift/rejected_abs": 0.14779672026634216, + "epoch": 0.09376241557409615, + "grad_norm": 64.5, + "learning_rate": 8.514905287128308e-08, + "logits/chosen": -1.6823537349700928, + "logits/rejected": -1.653702974319458, + "logps/chosen": -0.2833479344844818, + "logps/rejected": -0.30102211236953735, + "loss": 1.0796386003494263, + "loss/core": 1.046770691871643, + "loss/preference_sft": 0.29381901025772095, + "loss/reference_anchor": 0.6279757618904114, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 3.616563081741333, + "rewards/margins": 2.1403043270111084, + "rewards/rejected": 1.4762591123580933, + "ronpo/logit": 0.2140304297208786, + "ronpo/residual": 0.2224186658859253, + "ronpo/residual_abs": 0.360507071018219, + "ronpo/target": -0.008388230577111244, + "ronpo/target_abs": 0.08402254432439804, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 295 + }, + { + "drift/chosen_abs": 0.25922662019729614, + "drift/rejected_abs": 0.12114284187555313, + "epoch": 0.09535160905840286, + "grad_norm": 0.6875, + "learning_rate": 8.44528797416954e-08, + "logits/chosen": -1.9925893545150757, + "logits/rejected": -1.9968611001968384, + "logps/chosen": -0.2853473722934723, + "logps/rejected": -0.2928779125213623, + "loss": 0.6324478983879089, + "loss/core": 0.6148328185081482, + "loss/preference_sft": 0.2793324291706085, + "loss/reference_anchor": 0.3243691325187683, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.592266082763672, + "rewards/margins": 1.6641126871109009, + "rewards/rejected": 0.9281536340713501, + "ronpo/logit": 0.16641125082969666, + "ronpo/residual": 0.16771617531776428, + "ronpo/residual_abs": 0.2484259307384491, + "ronpo/target": -0.0013049064436927438, + "ronpo/target_abs": 0.07981012016534805, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 300 + }, + { + "drift/chosen_abs": 0.33662670850753784, + "drift/rejected_abs": 0.15090534090995789, + "epoch": 0.09694080254270958, + "grad_norm": 9.1875, + "learning_rate": 8.374375029305638e-08, + "logits/chosen": -1.8022260665893555, + "logits/rejected": -1.7453006505966187, + "logps/chosen": -0.2850312888622284, + "logps/rejected": -0.29107189178466797, + "loss": 0.6764329075813293, + "loss/core": 0.6424604654312134, + "loss/preference_sft": 0.291935533285141, + "loss/reference_anchor": 0.6502560377120972, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.3646740913391113, + "rewards/margins": 1.873510718345642, + "rewards/rejected": 1.4911634922027588, + "ronpo/logit": 0.1873510777950287, + "ronpo/residual": 0.1918567419052124, + "ronpo/residual_abs": 0.3251708447933197, + "ronpo/target": -0.004505653399974108, + "ronpo/target_abs": 0.08541780710220337, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 305 + }, + { + "drift/chosen_abs": 0.2991493344306946, + "drift/rejected_abs": 0.15711288154125214, + "epoch": 0.09852999602701629, + "grad_norm": 2.5625, + "learning_rate": 8.302193119995037e-08, + "logits/chosen": -1.8975536823272705, + "logits/rejected": -1.919425368309021, + "logps/chosen": -0.2862815260887146, + "logps/rejected": -0.27441829442977905, + "loss": 0.6102262735366821, + "loss/core": 0.576637327671051, + "loss/preference_sft": 0.27816590666770935, + "loss/reference_anchor": 0.643962025642395, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.991199016571045, + "rewards/margins": 1.4210230112075806, + "rewards/rejected": 1.570176362991333, + "ronpo/logit": 0.14210231602191925, + "ronpo/residual": 0.11606768518686295, + "ronpo/residual_abs": 0.18980509042739868, + "ronpo/target": 0.026034632697701454, + "ronpo/target_abs": 0.07823829352855682, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 310 + }, + { + "drift/chosen_abs": 0.26028409600257874, + "drift/rejected_abs": 0.10342870652675629, + "epoch": 0.10011918951132301, + "grad_norm": 25.875, + "learning_rate": 8.22876939090181e-08, + "logits/chosen": -1.6694761514663696, + "logits/rejected": -1.6918532848358154, + "logps/chosen": -0.27476266026496887, + "logps/rejected": -0.2757488787174225, + "loss": 0.43638157844543457, + "loss/core": 0.4186526834964752, + "loss/preference_sft": 0.27721649408340454, + "loss/reference_anchor": 0.3268568515777588, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.6024768352508545, + "rewards/margins": 1.5710161924362183, + "rewards/rejected": 1.0314604043960571, + "ronpo/logit": 0.15710163116455078, + "ronpo/residual": 0.1434870809316635, + "ronpo/residual_abs": 0.2207198143005371, + "ronpo/target": 0.013614545576274395, + "ronpo/target_abs": 0.08071252703666687, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 315 + }, + { + "drift/chosen_abs": 0.3579448461532593, + "drift/rejected_abs": 0.1325366348028183, + "epoch": 0.10170838299562972, + "grad_norm": 2.546875, + "learning_rate": 8.154131453687656e-08, + "logits/chosen": -1.8659942150115967, + "logits/rejected": -1.9224580526351929, + "logps/chosen": -0.2523338794708252, + "logps/rejected": -0.24946343898773193, + "loss": 0.8073204159736633, + "loss/core": 0.7731778621673584, + "loss/preference_sft": 0.24916532635688782, + "loss/reference_anchor": 0.6579362154006958, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.579383373260498, + "rewards/margins": 2.2741141319274902, + "rewards/rejected": 1.3052693605422974, + "ronpo/logit": 0.2274114340543747, + "ronpo/residual": 0.22022266685962677, + "ronpo/residual_abs": 0.2674153447151184, + "ronpo/target": 0.007188776973634958, + "ronpo/target_abs": 0.07314587384462357, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 320 + }, + { + "drift/chosen_abs": 0.23399193584918976, + "drift/rejected_abs": 0.09473086148500443, + "epoch": 0.10329757647993643, + "grad_norm": 364.0, + "learning_rate": 8.07830737662829e-08, + "logits/chosen": -1.743417739868164, + "logits/rejected": -1.7016518115997314, + "logps/chosen": -0.3154450058937073, + "logps/rejected": -0.3125259280204773, + "loss": 0.25524017214775085, + "loss/core": 0.23851685225963593, + "loss/preference_sft": 0.3184736371040344, + "loss/reference_anchor": 0.30261868238449097, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.336858034133911, + "rewards/margins": 1.3925182819366455, + "rewards/rejected": 0.9443395733833313, + "ronpo/logit": 0.13925182819366455, + "ronpo/residual": 0.15808895230293274, + "ronpo/residual_abs": 0.20290787518024445, + "ronpo/target": -0.018837137147784233, + "ronpo/target_abs": 0.08498185127973557, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 325 + }, + { + "drift/chosen_abs": 0.2572224736213684, + "drift/rejected_abs": 0.1335063874721527, + "epoch": 0.10488676996424315, + "grad_norm": 2.078125, + "learning_rate": 8.001325674058123e-08, + "logits/chosen": -1.8432238101959229, + "logits/rejected": -1.8311374187469482, + "logps/chosen": -0.2789004445075989, + "logps/rejected": -0.2724122703075409, + "loss": 0.45331627130508423, + "loss/core": 0.4269883632659912, + "loss/preference_sft": 0.2750396132469177, + "loss/reference_anchor": 0.4990541338920593, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.5702948570251465, + "rewards/margins": 1.2371727228164673, + "rewards/rejected": 1.3331226110458374, + "ronpo/logit": 0.12371727079153061, + "ronpo/residual": 0.1249464750289917, + "ronpo/residual_abs": 0.22960133850574493, + "ronpo/target": -0.0012291899183765054, + "ronpo/target_abs": 0.07105612754821777, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 330 + }, + { + "drift/chosen_abs": 0.3199670612812042, + "drift/rejected_abs": 0.1441265046596527, + "epoch": 0.10647596344854986, + "grad_norm": 59.5, + "learning_rate": 7.923215295647166e-08, + "logits/chosen": -1.8185367584228516, + "logits/rejected": -1.7632293701171875, + "logps/chosen": -0.28754284977912903, + "logps/rejected": -0.29002195596694946, + "loss": 0.5746418833732605, + "loss/core": 0.535438597202301, + "loss/preference_sft": 0.2888413071632385, + "loss/reference_anchor": 0.7551820874214172, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.1991424560546875, + "rewards/margins": 1.759937047958374, + "rewards/rejected": 1.439205527305603, + "ronpo/logit": 0.1759936809539795, + "ronpo/residual": 0.17700710892677307, + "ronpo/residual_abs": 0.2370648831129074, + "ronpo/target": -0.0010134152835235, + "ronpo/target_abs": 0.08590994030237198, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 335 + }, + { + "drift/chosen_abs": 0.1872010976076126, + "drift/rejected_abs": 0.12332087755203247, + "epoch": 0.10806515693285658, + "grad_norm": 3.65625, + "learning_rate": 7.844005615514259e-08, + "logits/chosen": -1.8502109050750732, + "logits/rejected": -1.8752959966659546, + "logps/chosen": -0.30195698142051697, + "logps/rejected": -0.2972937226295471, + "loss": 0.05311671644449234, + "loss/core": 0.043439559638500214, + "loss/preference_sft": 0.30265796184539795, + "loss/reference_anchor": 0.1632772982120514, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.8714481592178345, + "rewards/margins": 0.6399093866348267, + "rewards/rejected": 1.231539011001587, + "ronpo/logit": 0.06399093568325043, + "ronpo/residual": 0.07827167212963104, + "ronpo/residual_abs": 0.13036039471626282, + "ronpo/target": -0.014280728995800018, + "ronpo/target_abs": 0.07398774474859238, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 340 + }, + { + "drift/chosen_abs": 0.21623043715953827, + "drift/rejected_abs": 0.08763010054826736, + "epoch": 0.10965435041716329, + "grad_norm": 5.53125, + "learning_rate": 7.763726421180662e-08, + "logits/chosen": -1.8741140365600586, + "logits/rejected": -1.8508151769638062, + "logps/chosen": -0.27523356676101685, + "logps/rejected": -0.2796024680137634, + "loss": 0.331326425075531, + "loss/core": 0.31637856364250183, + "loss/preference_sft": 0.28105422854423523, + "loss/reference_anchor": 0.270852655172348, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.160953998565674, + "rewards/margins": 1.2863576412200928, + "rewards/rejected": 0.8745959401130676, + "ronpo/logit": 0.12863577902317047, + "ronpo/residual": 0.13460756838321686, + "ronpo/residual_abs": 0.18582451343536377, + "ronpo/target": -0.005971794482320547, + "ronpo/target_abs": 0.08206266909837723, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 345 + }, + { + "drift/chosen_abs": 0.4488181173801422, + "drift/rejected_abs": 0.1432909220457077, + "epoch": 0.11124354390147001, + "grad_norm": 744.0, + "learning_rate": 7.682407902368189e-08, + "logits/chosen": -1.8674952983856201, + "logits/rejected": -1.8539085388183594, + "logps/chosen": -0.2711220383644104, + "logps/rejected": -0.2666943073272705, + "loss": 1.336153268814087, + "loss/core": 1.280058741569519, + "loss/preference_sft": 0.2706068456172943, + "loss/reference_anchor": 1.094827651977539, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 4.485884666442871, + "rewards/margins": 3.054591178894043, + "rewards/rejected": 1.431293249130249, + "ronpo/logit": 0.3054591417312622, + "ronpo/residual": 0.3182205557823181, + "ronpo/residual_abs": 0.3483893871307373, + "ronpo/target": -0.012761471793055534, + "ronpo/target_abs": 0.08356453478336334, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 350 + }, + { + "drift/chosen_abs": 0.4042721688747406, + "drift/rejected_abs": 0.18896692991256714, + "epoch": 0.11283273738577672, + "grad_norm": 2.0, + "learning_rate": 7.600080639646076e-08, + "logits/chosen": -1.8509187698364258, + "logits/rejected": -1.8338550329208374, + "logps/chosen": -0.2988142967224121, + "logps/rejected": -0.30226239562034607, + "loss": 0.6167121529579163, + "loss/core": 0.5703554749488831, + "loss/preference_sft": 0.2945895195007324, + "loss/reference_anchor": 0.8976724743843079, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.042208194732666, + "rewards/margins": 2.1540372371673584, + "rewards/rejected": 1.888170838356018, + "ronpo/logit": 0.21540376543998718, + "ronpo/residual": 0.20375964045524597, + "ronpo/residual_abs": 0.26894694566726685, + "ronpo/target": 0.011644105426967144, + "ronpo/target_abs": 0.06865202635526657, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 355 + }, + { + "drift/chosen_abs": 0.24115343391895294, + "drift/rejected_abs": 0.13106033205986023, + "epoch": 0.11442193087008343, + "grad_norm": 4.9375, + "learning_rate": 7.516775592930899e-08, + "logits/chosen": -1.953965425491333, + "logits/rejected": -2.0095295906066895, + "logps/chosen": -0.3034398555755615, + "logps/rejected": -0.308123916387558, + "loss": 0.12772902846336365, + "loss/core": 0.11105193197727203, + "loss/preference_sft": 0.3055204749107361, + "loss/reference_anchor": 0.3029900789260864, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.409573793411255, + "rewards/margins": 1.099807620048523, + "rewards/rejected": 1.3097662925720215, + "ronpo/logit": 0.1099807471036911, + "ronpo/residual": 0.1174292117357254, + "ronpo/residual_abs": 0.17334052920341492, + "ronpo/target": -0.007448455784469843, + "ronpo/target_abs": 0.0786883682012558, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 360 + }, + { + "drift/chosen_abs": 0.3062785565853119, + "drift/rejected_abs": 0.130979984998703, + "epoch": 0.11601112435439015, + "grad_norm": 10.8125, + "learning_rate": 7.4325240898438e-08, + "logits/chosen": -1.8819477558135986, + "logits/rejected": -1.8994693756103516, + "logps/chosen": -0.2879257798194885, + "logps/rejected": -0.2965686619281769, + "loss": 0.4399788975715637, + "loss/core": 0.4211645722389221, + "loss/preference_sft": 0.296079158782959, + "loss/reference_anchor": 0.3466790020465851, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.0625298023223877, + "rewards/margins": 1.7536706924438477, + "rewards/rejected": 1.3088594675064087, + "ronpo/logit": 0.1753670871257782, + "ronpo/residual": 0.1744088977575302, + "ronpo/residual_abs": 0.22169923782348633, + "ronpo/target": 0.0009581688791513443, + "ronpo/target_abs": 0.07294227182865143, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 365 + }, + { + "drift/chosen_abs": 0.4208388924598694, + "drift/rejected_abs": 0.19213302433490753, + "epoch": 0.11760031783869686, + "grad_norm": 9.0, + "learning_rate": 7.347357813929453e-08, + "logits/chosen": -1.6949617862701416, + "logits/rejected": -1.5868123769760132, + "logps/chosen": -0.28393587470054626, + "logps/rejected": -0.26977697014808655, + "loss": 0.9468614459037781, + "loss/core": 0.8858458399772644, + "loss/preference_sft": 0.2782512903213501, + "loss/reference_anchor": 1.192486047744751, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 4.20812463760376, + "rewards/margins": 2.3017334938049316, + "rewards/rejected": 1.9063911437988281, + "ronpo/logit": 0.23017334938049316, + "ronpo/residual": 0.22761540114879608, + "ronpo/residual_abs": 0.30848538875579834, + "ronpo/target": 0.0025579137727618217, + "ronpo/target_abs": 0.06729411333799362, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 370 + }, + { + "drift/chosen_abs": 0.33695167303085327, + "drift/rejected_abs": 0.15755027532577515, + "epoch": 0.11918951132300358, + "grad_norm": 4.03125, + "learning_rate": 7.261308792741188e-08, + "logits/chosen": -1.9283231496810913, + "logits/rejected": -1.898046851158142, + "logps/chosen": -0.26773491501808167, + "logps/rejected": -0.2634897232055664, + "loss": 0.5323135256767273, + "loss/core": 0.5038132667541504, + "loss/preference_sft": 0.2725459039211273, + "loss/reference_anchor": 0.5427502393722534, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.3691153526306152, + "rewards/margins": 1.7936127185821533, + "rewards/rejected": 1.575502872467041, + "ronpo/logit": 0.1793612688779831, + "ronpo/residual": 0.1783147007226944, + "ronpo/residual_abs": 0.2645108103752136, + "ronpo/target": 0.0010465823579579592, + "ronpo/target_abs": 0.07507561147212982, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 375 + }, + { + "drift/chosen_abs": 0.24347706139087677, + "drift/rejected_abs": 0.0940445065498352, + "epoch": 0.12077870480731029, + "grad_norm": 0.66015625, + "learning_rate": 7.174409385796725e-08, + "logits/chosen": -1.512104868888855, + "logits/rejected": -1.5713586807250977, + "logps/chosen": -0.2921512722969055, + "logps/rejected": -0.288189172744751, + "loss": 0.7772008180618286, + "loss/core": 0.7538732290267944, + "loss/preference_sft": 0.2922476828098297, + "loss/reference_anchor": 0.43732795119285583, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.434725284576416, + "rewards/margins": 1.4957847595214844, + "rewards/rejected": 0.9389405250549316, + "ronpo/logit": 0.1495784968137741, + "ronpo/residual": 0.16179527342319489, + "ronpo/residual_abs": 0.20729383826255798, + "ronpo/target": -0.01221676729619503, + "ronpo/target_abs": 0.0761856958270073, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 380 + }, + { + "drift/chosen_abs": 0.36468371748924255, + "drift/rejected_abs": 0.15189920365810394, + "epoch": 0.12236789829161701, + "grad_norm": 4.6875, + "learning_rate": 7.08669227240909e-08, + "logits/chosen": -1.9026060104370117, + "logits/rejected": -1.8286371231079102, + "logps/chosen": -0.3003026247024536, + "logps/rejected": -0.30434030294418335, + "loss": 0.5205832123756409, + "loss/core": 0.4895382523536682, + "loss/preference_sft": 0.30446332693099976, + "loss/reference_anchor": 0.5904520750045776, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.646836757659912, + "rewards/margins": 2.139817714691162, + "rewards/rejected": 1.5070189237594604, + "ronpo/logit": 0.21398182213306427, + "ronpo/residual": 0.21957902610301971, + "ronpo/residual_abs": 0.2609708309173584, + "ronpo/target": -0.005597242154181004, + "ronpo/target_abs": 0.07350078225135803, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 385 + }, + { + "drift/chosen_abs": 0.18241234123706818, + "drift/rejected_abs": 0.07146935164928436, + "epoch": 0.12395709177592372, + "grad_norm": 2.03125, + "learning_rate": 6.99819043939726e-08, + "logits/chosen": -1.8736292123794556, + "logits/rejected": -1.856080412864685, + "logps/chosen": -0.2814716398715973, + "logps/rejected": -0.2780093252658844, + "loss": 0.18268391489982605, + "loss/core": 0.17545238137245178, + "loss/preference_sft": 0.2792362570762634, + "loss/reference_anchor": 0.1167074590921402, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8227360248565674, + "rewards/margins": 1.1081311702728271, + "rewards/rejected": 0.714604914188385, + "ronpo/logit": 0.11081312596797943, + "ronpo/residual": 0.11074452102184296, + "ronpo/residual_abs": 0.15610924363136292, + "ronpo/target": 6.859861605335027e-05, + "ronpo/target_abs": 0.07130326330661774, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 390 + }, + { + "drift/chosen_abs": 0.23421840369701385, + "drift/rejected_abs": 0.12544074654579163, + "epoch": 0.12554628526023043, + "grad_norm": 144.0, + "learning_rate": 6.908937168681174e-08, + "logits/chosen": -1.8029218912124634, + "logits/rejected": -1.891135811805725, + "logps/chosen": -0.28115424513816833, + "logps/rejected": -0.28048890829086304, + "loss": 0.12526974081993103, + "loss/core": 0.10842245817184448, + "loss/preference_sft": 0.28288403153419495, + "loss/reference_anchor": 0.3086569011211395, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.341761827468872, + "rewards/margins": 1.0884926319122314, + "rewards/rejected": 1.2532694339752197, + "ronpo/logit": 0.10884924978017807, + "ronpo/residual": 0.10868626832962036, + "ronpo/residual_abs": 0.15744046866893768, + "ronpo/target": 0.0001629740872886032, + "ronpo/target_abs": 0.07715004682540894, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 395 + }, + { + "drift/chosen_abs": 0.37465590238571167, + "drift/rejected_abs": 0.11181537806987762, + "epoch": 0.12713547874453715, + "grad_norm": 0.77734375, + "learning_rate": 6.818966024765757e-08, + "logits/chosen": -1.9342272281646729, + "logits/rejected": -1.9193241596221924, + "logps/chosen": -0.2912561595439911, + "logps/rejected": -0.2906317710876465, + "loss": 1.6204612255096436, + "loss/core": 1.5696020126342773, + "loss/preference_sft": 0.2945665419101715, + "loss/reference_anchor": 0.9877260327339172, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 3.7448832988739014, + "rewards/margins": 2.6269583702087402, + "rewards/rejected": 1.117925763130188, + "ronpo/logit": 0.2626958191394806, + "ronpo/residual": 0.27263596653938293, + "ronpo/residual_abs": 0.3709692656993866, + "ronpo/target": -0.009940169751644135, + "ronpo/target_abs": 0.08929356932640076, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 400 + }, + { + "drift/chosen_abs": 0.24711406230926514, + "drift/rejected_abs": 0.12497474998235703, + "epoch": 0.12872467222884387, + "grad_norm": 4.59375, + "learning_rate": 6.728310842118683e-08, + "logits/chosen": -1.7766101360321045, + "logits/rejected": -1.727318525314331, + "logps/chosen": -0.2852860391139984, + "logps/rejected": -0.2805282771587372, + "loss": 0.2622772455215454, + "loss/core": 0.24621637165546417, + "loss/preference_sft": 0.287511944770813, + "loss/reference_anchor": 0.2924651503562927, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.46588134765625, + "rewards/margins": 1.2168688774108887, + "rewards/rejected": 1.2490125894546509, + "ronpo/logit": 0.1216868907213211, + "ronpo/residual": 0.11308858543634415, + "ronpo/residual_abs": 0.18690213561058044, + "ronpo/target": 0.00859830342233181, + "ronpo/target_abs": 0.08837716281414032, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 405 + }, + { + "drift/chosen_abs": 0.15449269115924835, + "drift/rejected_abs": 0.07039482146501541, + "epoch": 0.13031386571315057, + "grad_norm": 0.7421875, + "learning_rate": 6.637005712446621e-08, + "logits/chosen": -1.8721539974212646, + "logits/rejected": -1.9462865591049194, + "logps/chosen": -0.28909945487976074, + "logps/rejected": -0.27632978558540344, + "loss": 0.08583154529333115, + "loss/core": 0.08015327155590057, + "loss/preference_sft": 0.283399373292923, + "loss/reference_anchor": 0.08522568643093109, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.5443317890167236, + "rewards/margins": 0.842542827129364, + "rewards/rejected": 0.7017889022827148, + "ronpo/logit": 0.08425427973270416, + "ronpo/residual": 0.07985297590494156, + "ronpo/residual_abs": 0.14067772030830383, + "ronpo/target": 0.0044013154692947865, + "ronpo/target_abs": 0.07584073394536972, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 410 + }, + { + "drift/chosen_abs": 0.1793230175971985, + "drift/rejected_abs": 0.08416859805583954, + "epoch": 0.1319030591974573, + "grad_norm": 2.109375, + "learning_rate": 6.545084971874738e-08, + "logits/chosen": -1.9961707592010498, + "logits/rejected": -1.9701541662216187, + "logps/chosen": -0.2783016562461853, + "logps/rejected": -0.27948951721191406, + "loss": 0.09949196875095367, + "loss/core": 0.09299661219120026, + "loss/preference_sft": 0.2811213433742523, + "loss/reference_anchor": 0.10179527848958969, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.7914680242538452, + "rewards/margins": 0.9507063031196594, + "rewards/rejected": 0.8407617807388306, + "ronpo/logit": 0.09507062286138535, + "ronpo/residual": 0.08666277676820755, + "ronpo/residual_abs": 0.13639622926712036, + "ronpo/target": 0.008407856337726116, + "ronpo/target_abs": 0.06797736883163452, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 415 + }, + { + "drift/chosen_abs": 0.1825791895389557, + "drift/rejected_abs": 0.07227766513824463, + "epoch": 0.133492252681764, + "grad_norm": 1.140625, + "learning_rate": 6.452583188034274e-08, + "logits/chosen": -1.8393170833587646, + "logits/rejected": -1.8407024145126343, + "logps/chosen": -0.2888352870941162, + "logps/rejected": -0.2917371392250061, + "loss": 0.22071552276611328, + "loss/core": 0.20773322880268097, + "loss/preference_sft": 0.2941259443759918, + "loss/reference_anchor": 0.23023299872875214, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8255313634872437, + "rewards/margins": 1.1031744480133057, + "rewards/rejected": 0.7223570942878723, + "ronpo/logit": 0.11031744629144669, + "ronpo/residual": 0.09504842758178711, + "ronpo/residual_abs": 0.15620620548725128, + "ronpo/target": 0.015269016847014427, + "ronpo/target_abs": 0.07178054749965668, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 420 + }, + { + "drift/chosen_abs": 0.3180219531059265, + "drift/rejected_abs": 0.08384044468402863, + "epoch": 0.1350814461660707, + "grad_norm": 4.5625, + "learning_rate": 6.35953514706309e-08, + "logits/chosen": -1.8810008764266968, + "logits/rejected": -1.9081312417984009, + "logps/chosen": -0.2878539562225342, + "logps/rejected": -0.2854660153388977, + "loss": 1.2196041345596313, + "loss/core": 1.1814826726913452, + "loss/preference_sft": 0.2875927686691284, + "loss/reference_anchor": 0.7336705327033997, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.178842544555664, + "rewards/margins": 2.3408150672912598, + "rewards/rejected": 0.8380276560783386, + "ronpo/logit": 0.23408150672912598, + "ronpo/residual": 0.2306959331035614, + "ronpo/residual_abs": 0.2859199643135071, + "ronpo/target": 0.0033855431247502565, + "ronpo/target_abs": 0.07320918142795563, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 425 + }, + { + "drift/chosen_abs": 0.2660025954246521, + "drift/rejected_abs": 0.11940312385559082, + "epoch": 0.13667063965037743, + "grad_norm": 37.25, + "learning_rate": 6.26597584052401e-08, + "logits/chosen": -1.7275598049163818, + "logits/rejected": -1.7136932611465454, + "logps/chosen": -0.2784407138824463, + "logps/rejected": -0.2639455199241638, + "loss": 0.23619727790355682, + "loss/core": 0.22276940941810608, + "loss/preference_sft": 0.27869319915771484, + "loss/reference_anchor": 0.24068863689899445, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.6599392890930176, + "rewards/margins": 1.4684311151504517, + "rewards/rejected": 1.1915079355239868, + "ronpo/logit": 0.14684312045574188, + "ronpo/residual": 0.14823973178863525, + "ronpo/residual_abs": 0.19733932614326477, + "ronpo/target": -0.0013966044643893838, + "ronpo/target_abs": 0.0798254981637001, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 430 + }, + { + "drift/chosen_abs": 0.259042352437973, + "drift/rejected_abs": 0.1040254458785057, + "epoch": 0.13825983313468415, + "grad_norm": 1008.0, + "learning_rate": 6.171940452245923e-08, + "logits/chosen": -1.948926568031311, + "logits/rejected": -1.890885353088379, + "logps/chosen": -0.2998539209365845, + "logps/rejected": -0.2981581687927246, + "loss": 0.5641499757766724, + "loss/core": 0.5412354469299316, + "loss/preference_sft": 0.31242674589157104, + "loss/reference_anchor": 0.42705029249191284, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.5896012783050537, + "rewards/margins": 1.551031470298767, + "rewards/rejected": 1.0385698080062866, + "ronpo/logit": 0.1551031768321991, + "ronpo/residual": 0.1529240757226944, + "ronpo/residual_abs": 0.19178906083106995, + "ronpo/target": 0.002179093658924103, + "ronpo/target_abs": 0.06712304055690765, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 435 + }, + { + "drift/chosen_abs": 0.16026827692985535, + "drift/rejected_abs": 0.1745794415473938, + "epoch": 0.13984902661899087, + "grad_norm": 25.125, + "learning_rate": 6.0774643450926e-08, + "logits/chosen": -2.0914130210876465, + "logits/rejected": -2.0466763973236084, + "logps/chosen": -0.2668331265449524, + "logps/rejected": -0.2701703906059265, + "loss": 0.4638034403324127, + "loss/core": 0.4482359290122986, + "loss/preference_sft": 0.26497453451156616, + "loss/reference_anchor": 0.28485244512557983, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.6023889780044556, + "rewards/margins": -0.14340558648109436, + "rewards/rejected": 1.745794653892517, + "ronpo/logit": -0.01434055995196104, + "ronpo/residual": -0.01068189274519682, + "ronpo/residual_abs": 0.2119809091091156, + "ronpo/target": -0.00365866394713521, + "ronpo/target_abs": 0.07263807952404022, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 440 + }, + { + "drift/chosen_abs": 0.43698692321777344, + "drift/rejected_abs": 0.24082942306995392, + "epoch": 0.14143822010329757, + "grad_norm": 28.5, + "learning_rate": 5.982583047664151e-08, + "logits/chosen": -1.8886873722076416, + "logits/rejected": -1.890425443649292, + "logps/chosen": -0.27502498030662537, + "logps/rejected": -0.2749865651130676, + "loss": 0.6122740507125854, + "loss/core": 0.5536811947822571, + "loss/preference_sft": 0.27798959612846375, + "loss/reference_anchor": 1.1440602540969849, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.3676042556762695, + "rewards/margins": 1.9618676900863647, + "rewards/rejected": 2.4057364463806152, + "ronpo/logit": 0.19618678092956543, + "ronpo/residual": 0.18561862409114838, + "ronpo/residual_abs": 0.3030679523944855, + "ronpo/target": 0.010568159632384777, + "ronpo/target_abs": 0.0740106850862503, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 445 + }, + { + "drift/chosen_abs": 0.3152461051940918, + "drift/rejected_abs": 0.1808178722858429, + "epoch": 0.1430274135876043, + "grad_norm": 116.5, + "learning_rate": 5.8873322409361756e-08, + "logits/chosen": -1.8608481884002686, + "logits/rejected": -1.8268924951553345, + "logps/chosen": -0.29395729303359985, + "logps/rejected": -0.2936895489692688, + "loss": 0.30116790533065796, + "loss/core": 0.26753267645835876, + "loss/preference_sft": 0.2933436632156372, + "loss/reference_anchor": 0.6433706879615784, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.1521642208099365, + "rewards/margins": 1.3444736003875732, + "rewards/rejected": 1.8076906204223633, + "ronpo/logit": 0.1344473659992218, + "ronpo/residual": 0.1293283849954605, + "ronpo/residual_abs": 0.19568553566932678, + "ronpo/target": 0.005118980538100004, + "ronpo/target_abs": 0.0815843939781189, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 450 + }, + { + "drift/chosen_abs": 0.27072873711586, + "drift/rejected_abs": 0.13820311427116394, + "epoch": 0.144616607071911, + "grad_norm": 16.5, + "learning_rate": 5.7917477448416144e-08, + "logits/chosen": -1.8398069143295288, + "logits/rejected": -1.8747144937515259, + "logps/chosen": -0.29323771595954895, + "logps/rejected": -0.2729089856147766, + "loss": 0.1377260684967041, + "loss/core": 0.12696051597595215, + "loss/preference_sft": 0.28171589970588684, + "loss/reference_anchor": 0.1871393471956253, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.706803321838379, + "rewards/margins": 1.331863284111023, + "rewards/rejected": 1.3749401569366455, + "ronpo/logit": 0.13318632543087006, + "ronpo/residual": 0.12540921568870544, + "ronpo/residual_abs": 0.18542072176933289, + "ronpo/target": 0.007777099963277578, + "ronpo/target_abs": 0.08351351320743561, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 455 + }, + { + "drift/chosen_abs": 0.3840409815311432, + "drift/rejected_abs": 0.16967609524726868, + "epoch": 0.1462058005562177, + "grad_norm": 12.6875, + "learning_rate": 5.695865504800327e-08, + "logits/chosen": -1.6525131464004517, + "logits/rejected": -1.67450749874115, + "logps/chosen": -0.3037227392196655, + "logps/rejected": -0.27811527252197266, + "loss": 0.7146202325820923, + "loss/core": 0.6745911240577698, + "loss/preference_sft": 0.3022574484348297, + "loss/reference_anchor": 0.7703584432601929, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.840378522872925, + "rewards/margins": 2.1443047523498535, + "rewards/rejected": 1.6960737705230713, + "ronpo/logit": 0.21443048119544983, + "ronpo/residual": 0.19126150012016296, + "ronpo/residual_abs": 0.2504216134548187, + "ronpo/target": 0.02316897362470627, + "ronpo/target_abs": 0.07098586112260818, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 460 + }, + { + "drift/chosen_abs": 0.3566533029079437, + "drift/rejected_abs": 0.18750634789466858, + "epoch": 0.14779499404052443, + "grad_norm": 19.625, + "learning_rate": 5.5997215782014984e-08, + "logits/chosen": -1.612074851989746, + "logits/rejected": -1.6693141460418701, + "logps/chosen": -0.31581148505210876, + "logps/rejected": -0.2955818772315979, + "loss": 0.3881073594093323, + "loss/core": 0.3498273491859436, + "loss/preference_sft": 0.31020432710647583, + "loss/reference_anchor": 0.7345792651176453, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 3.5655674934387207, + "rewards/margins": 1.6920642852783203, + "rewards/rejected": 1.87350332736969, + "ronpo/logit": 0.1692064106464386, + "ronpo/residual": 0.15738491714000702, + "ronpo/residual_abs": 0.23218849301338196, + "ronpo/target": 0.011821477673947811, + "ronpo/target_abs": 0.08283017575740814, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 465 + }, + { + "drift/chosen_abs": 0.2089032381772995, + "drift/rejected_abs": 0.1217624768614769, + "epoch": 0.14938418752483115, + "grad_norm": 7.96875, + "learning_rate": 5.503352120843923e-08, + "logits/chosen": -1.8853909969329834, + "logits/rejected": -1.8994200229644775, + "logps/chosen": -0.27790898084640503, + "logps/rejected": -0.26950299739837646, + "loss": 0.09184792637825012, + "loss/core": 0.08480606973171234, + "loss/preference_sft": 0.2741137444972992, + "loss/reference_anchor": 0.11342555284500122, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.0881950855255127, + "rewards/margins": 0.8740318417549133, + "rewards/rejected": 1.2141634225845337, + "ronpo/logit": 0.08740317821502686, + "ronpo/residual": 0.09419047832489014, + "ronpo/residual_abs": 0.16959841549396515, + "ronpo/target": -0.006787297315895557, + "ronpo/target_abs": 0.07587598264217377, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 470 + }, + { + "drift/chosen_abs": 0.3570539653301239, + "drift/rejected_abs": 0.15286162495613098, + "epoch": 0.15097338100913787, + "grad_norm": 2.75, + "learning_rate": 5.406793373339291e-08, + "logits/chosen": -1.8066047430038452, + "logits/rejected": -1.767642617225647, + "logps/chosen": -0.26015806198120117, + "logps/rejected": -0.2632463574409485, + "loss": 0.34153276681900024, + "loss/core": 0.3111182451248169, + "loss/preference_sft": 0.264599084854126, + "loss/reference_anchor": 0.5818300843238831, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.570539951324463, + "rewards/margins": 2.0685620307922363, + "rewards/rejected": 1.5019776821136475, + "ronpo/logit": 0.20685617625713348, + "ronpo/residual": 0.20940890908241272, + "ronpo/residual_abs": 0.2527911961078644, + "ronpo/target": -0.0025527128018438816, + "ronpo/target_abs": 0.07948826253414154, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 475 + }, + { + "drift/chosen_abs": 0.28908276557922363, + "drift/rejected_abs": 0.06378751993179321, + "epoch": 0.15256257449344457, + "grad_norm": 2.3125, + "learning_rate": 5.3100816474835766e-08, + "logits/chosen": -1.830521583557129, + "logits/rejected": -1.8023853302001953, + "logps/chosen": -0.291806161403656, + "logps/rejected": -0.2972370982170105, + "loss": 1.2332208156585693, + "loss/core": 1.1992995738983154, + "loss/preference_sft": 0.29961657524108887, + "loss/reference_anchor": 0.6484619379043579, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.889061450958252, + "rewards/margins": 2.2527663707733154, + "rewards/rejected": 0.6362951397895813, + "ronpo/logit": 0.2252766638994217, + "ronpo/residual": 0.2135426253080368, + "ronpo/residual_abs": 0.28047823905944824, + "ronpo/target": 0.011734018102288246, + "ronpo/target_abs": 0.08371776342391968, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 480 + }, + { + "drift/chosen_abs": 0.28634947538375854, + "drift/rejected_abs": 0.09996393322944641, + "epoch": 0.1541517679777513, + "grad_norm": 4.0, + "learning_rate": 5.2132533126016534e-08, + "logits/chosen": -1.8586909770965576, + "logits/rejected": -1.8007152080535889, + "logps/chosen": -0.29752618074417114, + "logps/rejected": -0.2984181046485901, + "loss": 0.8611629605293274, + "loss/core": 0.83111971616745, + "loss/preference_sft": 0.29279690980911255, + "loss/reference_anchor": 0.5715832710266113, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.8625049591064453, + "rewards/margins": 1.864074945449829, + "rewards/rejected": 0.9984301328659058, + "ronpo/logit": 0.18640747666358948, + "ronpo/residual": 0.19194810092449188, + "ronpo/residual_abs": 0.24163195490837097, + "ronpo/target": -0.0055406298488378525, + "ronpo/target_abs": 0.06841053813695908, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 485 + }, + { + "drift/chosen_abs": 0.2852783799171448, + "drift/rejected_abs": 0.09443080425262451, + "epoch": 0.15574096146205801, + "grad_norm": 196.0, + "learning_rate": 5.116344781870281e-08, + "logits/chosen": -1.823598861694336, + "logits/rejected": -1.7952381372451782, + "logps/chosen": -0.30401426553726196, + "logps/rejected": -0.3061407506465912, + "loss": 0.7646321058273315, + "loss/core": 0.7394935488700867, + "loss/preference_sft": 0.3087760806083679, + "loss/reference_anchor": 0.4718933701515198, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.852783679962158, + "rewards/margins": 1.9092872142791748, + "rewards/rejected": 0.943496823310852, + "ronpo/logit": 0.19092872738838196, + "ronpo/residual": 0.19168706238269806, + "ronpo/residual_abs": 0.24266991019248962, + "ronpo/target": -0.0007583644473925233, + "ronpo/target_abs": 0.07142162322998047, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 490 + }, + { + "drift/chosen_abs": 0.17704372107982635, + "drift/rejected_abs": 0.10839326679706573, + "epoch": 0.1573301549463647, + "grad_norm": 2.140625, + "learning_rate": 5.0193924986246017e-08, + "logits/chosen": -1.7229465246200562, + "logits/rejected": -1.6664669513702393, + "logps/chosen": -0.32384443283081055, + "logps/rejected": -0.31671226024627686, + "loss": 0.11827144771814346, + "loss/core": 0.10038117319345474, + "loss/preference_sft": 0.32083967328071594, + "loss/reference_anchor": 0.32572153210639954, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7700849771499634, + "rewards/margins": 0.6877198219299316, + "rewards/rejected": 1.0823650360107422, + "ronpo/logit": 0.06877198815345764, + "ronpo/residual": 0.07820957899093628, + "ronpo/residual_abs": 0.11904873698949814, + "ronpo/target": -0.009437596425414085, + "ronpo/target_abs": 0.06534653156995773, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 495 + }, + { + "drift/chosen_abs": 0.3221167027950287, + "drift/rejected_abs": 0.16330395638942719, + "epoch": 0.15891934843067143, + "grad_norm": 1.5390625, + "learning_rate": 4.9224329226532834e-08, + "logits/chosen": -1.8693422079086304, + "logits/rejected": -1.8676942586898804, + "logps/chosen": -0.2746179401874542, + "logps/rejected": -0.2810618281364441, + "loss": 0.5076322555541992, + "loss/core": 0.4757213592529297, + "loss/preference_sft": 0.272911012172699, + "loss/reference_anchor": 0.610925555229187, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.2211670875549316, + "rewards/margins": 1.5898401737213135, + "rewards/rejected": 1.6313269138336182, + "ronpo/logit": 0.15898403525352478, + "ronpo/residual": 0.15926262736320496, + "ronpo/residual_abs": 0.23408159613609314, + "ronpo/target": -0.00027860701084136963, + "ronpo/target_abs": 0.07415109127759933, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 500 + }, + { + "drift/chosen_abs": 0.22892192006111145, + "drift/rejected_abs": 0.15709178149700165, + "epoch": 0.16050854191497815, + "grad_norm": 8.5, + "learning_rate": 4.8255025164874966e-08, + "logits/chosen": -1.7380540370941162, + "logits/rejected": -1.744060754776001, + "logps/chosen": -0.3805738389492035, + "logps/rejected": -0.3870905041694641, + "loss": 0.4102400243282318, + "loss/core": 0.392947793006897, + "loss/preference_sft": 0.38436800241470337, + "loss/reference_anchor": 0.30740922689437866, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.2868499755859375, + "rewards/margins": 1.78729248046875, + "rewards/rejected": 0.4995573163032532, + "ronpo/logit": 0.17872926592826843, + "ronpo/residual": 0.1838955581188202, + "ronpo/residual_abs": 0.22993488609790802, + "ronpo/target": -0.005166276823729277, + "ronpo/target_abs": 0.07546278089284897, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 505 + }, + { + "drift/chosen_abs": 0.5437565445899963, + "drift/rejected_abs": 0.19869248569011688, + "epoch": 0.16209773539928488, + "grad_norm": 450.0, + "learning_rate": 4.7286377316888315e-08, + "logits/chosen": -1.8554989099502563, + "logits/rejected": -1.8838293552398682, + "logps/chosen": -0.2770172357559204, + "logps/rejected": -0.2674543261528015, + "loss": 2.1095359325408936, + "loss/core": 2.0345797538757324, + "loss/preference_sft": 0.28003743290901184, + "loss/reference_anchor": 1.471118450164795, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 5.4361066818237305, + "rewards/margins": 3.4505832195281982, + "rewards/rejected": 1.9855238199234009, + "ronpo/logit": 0.3450583517551422, + "ronpo/residual": 0.36223721504211426, + "ronpo/residual_abs": 0.4740299582481384, + "ronpo/target": -0.017178861424326897, + "ronpo/target_abs": 0.0916592925786972, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 510 + }, + { + "drift/chosen_abs": 0.17446205019950867, + "drift/rejected_abs": 0.06303250789642334, + "epoch": 0.16368692888359157, + "grad_norm": 6.0, + "learning_rate": 4.631874995141376e-08, + "logits/chosen": -1.7710046768188477, + "logits/rejected": -1.81719172000885, + "logps/chosen": -0.311026394367218, + "logps/rejected": -0.31932035088539124, + "loss": 0.2285696566104889, + "loss/core": 0.21945735812187195, + "loss/preference_sft": 0.30587515234947205, + "loss/reference_anchor": 0.15165886282920837, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.744620680809021, + "rewards/margins": 1.1147360801696777, + "rewards/rejected": 0.6298844814300537, + "ronpo/logit": 0.11147361993789673, + "ronpo/residual": 0.12036045640707016, + "ronpo/residual_abs": 0.17754912376403809, + "ronpo/target": -0.00888682622462511, + "ronpo/target_abs": 0.08302748203277588, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 515 + }, + { + "drift/chosen_abs": 0.2479497194290161, + "drift/rejected_abs": 0.15151138603687286, + "epoch": 0.1652761223678983, + "grad_norm": 95.5, + "learning_rate": 4.535250695353039e-08, + "logits/chosen": -2.097480058670044, + "logits/rejected": -2.106612205505371, + "logps/chosen": -0.238003209233284, + "logps/rejected": -0.2355692833662033, + "loss": 0.15418457984924316, + "loss/core": 0.14264442026615143, + "loss/preference_sft": 0.23966360092163086, + "loss/reference_anchor": 0.20683690905570984, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.4794869422912598, + "rewards/margins": 0.9650307893753052, + "rewards/rejected": 1.514456033706665, + "ronpo/logit": 0.09650308638811111, + "ronpo/residual": 0.08097958564758301, + "ronpo/residual_abs": 0.1965085119009018, + "ronpo/target": 0.015523498877882957, + "ronpo/target_abs": 0.07920856773853302, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 520 + }, + { + "drift/chosen_abs": 0.2776569724082947, + "drift/rejected_abs": 0.15892525017261505, + "epoch": 0.16686531585220502, + "grad_norm": 54.0, + "learning_rate": 4.438801168771327e-08, + "logits/chosen": -2.0010154247283936, + "logits/rejected": -2.0862135887145996, + "logps/chosen": -0.3032376170158386, + "logps/rejected": -0.29709702730178833, + "loss": 0.501846194267273, + "loss/core": 0.4632810056209564, + "loss/preference_sft": 0.2949376106262207, + "loss/reference_anchor": 0.7418104410171509, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.7760863304138184, + "rewards/margins": 1.1882919073104858, + "rewards/rejected": 1.587794303894043, + "ronpo/logit": 0.11882920563220978, + "ronpo/residual": 0.1192612275481224, + "ronpo/residual_abs": 0.2238401472568512, + "ronpo/target": -0.00043202488450333476, + "ronpo/target_abs": 0.0579981803894043, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 525 + }, + { + "drift/chosen_abs": 0.28078198432922363, + "drift/rejected_abs": 0.10268757492303848, + "epoch": 0.1684545093365117, + "grad_norm": 0.64453125, + "learning_rate": 4.342562686118687e-08, + "logits/chosen": -1.7843738794326782, + "logits/rejected": -1.7553303241729736, + "logps/chosen": -0.2939331531524658, + "logps/rejected": -0.2914874851703644, + "loss": 0.7988781332969666, + "loss/core": 0.7687490582466125, + "loss/preference_sft": 0.2921197712421417, + "loss/reference_anchor": 0.5733689069747925, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.803372383117676, + "rewards/margins": 1.7815815210342407, + "rewards/rejected": 1.0217907428741455, + "ronpo/logit": 0.17815817892551422, + "ronpo/residual": 0.16743174195289612, + "ronpo/residual_abs": 0.23127658665180206, + "ronpo/target": 0.010726441629230976, + "ronpo/target_abs": 0.08158265054225922, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 530 + }, + { + "drift/chosen_abs": 0.22689418494701385, + "drift/rejected_abs": 0.09927350282669067, + "epoch": 0.17004370282081843, + "grad_norm": 3.0625, + "learning_rate": 4.2465714387525845e-08, + "logits/chosen": -1.7310540676116943, + "logits/rejected": -1.8183752298355103, + "logps/chosen": -0.29117313027381897, + "logps/rejected": -0.29093536734580994, + "loss": 0.14975737035274506, + "loss/core": 0.13723108172416687, + "loss/preference_sft": 0.2861308157444, + "loss/reference_anchor": 0.2219126671552658, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.2661190032958984, + "rewards/margins": 1.2744437456130981, + "rewards/rejected": 0.9916753768920898, + "ronpo/logit": 0.12744438648223877, + "ronpo/residual": 0.09695129096508026, + "ronpo/residual_abs": 0.17214377224445343, + "ronpo/target": 0.030493084341287613, + "ronpo/target_abs": 0.08196470886468887, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 535 + }, + { + "drift/chosen_abs": 0.2985483407974243, + "drift/rejected_abs": 0.0866260677576065, + "epoch": 0.17163289630512515, + "grad_norm": 4.0625, + "learning_rate": 4.1508635250553965e-08, + "logits/chosen": -1.7881622314453125, + "logits/rejected": -1.8750550746917725, + "logps/chosen": -0.2710305452346802, + "logps/rejected": -0.26017826795578003, + "loss": 0.8527200818061829, + "loss/core": 0.8249381184577942, + "loss/preference_sft": 0.2635522186756134, + "loss/reference_anchor": 0.5292860269546509, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.984388828277588, + "rewards/margins": 2.1198816299438477, + "rewards/rejected": 0.8645073175430298, + "ronpo/logit": 0.211988165974617, + "ronpo/residual": 0.23462891578674316, + "ronpo/residual_abs": 0.26362109184265137, + "ronpo/target": -0.022640716284513474, + "ronpo/target_abs": 0.0714339092373848, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 540 + }, + { + "drift/chosen_abs": 0.22164639830589294, + "drift/rejected_abs": 0.10531902313232422, + "epoch": 0.17322208978943188, + "grad_norm": 1.546875, + "learning_rate": 4.0554749368592995e-08, + "logits/chosen": -1.8922348022460938, + "logits/rejected": -1.908355712890625, + "logps/chosen": -0.28418028354644775, + "logps/rejected": -0.2719399929046631, + "loss": 0.0991106778383255, + "loss/core": 0.08887199312448502, + "loss/preference_sft": 0.27457016706466675, + "loss/reference_anchor": 0.1773165613412857, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.215404748916626, + "rewards/margins": 1.1632106304168701, + "rewards/rejected": 1.0521941184997559, + "ronpo/logit": 0.11632106453180313, + "ronpo/residual": 0.11025375127792358, + "ronpo/residual_abs": 0.15504197776317596, + "ronpo/target": 0.006067323964089155, + "ronpo/target_abs": 0.060761380940675735, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 545 + }, + { + "drift/chosen_abs": 0.25841450691223145, + "drift/rejected_abs": 0.06605127453804016, + "epoch": 0.17481128327373857, + "grad_norm": 106.5, + "learning_rate": 3.960441545911204e-08, + "logits/chosen": -1.757881760597229, + "logits/rejected": -1.7778260707855225, + "logps/chosen": -0.2987707555294037, + "logps/rejected": -0.2908412516117096, + "loss": 0.6568024754524231, + "loss/core": 0.6380494832992554, + "loss/preference_sft": 0.2974953055381775, + "loss/reference_anchor": 0.3453090488910675, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.58390474319458, + "rewards/margins": 1.9925460815429688, + "rewards/rejected": 0.5913587808609009, + "ronpo/logit": 0.1992546170949936, + "ronpo/residual": 0.18910114467144012, + "ronpo/residual_abs": 0.2484695166349411, + "ronpo/target": 0.010153469629585743, + "ronpo/target_abs": 0.08087723702192307, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 550 + }, + { + "drift/chosen_abs": 0.1866646111011505, + "drift/rejected_abs": 0.10433971881866455, + "epoch": 0.1764004767580453, + "grad_norm": 24.75, + "learning_rate": 3.8657990903828655e-08, + "logits/chosen": -1.8334095478057861, + "logits/rejected": -1.8890628814697266, + "logps/chosen": -0.29963818192481995, + "logps/rejected": -0.29799216985702515, + "loss": 0.08355274796485901, + "loss/core": 0.07509630918502808, + "loss/preference_sft": 0.30388861894607544, + "loss/reference_anchor": 0.13874009251594543, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.8640848398208618, + "rewards/margins": 0.8224803805351257, + "rewards/rejected": 1.0416042804718018, + "ronpo/logit": 0.08224803954362869, + "ronpo/residual": 0.08422321081161499, + "ronpo/residual_abs": 0.1697530597448349, + "ronpo/target": -0.0019751801155507565, + "ronpo/target_abs": 0.09187708050012589, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 555 + }, + { + "drift/chosen_abs": 0.23153360188007355, + "drift/rejected_abs": 0.13602390885353088, + "epoch": 0.17798967024235202, + "grad_norm": 7.09375, + "learning_rate": 3.7715831614312174e-08, + "logits/chosen": -1.848433256149292, + "logits/rejected": -1.8488585948944092, + "logps/chosen": -0.29612210392951965, + "logps/rejected": -0.29059985280036926, + "loss": 0.05996299535036087, + "loss/core": 0.051027536392211914, + "loss/preference_sft": 0.2972286343574524, + "loss/reference_anchor": 0.14898619055747986, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.315253734588623, + "rewards/margins": 0.9552527666091919, + "rewards/rejected": 1.3600012063980103, + "ronpo/logit": 0.09552527964115143, + "ronpo/residual": 0.09656206518411636, + "ronpo/residual_abs": 0.16236624121665955, + "ronpo/target": -0.001036791130900383, + "ronpo/target_abs": 0.08028535544872284, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 560 + }, + { + "drift/chosen_abs": 0.13719969987869263, + "drift/rejected_abs": 0.06547123938798904, + "epoch": 0.1795788637266587, + "grad_norm": 16.75, + "learning_rate": 3.67782918981399e-08, + "logits/chosen": -1.8791453838348389, + "logits/rejected": -1.8915935754776, + "logps/chosen": -0.25503629446029663, + "logps/rejected": -0.2641448974609375, + "loss": 0.04704626649618149, + "loss/core": 0.04318038374185562, + "loss/preference_sft": 0.2602216601371765, + "loss/reference_anchor": 0.05129551887512207, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.3719971179962158, + "rewards/margins": 0.7177979350090027, + "rewards/rejected": 0.6541991829872131, + "ronpo/logit": 0.07177979499101639, + "ronpo/residual": 0.06935735046863556, + "ronpo/residual_abs": 0.13249507546424866, + "ronpo/target": 0.00242243567481637, + "ronpo/target_abs": 0.08676055818796158, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 565 + }, + { + "drift/chosen_abs": 0.15917405486106873, + "drift/rejected_abs": 0.12015297263860703, + "epoch": 0.18116805721096543, + "grad_norm": 0.62890625, + "learning_rate": 3.584572432565648e-08, + "logits/chosen": -1.7565100193023682, + "logits/rejected": -1.7765508890151978, + "logps/chosen": -0.36137717962265015, + "logps/rejected": -0.29022398591041565, + "loss": 0.09365180879831314, + "loss/core": 0.0859026312828064, + "loss/preference_sft": 0.2855459153652191, + "loss/reference_anchor": 0.12642887234687805, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.5901105403900146, + "rewards/margins": 0.3890073299407959, + "rewards/rejected": 1.2011032104492188, + "ronpo/logit": 0.03890075162053108, + "ronpo/residual": 0.04145831614732742, + "ronpo/residual_abs": 0.14707839488983154, + "ronpo/target": -0.00255756638944149, + "ronpo/target_abs": 0.08257381618022919, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 570 + }, + { + "drift/chosen_abs": 0.2945803999900818, + "drift/rejected_abs": 0.22317442297935486, + "epoch": 0.18275725069527216, + "grad_norm": 161.0, + "learning_rate": 3.491847959738672e-08, + "logits/chosen": -1.6871364116668701, + "logits/rejected": -1.653978943824768, + "logps/chosen": -0.27193376421928406, + "logps/rejected": -0.2878838777542114, + "loss": 0.960303008556366, + "loss/core": 0.9258346557617188, + "loss/preference_sft": 0.28949350118637085, + "loss/reference_anchor": 0.6604174375534058, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.9458041191101074, + "rewards/margins": 0.7202137112617493, + "rewards/rejected": 2.225590467453003, + "ronpo/logit": 0.07202135026454926, + "ronpo/residual": 0.07455171644687653, + "ronpo/residual_abs": 0.29728493094444275, + "ronpo/target": -0.002530353609472513, + "ronpo/target_abs": 0.07750637829303741, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 575 + }, + { + "drift/chosen_abs": 0.2119152992963791, + "drift/rejected_abs": 0.1600765585899353, + "epoch": 0.18434644417957885, + "grad_norm": 1.546875, + "learning_rate": 3.3996906412151415e-08, + "logits/chosen": -1.7865766286849976, + "logits/rejected": -1.7664226293563843, + "logps/chosen": -0.28624483942985535, + "logps/rejected": -0.29210740327835083, + "loss": 0.2576879560947418, + "loss/core": 0.2448233664035797, + "loss/preference_sft": 0.3036908507347107, + "loss/reference_anchor": 0.2269231379032135, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.117682933807373, + "rewards/margins": 0.5200752019882202, + "rewards/rejected": 1.5976078510284424, + "ronpo/logit": 0.0520075187087059, + "ronpo/residual": 0.04997884854674339, + "ronpo/residual_abs": 0.2069791555404663, + "ronpo/target": 0.002028673654422164, + "ronpo/target_abs": 0.08073200285434723, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 580 + }, + { + "drift/chosen_abs": 0.2667069733142853, + "drift/rejected_abs": 0.08909569680690765, + "epoch": 0.18593563766388557, + "grad_norm": 9.625, + "learning_rate": 3.3081351335935947e-08, + "logits/chosen": -1.7589290142059326, + "logits/rejected": -1.7354164123535156, + "logps/chosen": -0.276340514421463, + "logps/rejected": -0.2858516573905945, + "loss": 0.8393601179122925, + "loss/core": 0.8143278360366821, + "loss/preference_sft": 0.2846840023994446, + "loss/reference_anchor": 0.4721766412258148, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.6665663719177246, + "rewards/margins": 1.7798312902450562, + "rewards/rejected": 0.8867348432540894, + "ronpo/logit": 0.1779831200838089, + "ronpo/residual": 0.18403668701648712, + "ronpo/residual_abs": 0.23977021872997284, + "ronpo/target": -0.006053552031517029, + "ronpo/target_abs": 0.08337777107954025, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 585 + }, + { + "drift/chosen_abs": 0.20994123816490173, + "drift/rejected_abs": 0.10977955162525177, + "epoch": 0.1875248311481923, + "grad_norm": 32.25, + "learning_rate": 3.2172158671561e-08, + "logits/chosen": -1.847988486289978, + "logits/rejected": -1.746583342552185, + "logps/chosen": -0.27806663513183594, + "logps/rejected": -0.28961890935897827, + "loss": 0.10898953676223755, + "loss/core": 0.09721212089061737, + "loss/preference_sft": 0.27743273973464966, + "loss/reference_anchor": 0.2078050822019577, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.09867525100708, + "rewards/margins": 1.0014612674713135, + "rewards/rejected": 1.0972139835357666, + "ronpo/logit": 0.10014612972736359, + "ronpo/residual": 0.0981498658657074, + "ronpo/residual_abs": 0.16482563316822052, + "ronpo/target": 0.0019962673541158438, + "ronpo/target_abs": 0.08062820136547089, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 590 + }, + { + "drift/chosen_abs": 0.4715874195098877, + "drift/rejected_abs": 0.19329656660556793, + "epoch": 0.18911402463249902, + "grad_norm": 384.0, + "learning_rate": 3.126967032920439e-08, + "logits/chosen": -1.7966649532318115, + "logits/rejected": -1.8083879947662354, + "logps/chosen": -0.277448832988739, + "logps/rejected": -0.2569620609283447, + "loss": 1.5912710428237915, + "loss/core": 1.5417354106903076, + "loss/preference_sft": 0.2574903070926666, + "loss/reference_anchor": 0.9649633169174194, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.715729236602783, + "rewards/margins": 2.7836599349975586, + "rewards/rejected": 1.932069182395935, + "ronpo/logit": 0.2783660292625427, + "ronpo/residual": 0.29117247462272644, + "ronpo/residual_abs": 0.3995015025138855, + "ronpo/target": -0.01280647236853838, + "ronpo/target_abs": 0.08523674309253693, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 595 + }, + { + "drift/chosen_abs": 0.23165254294872284, + "drift/rejected_abs": 0.14367060363292694, + "epoch": 0.1907032181168057, + "grad_norm": 6.28125, + "learning_rate": 3.037422569782264e-08, + "logits/chosen": -1.9908279180526733, + "logits/rejected": -1.9903548955917358, + "logps/chosen": -0.2832784950733185, + "logps/rejected": -0.28507620096206665, + "loss": 0.03948095068335533, + "loss/core": 0.02752791717648506, + "loss/preference_sft": 0.2857871651649475, + "loss/reference_anchor": 0.21048197150230408, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.3162155151367188, + "rewards/margins": 0.8797648549079895, + "rewards/rejected": 1.4364506006240845, + "ronpo/logit": 0.08797647804021835, + "ronpo/residual": 0.0719924122095108, + "ronpo/residual_abs": 0.10735388845205307, + "ronpo/target": 0.015984075143933296, + "ronpo/target_abs": 0.0789966955780983, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 600 + }, + { + "drift/chosen_abs": 0.30132848024368286, + "drift/rejected_abs": 0.12802016735076904, + "epoch": 0.19229241160111243, + "grad_norm": 7.125, + "learning_rate": 2.9486161517520596e-08, + "logits/chosen": -1.7950258255004883, + "logits/rejected": -1.8050203323364258, + "logps/chosen": -0.24900193512439728, + "logps/rejected": -0.25392305850982666, + "loss": 0.19868217408657074, + "loss/core": 0.17862369120121002, + "loss/preference_sft": 0.24964216351509094, + "loss/reference_anchor": 0.3762053847312927, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.0126354694366455, + "rewards/margins": 1.741882562637329, + "rewards/rejected": 1.2707531452178955, + "ronpo/logit": 0.17418824136257172, + "ronpo/residual": 0.17219789326190948, + "ronpo/residual_abs": 0.23744669556617737, + "ronpo/target": 0.0019903441425412893, + "ronpo/target_abs": 0.0929965078830719, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 605 + }, + { + "drift/chosen_abs": 0.21064424514770508, + "drift/rejected_abs": 0.06865555047988892, + "epoch": 0.19388160508541916, + "grad_norm": 2.1875, + "learning_rate": 2.8605811752917298e-08, + "logits/chosen": -1.791255235671997, + "logits/rejected": -1.8882793188095093, + "logps/chosen": -0.28656139969825745, + "logps/rejected": -0.28957584500312805, + "loss": 0.46913671493530273, + "loss/core": 0.4557432532310486, + "loss/preference_sft": 0.29006972908973694, + "loss/reference_anchor": 0.23886322975158691, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.106372356414795, + "rewards/margins": 1.4200407266616821, + "rewards/rejected": 0.6863317489624023, + "ronpo/logit": 0.14200405776500702, + "ronpo/residual": 0.15059545636177063, + "ronpo/residual_abs": 0.2058655023574829, + "ronpo/target": -0.00859138835221529, + "ronpo/target_abs": 0.08471350371837616, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 610 + }, + { + "drift/chosen_abs": 0.20590801537036896, + "drift/rejected_abs": 0.12859569489955902, + "epoch": 0.19547079856972585, + "grad_norm": 47.5, + "learning_rate": 2.7733507467555527e-08, + "logits/chosen": -1.9091287851333618, + "logits/rejected": -1.945540189743042, + "logps/chosen": -0.2883964776992798, + "logps/rejected": -0.28995269536972046, + "loss": 0.06815268099308014, + "loss/core": 0.054342083632946014, + "loss/preference_sft": 0.28977853059768677, + "loss/reference_anchor": 0.24723413586616516, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.0582265853881836, + "rewards/margins": 0.7743476629257202, + "rewards/rejected": 1.283879041671753, + "ronpo/logit": 0.07743476331233978, + "ronpo/residual": 0.08360174298286438, + "ronpo/residual_abs": 0.13559815287590027, + "ronpo/target": -0.006166977342218161, + "ronpo/target_abs": 0.06952059268951416, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 615 + }, + { + "drift/chosen_abs": 0.1789097934961319, + "drift/rejected_abs": 0.08338747918605804, + "epoch": 0.19705999205403257, + "grad_norm": 0.63671875, + "learning_rate": 2.686957669940242e-08, + "logits/chosen": -1.8341875076293945, + "logits/rejected": -1.810105562210083, + "logps/chosen": -0.303658127784729, + "logps/rejected": -0.2884838283061981, + "loss": 0.13995635509490967, + "loss/core": 0.13397568464279175, + "loss/preference_sft": 0.2918221652507782, + "loss/reference_anchor": 0.09043111652135849, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7879546880722046, + "rewards/margins": 0.9541919827461243, + "rewards/rejected": 0.8337627649307251, + "ronpo/logit": 0.09541921317577362, + "ronpo/residual": 0.10469168424606323, + "ronpo/residual_abs": 0.16267558932304382, + "ronpo/target": -0.00927248690277338, + "ronpo/target_abs": 0.0726732388138771, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 620 + }, + { + "drift/chosen_abs": 0.34457817673683167, + "drift/rejected_abs": 0.14021232724189758, + "epoch": 0.1986491855383393, + "grad_norm": 2.484375, + "learning_rate": 2.6014344337487702e-08, + "logits/chosen": -1.959282636642456, + "logits/rejected": -1.9119141101837158, + "logps/chosen": -0.3181460499763489, + "logps/rejected": -0.2812593877315521, + "loss": 1.1086939573287964, + "loss/core": 1.0717629194259644, + "loss/preference_sft": 0.3202729821205139, + "loss/reference_anchor": 0.7065903544425964, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.445781707763672, + "rewards/margins": 2.05183482170105, + "rewards/rejected": 1.393946886062622, + "ronpo/logit": 0.2051835060119629, + "ronpo/residual": 0.19359295070171356, + "ronpo/residual_abs": 0.29983606934547424, + "ronpo/target": 0.011590538546442986, + "ronpo/target_abs": 0.07752411812543869, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 625 + }, + { + "drift/chosen_abs": 0.28649866580963135, + "drift/rejected_abs": 0.10617027431726456, + "epoch": 0.20023837902264602, + "grad_norm": 26.125, + "learning_rate": 2.51681319997262e-08, + "logits/chosen": -1.5398986339569092, + "logits/rejected": -1.5675451755523682, + "logps/chosen": -0.2831787168979645, + "logps/rejected": -0.2901644706726074, + "loss": 0.3452921509742737, + "loss/core": 0.33066895604133606, + "loss/preference_sft": 0.2819228768348694, + "loss/reference_anchor": 0.26427116990089417, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.864680051803589, + "rewards/margins": 1.8066120147705078, + "rewards/rejected": 1.0580682754516602, + "ronpo/logit": 0.1806611865758896, + "ronpo/residual": 0.1922551691532135, + "ronpo/residual_abs": 0.23666691780090332, + "ronpo/target": -0.011593970470130444, + "ronpo/target_abs": 0.07857140153646469, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 630 + }, + { + "drift/chosen_abs": 0.15038815140724182, + "drift/rejected_abs": 0.0819171741604805, + "epoch": 0.2018275725069527, + "grad_norm": 2.234375, + "learning_rate": 2.4331257911970626e-08, + "logits/chosen": -1.8283817768096924, + "logits/rejected": -1.9530715942382812, + "logps/chosen": -0.27565374970436096, + "logps/rejected": -0.2725347578525543, + "loss": 0.026056716218590736, + "loss/core": 0.02213389240205288, + "loss/preference_sft": 0.27325448393821716, + "loss/reference_anchor": 0.051131002604961395, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 1.5036888122558594, + "rewards/margins": 0.6845813989639282, + "rewards/rejected": 0.8191075325012207, + "ronpo/logit": 0.06845813244581223, + "ronpo/residual": 0.06282701343297958, + "ronpo/residual_abs": 0.11277289688587189, + "ronpo/target": 0.00563112273812294, + "ronpo/target_abs": 0.07500731945037842, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 635 + }, + { + "drift/chosen_abs": 0.31310132145881653, + "drift/rejected_abs": 0.18400482833385468, + "epoch": 0.20341676599125944, + "grad_norm": 322.0, + "learning_rate": 2.350403678833976e-08, + "logits/chosen": -1.6889979839324951, + "logits/rejected": -1.654799222946167, + "logps/chosen": -0.29637211561203003, + "logps/rejected": -0.3162195682525635, + "loss": 0.19557903707027435, + "loss/core": 0.16963447630405426, + "loss/preference_sft": 0.3104371428489685, + "loss/reference_anchor": 0.4878477156162262, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.129647731781006, + "rewards/margins": 1.2897276878356934, + "rewards/rejected": 1.8399202823638916, + "ronpo/logit": 0.12897278368473053, + "ronpo/residual": 0.11268714815378189, + "ronpo/residual_abs": 0.1901034414768219, + "ronpo/target": 0.016285624355077744, + "ronpo/target_abs": 0.08053125441074371, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 640 + }, + { + "drift/chosen_abs": 0.4701056480407715, + "drift/rejected_abs": 0.23759059607982635, + "epoch": 0.20500595947556616, + "grad_norm": 266.0, + "learning_rate": 2.2686779712867315e-08, + "logits/chosen": -1.9577453136444092, + "logits/rejected": -1.9497134685516357, + "logps/chosen": -0.23990201950073242, + "logps/rejected": -0.25109779834747314, + "loss": 1.5428487062454224, + "loss/core": 1.4728646278381348, + "loss/preference_sft": 0.24962405860424042, + "loss/reference_anchor": 1.3747169971466064, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.700854778289795, + "rewards/margins": 2.325613498687744, + "rewards/rejected": 2.375241756439209, + "ronpo/logit": 0.23256132006645203, + "ronpo/residual": 0.2318817675113678, + "ronpo/residual_abs": 0.38768142461776733, + "ronpo/target": 0.0006795950466766953, + "ronpo/target_abs": 0.06739381700754166, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 645 + }, + { + "drift/chosen_abs": 0.28680965304374695, + "drift/rejected_abs": 0.09994098544120789, + "epoch": 0.20659515295987285, + "grad_norm": 5.78125, + "learning_rate": 2.1879794022516e-08, + "logits/chosen": -1.7325706481933594, + "logits/rejected": -1.8033367395401, + "logps/chosen": -0.2998145818710327, + "logps/rejected": -0.29500865936279297, + "loss": 0.3449249863624573, + "loss/core": 0.3255670964717865, + "loss/preference_sft": 0.29536116123199463, + "loss/reference_anchor": 0.35762161016464233, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.8669979572296143, + "rewards/margins": 1.875012993812561, + "rewards/rejected": 0.991985023021698, + "ronpo/logit": 0.18750128149986267, + "ronpo/residual": 0.18114595115184784, + "ronpo/residual_abs": 0.23968927562236786, + "ronpo/target": 0.00635533919557929, + "ronpo/target_abs": 0.07919994741678238, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 650 + }, + { + "drift/chosen_abs": 0.4130421280860901, + "drift/rejected_abs": 0.10792286694049835, + "epoch": 0.20818434644417957, + "grad_norm": 4.8125, + "learning_rate": 2.1083383191600673e-08, + "logits/chosen": -1.7494007349014282, + "logits/rejected": -1.7612907886505127, + "logps/chosen": -0.3074556589126587, + "logps/rejected": -0.3002517521381378, + "loss": 1.3415347337722778, + "loss/core": 1.3003575801849365, + "loss/preference_sft": 0.30100753903388977, + "loss/reference_anchor": 0.793440580368042, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.130122184753418, + "rewards/margins": 3.0868916511535645, + "rewards/rejected": 1.0432310104370117, + "ronpo/logit": 0.3086891770362854, + "ronpo/residual": 0.31742364168167114, + "ronpo/residual_abs": 0.3702516555786133, + "ronpo/target": -0.008734484203159809, + "ronpo/target_abs": 0.08217312395572662, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 655 + }, + { + "drift/chosen_abs": 0.27077609300613403, + "drift/rejected_abs": 0.12126465141773224, + "epoch": 0.2097735399284863, + "grad_norm": 5.0, + "learning_rate": 2.029784671766404e-08, + "logits/chosen": -1.7809921503067017, + "logits/rejected": -1.7488667964935303, + "logps/chosen": -0.2880242168903351, + "logps/rejected": -0.2796364724636078, + "loss": 0.36306077241897583, + "loss/core": 0.33687418699264526, + "loss/preference_sft": 0.28567779064178467, + "loss/reference_anchor": 0.4951632022857666, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.7072110176086426, + "rewards/margins": 1.4951145648956299, + "rewards/rejected": 1.2120963335037231, + "ronpo/logit": 0.1495114415884018, + "ronpo/residual": 0.14324429631233215, + "ronpo/residual_abs": 0.19393035769462585, + "ronpo/target": 0.006267151795327663, + "ronpo/target_abs": 0.0730602815747261, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 660 + }, + { + "drift/chosen_abs": 0.34274253249168396, + "drift/rejected_abs": 0.11544802039861679, + "epoch": 0.21136273341279302, + "grad_norm": 102.5, + "learning_rate": 1.9523480008847854e-08, + "logits/chosen": -1.725452184677124, + "logits/rejected": -1.6947933435440063, + "logps/chosen": -0.2793700695037842, + "logps/rejected": -0.27260226011276245, + "loss": 0.6120738387107849, + "loss/core": 0.5833312273025513, + "loss/preference_sft": 0.27295297384262085, + "loss/reference_anchor": 0.5475563406944275, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.4263482093811035, + "rewards/margins": 2.2736732959747314, + "rewards/rejected": 1.152674913406372, + "ronpo/logit": 0.2273673117160797, + "ronpo/residual": 0.21752388775348663, + "ronpo/residual_abs": 0.290507972240448, + "ronpo/target": 0.00984344445168972, + "ronpo/target_abs": 0.07925274968147278, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 665 + }, + { + "drift/chosen_abs": 0.17593860626220703, + "drift/rejected_abs": 0.08771608769893646, + "epoch": 0.2129519268970997, + "grad_norm": 2.640625, + "learning_rate": 1.8760574272802e-08, + "logits/chosen": -2.0317234992980957, + "logits/rejected": -1.96999990940094, + "logps/chosen": -0.2675436735153198, + "logps/rejected": -0.29036182165145874, + "loss": 0.06669995933771133, + "loss/core": 0.05987640097737312, + "loss/preference_sft": 0.29216253757476807, + "loss/reference_anchor": 0.1072549819946289, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7587525844573975, + "rewards/margins": 0.8839167356491089, + "rewards/rejected": 0.8748357892036438, + "ronpo/logit": 0.08839168399572372, + "ronpo/residual": 0.08926811814308167, + "ronpo/residual_abs": 0.1414795219898224, + "ronpo/target": -0.0008764358353801072, + "ronpo/target_abs": 0.07178516685962677, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 670 + }, + { + "drift/chosen_abs": 0.19702181220054626, + "drift/rejected_abs": 0.10953919589519501, + "epoch": 0.21454112038140644, + "grad_norm": 5.375, + "learning_rate": 1.8009416407173257e-08, + "logits/chosen": -1.8570398092269897, + "logits/rejected": -1.8211997747421265, + "logps/chosen": -0.2882757782936096, + "logps/rejected": -0.29082340002059937, + "loss": 0.06841497123241425, + "loss/core": 0.06168912723660469, + "loss/preference_sft": 0.28591659665107727, + "loss/reference_anchor": 0.10592524707317352, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.9673035144805908, + "rewards/margins": 0.8742406964302063, + "rewards/rejected": 1.0930628776550293, + "ronpo/logit": 0.08742406964302063, + "ronpo/residual": 0.08799617737531662, + "ronpo/residual_abs": 0.14206573367118835, + "ronpo/target": -0.0005720962071791291, + "ronpo/target_abs": 0.0792655348777771, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 675 + }, + { + "drift/chosen_abs": 0.24524518847465515, + "drift/rejected_abs": 0.13547000288963318, + "epoch": 0.21613031386571316, + "grad_norm": 7.53125, + "learning_rate": 1.7270288891714813e-08, + "logits/chosen": -1.7146110534667969, + "logits/rejected": -1.7872194051742554, + "logps/chosen": -0.2838783264160156, + "logps/rejected": -0.27856138348579407, + "loss": 0.13877931237220764, + "loss/core": 0.1273031383752823, + "loss/preference_sft": 0.28666257858276367, + "loss/reference_anchor": 0.2008572518825531, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.4522125720977783, + "rewards/margins": 1.1872022151947021, + "rewards/rejected": 1.2650099992752075, + "ronpo/logit": 0.11872021853923798, + "ronpo/residual": 0.1270017921924591, + "ronpo/residual_abs": 0.1994335949420929, + "ronpo/target": -0.00828153919428587, + "ronpo/target_abs": 0.07877428829669952, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 680 + }, + { + "drift/chosen_abs": 0.2916144132614136, + "drift/rejected_abs": 0.11980894953012466, + "epoch": 0.21771950735001985, + "grad_norm": 8.9375, + "learning_rate": 1.6543469682057103e-08, + "logits/chosen": -1.750420331954956, + "logits/rejected": -1.7389971017837524, + "logps/chosen": -0.2631601393222809, + "logps/rejected": -0.2538334131240845, + "loss": 0.27099448442459106, + "loss/core": 0.25824078917503357, + "loss/preference_sft": 0.2480827122926712, + "loss/reference_anchor": 0.23026630282402039, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.9157261848449707, + "rewards/margins": 1.7208629846572876, + "rewards/rejected": 1.1948630809783936, + "ronpo/logit": 0.17208631336688995, + "ronpo/residual": 0.16664986312389374, + "ronpo/residual_abs": 0.20796065032482147, + "ronpo/target": 0.005436430685222149, + "ronpo/target_abs": 0.07201899588108063, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 685 + }, + { + "drift/chosen_abs": 0.2702964246273041, + "drift/rejected_abs": 0.07891695201396942, + "epoch": 0.21930870083432658, + "grad_norm": 1.484375, + "learning_rate": 1.582923210518014e-08, + "logits/chosen": -1.795771598815918, + "logits/rejected": -1.7635061740875244, + "logps/chosen": -0.29897016286849976, + "logps/rejected": -0.2939114570617676, + "loss": 0.30770987272262573, + "loss/core": 0.2953275740146637, + "loss/preference_sft": 0.28765055537223816, + "loss/reference_anchor": 0.21888060867786407, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.7028443813323975, + "rewards/margins": 1.9148327112197876, + "rewards/rejected": 0.788011908531189, + "ronpo/logit": 0.191483274102211, + "ronpo/residual": 0.18644031882286072, + "ronpo/residual_abs": 0.24509191513061523, + "ronpo/target": 0.0050429413095116615, + "ronpo/target_abs": 0.08253506571054459, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 690 + }, + { + "drift/chosen_abs": 0.19787338376045227, + "drift/rejected_abs": 0.1597171276807785, + "epoch": 0.2208978943186333, + "grad_norm": 3.171875, + "learning_rate": 1.5127844756626436e-08, + "logits/chosen": -1.7620786428451538, + "logits/rejected": -1.723760962486267, + "logps/chosen": -0.3214377760887146, + "logps/rejected": -0.3080599308013916, + "loss": 0.4889402985572815, + "loss/core": 0.47200194001197815, + "loss/preference_sft": 0.32001546025276184, + "loss/reference_anchor": 0.30676406621932983, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.9777981042861938, + "rewards/margins": 0.40713924169540405, + "rewards/rejected": 1.570658802986145, + "ronpo/logit": 0.040713924914598465, + "ronpo/residual": 0.03379645571112633, + "ronpo/residual_abs": 0.2298135757446289, + "ronpo/target": 0.0069174692034721375, + "ronpo/target_abs": 0.08368168771266937, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 695 + }, + { + "drift/chosen_abs": 0.267596572637558, + "drift/rejected_abs": 0.12422071397304535, + "epoch": 0.22248708780294002, + "grad_norm": 0.9296875, + "learning_rate": 1.4439571399493145e-08, + "logits/chosen": -1.9247478246688843, + "logits/rejected": -1.9092788696289062, + "logps/chosen": -0.30226415395736694, + "logps/rejected": -0.28551197052001953, + "loss": 0.23500370979309082, + "loss/core": 0.21412146091461182, + "loss/preference_sft": 0.2953455150127411, + "loss/reference_anchor": 0.3881102502346039, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.675750732421875, + "rewards/margins": 1.4344958066940308, + "rewards/rejected": 1.2412550449371338, + "ronpo/logit": 0.1434495896100998, + "ronpo/residual": 0.15175478160381317, + "ronpo/residual_abs": 0.19151520729064941, + "ronpo/target": -0.008305169641971588, + "ronpo/target_abs": 0.06857885420322418, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 700 + }, + { + "drift/chosen_abs": 0.19850663840770721, + "drift/rejected_abs": 0.11235801875591278, + "epoch": 0.22407628128724671, + "grad_norm": 8.1875, + "learning_rate": 1.3764670865241557e-08, + "logits/chosen": -1.8514108657836914, + "logits/rejected": -1.7679784297943115, + "logps/chosen": -0.2819150388240814, + "logps/rejected": -0.28714990615844727, + "loss": 0.10464034229516983, + "loss/core": 0.09731554239988327, + "loss/preference_sft": 0.27761921286582947, + "loss/reference_anchor": 0.11873352527618408, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9850664138793945, + "rewards/margins": 0.8684585690498352, + "rewards/rejected": 1.1166080236434937, + "ronpo/logit": 0.08684585243463516, + "ronpo/residual": 0.09302155673503876, + "ronpo/residual_abs": 0.16833505034446716, + "ronpo/target": -0.006175708491355181, + "ronpo/target_abs": 0.08478256314992905, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 705 + }, + { + "drift/chosen_abs": 0.16192565858364105, + "drift/rejected_abs": 0.0836598128080368, + "epoch": 0.22566547477155344, + "grad_norm": 7.28125, + "learning_rate": 1.310339695636118e-08, + "logits/chosen": -2.020831346511841, + "logits/rejected": -2.0246784687042236, + "logps/chosen": -0.2766228914260864, + "logps/rejected": -0.28187552094459534, + "loss": 0.03390822932124138, + "loss/core": 0.028157252818346024, + "loss/preference_sft": 0.2825476825237274, + "loss/reference_anchor": 0.08676482737064362, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.6182422637939453, + "rewards/margins": 0.7829471826553345, + "rewards/rejected": 0.8352952003479004, + "ronpo/logit": 0.07829473167657852, + "ronpo/residual": 0.0662197396159172, + "ronpo/residual_abs": 0.11468762159347534, + "ronpo/target": 0.012074981816112995, + "ronpo/target_abs": 0.072537861764431, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 710 + }, + { + "drift/chosen_abs": 0.46716541051864624, + "drift/rejected_abs": 0.17989280819892883, + "epoch": 0.22725466825586016, + "grad_norm": 632.0, + "learning_rate": 1.245599835092504e-08, + "logits/chosen": -1.710375189781189, + "logits/rejected": -1.7550632953643799, + "logps/chosen": -0.2771041989326477, + "logps/rejected": -0.2840449810028076, + "loss": 1.0326440334320068, + "loss/core": 0.9840595126152039, + "loss/preference_sft": 0.2831014096736908, + "loss/reference_anchor": 0.9433791041374207, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.670206546783447, + "rewards/margins": 2.8777670860290527, + "rewards/rejected": 1.7924392223358154, + "ronpo/logit": 0.2877767086029053, + "ronpo/residual": 0.2957441806793213, + "ronpo/residual_abs": 0.35328811407089233, + "ronpo/target": -0.007967445999383926, + "ronpo/target_abs": 0.07957801222801208, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 715 + }, + { + "drift/chosen_abs": 0.3361731469631195, + "drift/rejected_abs": 0.18497583270072937, + "epoch": 0.22884386174016685, + "grad_norm": 348.0, + "learning_rate": 1.1822718509071988e-08, + "logits/chosen": -1.8438173532485962, + "logits/rejected": -1.8709602355957031, + "logps/chosen": -0.2885790169239044, + "logps/rejected": -0.317440927028656, + "loss": 0.6532329320907593, + "loss/core": 0.6089370846748352, + "loss/preference_sft": 0.31347936391830444, + "loss/reference_anchor": 0.8545686602592468, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.3605964183807373, + "rewards/margins": 1.5140485763549805, + "rewards/rejected": 1.8465477228164673, + "ronpo/logit": 0.15140488743782043, + "ronpo/residual": 0.1639949083328247, + "ronpo/residual_abs": 0.25466206669807434, + "ronpo/target": -0.01259003859013319, + "ronpo/target_abs": 0.06609584391117096, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 720 + }, + { + "drift/chosen_abs": 0.36866021156311035, + "drift/rejected_abs": 0.09367374330759048, + "epoch": 0.23043305522447358, + "grad_norm": 198.0, + "learning_rate": 1.1203795581451287e-08, + "logits/chosen": -1.7415307760238647, + "logits/rejected": -1.8097641468048096, + "logps/chosen": -0.26107972860336304, + "logps/rejected": -0.2517997622489929, + "loss": 1.3886072635650635, + "loss/core": 1.350977897644043, + "loss/preference_sft": 0.25317317247390747, + "loss/reference_anchor": 0.7272680997848511, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.6863365173339844, + "rewards/margins": 2.7501847743988037, + "rewards/rejected": 0.9361515045166016, + "ronpo/logit": 0.27501851320266724, + "ronpo/residual": 0.2745300233364105, + "ronpo/residual_abs": 0.33074140548706055, + "ronpo/target": 0.0004884883528575301, + "ronpo/target_abs": 0.07590387016534805, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 725 + }, + { + "drift/chosen_abs": 0.5398408770561218, + "drift/rejected_abs": 0.16165503859519958, + "epoch": 0.2320222487087803, + "grad_norm": 3.734375, + "learning_rate": 1.0599462319663904e-08, + "logits/chosen": -1.5984643697738647, + "logits/rejected": -1.6125961542129517, + "logps/chosen": -0.28152695298194885, + "logps/rejected": -0.27059173583984375, + "loss": 1.8108030557632446, + "loss/core": 1.7431504726409912, + "loss/preference_sft": 0.2749939262866974, + "loss/reference_anchor": 1.3255517482757568, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 5.396024703979492, + "rewards/margins": 3.786198854446411, + "rewards/rejected": 1.609825849533081, + "ronpo/logit": 0.37861987948417664, + "ronpo/residual": 0.37991079688072205, + "ronpo/residual_abs": 0.4221607744693756, + "ronpo/target": -0.0012909111101180315, + "ronpo/target_abs": 0.07381873577833176, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 730 + }, + { + "drift/chosen_abs": 0.3352048397064209, + "drift/rejected_abs": 0.058885980397462845, + "epoch": 0.23361144219308702, + "grad_norm": 3.828125, + "learning_rate": 1.0009945988734203e-08, + "logits/chosen": -1.7541921138763428, + "logits/rejected": -1.8055585622787476, + "logps/chosen": -0.2714094817638397, + "logps/rejected": -0.26239532232284546, + "loss": 0.8836636543273926, + "loss/core": 0.8595617413520813, + "loss/preference_sft": 0.2709152102470398, + "loss/reference_anchor": 0.4549466669559479, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.3511319160461426, + "rewards/margins": 2.791016101837158, + "rewards/rejected": 0.5601160526275635, + "ronpo/logit": 0.2791016399860382, + "ronpo/residual": 0.28273439407348633, + "ronpo/residual_abs": 0.32099446654319763, + "ronpo/target": -0.0036327368579804897, + "ronpo/target_abs": 0.061633963137865067, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 735 + }, + { + "drift/chosen_abs": 0.2550628185272217, + "drift/rejected_abs": 0.10268987715244293, + "epoch": 0.23520063567739372, + "grad_norm": 12.4375, + "learning_rate": 9.435468281644798e-09, + "logits/chosen": -1.7330278158187866, + "logits/rejected": -1.7721294164657593, + "logps/chosen": -0.2473641335964203, + "logps/rejected": -0.24295814335346222, + "loss": 0.14805634319782257, + "loss/core": 0.14048026502132416, + "loss/preference_sft": 0.24413037300109863, + "loss/reference_anchor": 0.12710848450660706, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.5494306087493896, + "rewards/margins": 1.5241954326629639, + "rewards/rejected": 1.0252354145050049, + "ronpo/logit": 0.1524195522069931, + "ronpo/residual": 0.17534348368644714, + "ronpo/residual_abs": 0.20049890875816345, + "ronpo/target": -0.02292392961680889, + "ronpo/target_abs": 0.06671784818172455, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 740 + }, + { + "drift/chosen_abs": 0.22789403796195984, + "drift/rejected_abs": 0.15626075863838196, + "epoch": 0.23678982916170044, + "grad_norm": 31.875, + "learning_rate": 8.876245235966883e-09, + "logits/chosen": -1.8382848501205444, + "logits/rejected": -1.8339512348175049, + "logps/chosen": -0.2761487364768982, + "logps/rejected": -0.282112181186676, + "loss": 0.2007966786623001, + "loss/core": 0.1876274198293686, + "loss/preference_sft": 0.28972652554512024, + "loss/reference_anchor": 0.23441258072853088, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.2789406776428223, + "rewards/margins": 0.7250955104827881, + "rewards/rejected": 1.5538451671600342, + "ronpo/logit": 0.07250954210758209, + "ronpo/residual": 0.06640531122684479, + "ronpo/residual_abs": 0.1977664977312088, + "ronpo/target": 0.00610423693433404, + "ronpo/target_abs": 0.07519592344760895, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 745 + }, + { + "drift/chosen_abs": 0.17921927571296692, + "drift/rejected_abs": 0.10521320253610611, + "epoch": 0.23837902264600716, + "grad_norm": 2.578125, + "learning_rate": 8.332487152617423e-09, + "logits/chosen": -1.672652244567871, + "logits/rejected": -1.641906976699829, + "logps/chosen": -0.29797834157943726, + "logps/rejected": -0.2950153052806854, + "loss": 0.05657457560300827, + "loss/core": 0.05042596533894539, + "loss/preference_sft": 0.3005446791648865, + "loss/reference_anchor": 0.09291768074035645, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.790204644203186, + "rewards/margins": 0.7381783723831177, + "rewards/rejected": 1.0520262718200684, + "ronpo/logit": 0.07381784170866013, + "ronpo/residual": 0.05989062041044235, + "ronpo/residual_abs": 0.14463061094284058, + "ronpo/target": 0.013927215710282326, + "ronpo/target_abs": 0.08233248442411423, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 750 + }, + { + "drift/chosen_abs": 0.40166425704956055, + "drift/rejected_abs": 0.09812676906585693, + "epoch": 0.23996821613031386, + "grad_norm": 2320.0, + "learning_rate": 7.804398516773464e-09, + "logits/chosen": -1.9683334827423096, + "logits/rejected": -1.9788726568222046, + "logps/chosen": -0.283616840839386, + "logps/rejected": -0.2929161489009857, + "loss": 1.8589715957641602, + "loss/core": 1.8126780986785889, + "loss/preference_sft": 0.2952941060066223, + "loss/reference_anchor": 0.8963397741317749, + "rewards/accuracies": 0.875, + "rewards/chosen": 4.014865875244141, + "rewards/margins": 3.2176384925842285, + "rewards/rejected": 0.7972274422645569, + "ronpo/logit": 0.3217638432979584, + "ronpo/residual": 0.2891455590724945, + "ronpo/residual_abs": 0.35307732224464417, + "ronpo/target": 0.03261827304959297, + "ronpo/target_abs": 0.08080680668354034, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 755 + }, + { + "drift/chosen_abs": 0.23729880154132843, + "drift/rejected_abs": 0.11338727176189423, + "epoch": 0.24155740961462058, + "grad_norm": 11.3125, + "learning_rate": 7.292177920973724e-09, + "logits/chosen": -2.0670156478881836, + "logits/rejected": -1.978662133216858, + "logps/chosen": -0.2790946364402771, + "logps/rejected": -0.28680047392845154, + "loss": 0.584754228591919, + "loss/core": 0.5627850890159607, + "loss/preference_sft": 0.28632253408432007, + "loss/reference_anchor": 0.4107509255409241, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.371988534927368, + "rewards/margins": 1.239213228225708, + "rewards/rejected": 1.1327753067016602, + "ronpo/logit": 0.12392131984233856, + "ronpo/residual": 0.1416681855916977, + "ronpo/residual_abs": 0.21293732523918152, + "ronpo/target": -0.01774686574935913, + "ronpo/target_abs": 0.08494757860898972, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 760 + }, + { + "drift/chosen_abs": 0.16878637671470642, + "drift/rejected_abs": 0.11679929494857788, + "epoch": 0.2431466030989273, + "grad_norm": 0.7109375, + "learning_rate": 6.796017990435976e-09, + "logits/chosen": -1.8809000253677368, + "logits/rejected": -1.863707184791565, + "logps/chosen": -0.2994479835033417, + "logps/rejected": -0.2976071238517761, + "loss": 0.05739004537463188, + "loss/core": 0.049158744513988495, + "loss/preference_sft": 0.29969534277915955, + "loss/reference_anchor": 0.13465645909309387, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.6868946552276611, + "rewards/margins": 0.5216689109802246, + "rewards/rejected": 1.1652257442474365, + "ronpo/logit": 0.0521668903529644, + "ronpo/residual": 0.06433439254760742, + "ronpo/residual_abs": 0.12495573610067368, + "ronpo/target": -0.012167500331997871, + "ronpo/target_abs": 0.0812489464879036, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 765 + }, + { + "drift/chosen_abs": 0.16920286417007446, + "drift/rejected_abs": 0.10078708082437515, + "epoch": 0.24473579658323402, + "grad_norm": 0.921875, + "learning_rate": 6.316105310618664e-09, + "logits/chosen": -1.8295981884002686, + "logits/rejected": -1.8808387517929077, + "logps/chosen": -0.2839786112308502, + "logps/rejected": -0.288639098405838, + "loss": 0.039382454007864, + "loss/core": 0.03182164579629898, + "loss/preference_sft": 0.28843414783477783, + "loss/reference_anchor": 0.12237273156642914, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.6907180547714233, + "rewards/margins": 0.6828474402427673, + "rewards/rejected": 1.0078706741333008, + "ronpo/logit": 0.06828473508358002, + "ronpo/residual": 0.06084071844816208, + "ronpo/residual_abs": 0.1161794438958168, + "ronpo/target": 0.00744402501732111, + "ronpo/target_abs": 0.07908720523118973, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 770 + }, + { + "drift/chosen_abs": 0.2419697493314743, + "drift/rejected_abs": 0.11826857179403305, + "epoch": 0.24632499006754072, + "grad_norm": 5.9375, + "learning_rate": 5.8526203570536504e-09, + "logits/chosen": -1.7400401830673218, + "logits/rejected": -1.7736377716064453, + "logps/chosen": -0.2894227206707001, + "logps/rejected": -0.2918685972690582, + "loss": 0.08226810395717621, + "loss/core": 0.07246662676334381, + "loss/preference_sft": 0.29761046171188354, + "loss/reference_anchor": 0.16626867651939392, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.4190304279327393, + "rewards/margins": 1.2379631996154785, + "rewards/rejected": 1.1810672283172607, + "ronpo/logit": 0.12379632890224457, + "ronpo/residual": 0.11871888488531113, + "ronpo/residual_abs": 0.17133726179599762, + "ronpo/target": 0.0050774430856108665, + "ronpo/target_abs": 0.09046323597431183, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 775 + }, + { + "drift/chosen_abs": 0.26481837034225464, + "drift/rejected_abs": 0.0806707888841629, + "epoch": 0.24791418355184744, + "grad_norm": 6.53125, + "learning_rate": 5.405737427476853e-09, + "logits/chosen": -1.9032214879989624, + "logits/rejected": -1.9158647060394287, + "logps/chosen": -0.275672972202301, + "logps/rejected": -0.27686625719070435, + "loss": 1.127648115158081, + "loss/core": 1.096915364265442, + "loss/preference_sft": 0.2758302688598633, + "loss/reference_anchor": 0.5870689749717712, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.6463587284088135, + "rewards/margins": 1.8401899337768555, + "rewards/rejected": 0.8061690330505371, + "ronpo/logit": 0.18401899933815002, + "ronpo/residual": 0.19550299644470215, + "ronpo/residual_abs": 0.2645988464355469, + "ronpo/target": -0.011484002694487572, + "ronpo/target_abs": 0.09347420930862427, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 780 + }, + { + "drift/chosen_abs": 0.18544304370880127, + "drift/rejected_abs": 0.06540529429912567, + "epoch": 0.24950337703615416, + "grad_norm": 2.1875, + "learning_rate": 4.975624576281889e-09, + "logits/chosen": -1.76251220703125, + "logits/rejected": -1.7324647903442383, + "logps/chosen": -0.2766936719417572, + "logps/rejected": -0.2744191288948059, + "loss": 0.19484534859657288, + "loss/core": 0.18755874037742615, + "loss/preference_sft": 0.27636733651161194, + "loss/reference_anchor": 0.11809553951025009, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8540092706680298, + "rewards/margins": 1.2056849002838135, + "rewards/rejected": 0.6483243703842163, + "ronpo/logit": 0.12056849896907806, + "ronpo/residual": 0.12910667061805725, + "ronpo/residual_abs": 0.16958020627498627, + "ronpo/target": -0.00853817630559206, + "ronpo/target_abs": 0.06679105758666992, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 785 + }, + { + "drift/chosen_abs": 0.23630964756011963, + "drift/rejected_abs": 0.12132765352725983, + "epoch": 0.25109257052046086, + "grad_norm": 16.125, + "learning_rate": 4.562443551321787e-09, + "logits/chosen": -1.8636810779571533, + "logits/rejected": -1.7788467407226562, + "logps/chosen": -0.2716495394706726, + "logps/rejected": -0.2747117877006531, + "loss": 0.19760456681251526, + "loss/core": 0.1853138655424118, + "loss/preference_sft": 0.2780051827430725, + "loss/reference_anchor": 0.21801309287548065, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.3603432178497314, + "rewards/margins": 1.1492493152618408, + "rewards/rejected": 1.211093783378601, + "ronpo/logit": 0.11492495238780975, + "ronpo/residual": 0.12750717997550964, + "ronpo/residual_abs": 0.19350960850715637, + "ronpo/target": -0.01258223969489336, + "ronpo/target_abs": 0.07843288779258728, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 790 + }, + { + "drift/chosen_abs": 0.2949658930301666, + "drift/rejected_abs": 0.15656772255897522, + "epoch": 0.2526817640047676, + "grad_norm": 5.03125, + "learning_rate": 4.166349733082153e-09, + "logits/chosen": -1.9113426208496094, + "logits/rejected": -1.931244134902954, + "logps/chosen": -0.28906121850013733, + "logps/rejected": -0.30459824204444885, + "loss": 0.7576580047607422, + "loss/core": 0.732965886592865, + "loss/preference_sft": 0.28880181908607483, + "loss/reference_anchor": 0.46496111154556274, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.9488015174865723, + "rewards/margins": 1.3835405111312866, + "rewards/rejected": 1.565260648727417, + "ronpo/logit": 0.13835404813289642, + "ronpo/residual": 0.17754340171813965, + "ronpo/residual_abs": 0.3039698004722595, + "ronpo/target": -0.03918936103582382, + "ronpo/target_abs": 0.08568587899208069, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 795 + }, + { + "drift/chosen_abs": 0.4828338027000427, + "drift/rejected_abs": 0.11110155284404755, + "epoch": 0.2542709574890743, + "grad_norm": 136.0, + "learning_rate": 3.7874920762489935e-09, + "logits/chosen": -1.885493278503418, + "logits/rejected": -1.9146902561187744, + "logps/chosen": -0.2795495390892029, + "logps/rejected": -0.2741713523864746, + "loss": 1.7612409591674805, + "loss/core": 1.7040965557098389, + "loss/preference_sft": 0.27895671129226685, + "loss/reference_anchor": 1.1149905920028687, + "rewards/accuracies": 0.9375, + "rewards/chosen": 4.827905178070068, + "rewards/margins": 3.717952251434326, + "rewards/rejected": 1.109952688217163, + "ronpo/logit": 0.37179526686668396, + "ronpo/residual": 0.3783615827560425, + "ronpo/residual_abs": 0.4227665960788727, + "ronpo/target": -0.006566292606294155, + "ronpo/target_abs": 0.07662634551525116, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 800 + }, + { + "drift/chosen_abs": 0.18141977488994598, + "drift/rejected_abs": 0.08246532082557678, + "epoch": 0.255860150973381, + "grad_norm": 1.6484375, + "learning_rate": 3.426013053692878e-09, + "logits/chosen": -1.7892440557479858, + "logits/rejected": -1.836334466934204, + "logps/chosen": -0.2862665057182312, + "logps/rejected": -0.2810719907283783, + "loss": 0.12512250244617462, + "loss/core": 0.11690256744623184, + "loss/preference_sft": 0.2856505513191223, + "loss/reference_anchor": 0.13583365082740784, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8133046627044678, + "rewards/margins": 0.9899392127990723, + "rewards/rejected": 0.8233655691146851, + "ronpo/logit": 0.0989939272403717, + "ronpo/residual": 0.10105105489492416, + "ronpo/residual_abs": 0.15274189412593842, + "ronpo/target": -0.0020571290515363216, + "ronpo/target_abs": 0.0786915272474289, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 805 + }, + { + "drift/chosen_abs": 0.2540569007396698, + "drift/rejected_abs": 0.10452580451965332, + "epoch": 0.25744934445768775, + "grad_norm": 2.15625, + "learning_rate": 3.0820486028908077e-09, + "logits/chosen": -1.997826337814331, + "logits/rejected": -2.0715103149414062, + "logps/chosen": -0.2701925039291382, + "logps/rejected": -0.26402124762535095, + "loss": 0.43526023626327515, + "loss/core": 0.41667813062667847, + "loss/preference_sft": 0.2734178304672241, + "loss/reference_anchor": 0.3443005383014679, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.5394034385681152, + "rewards/margins": 1.49575674533844, + "rewards/rejected": 1.0436466932296753, + "ronpo/logit": 0.14957566559314728, + "ronpo/residual": 0.16587159037590027, + "ronpo/residual_abs": 0.2100309580564499, + "ronpo/target": -0.01629592850804329, + "ronpo/target_abs": 0.07158873975276947, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 810 + }, + { + "drift/chosen_abs": 0.26953545212745667, + "drift/rejected_abs": 0.11704172194004059, + "epoch": 0.25903853794199444, + "grad_norm": 0.8203125, + "learning_rate": 2.7557280748055966e-09, + "logits/chosen": -1.7852869033813477, + "logits/rejected": -1.8557426929473877, + "logps/chosen": -0.2673250436782837, + "logps/rejected": -0.2700131833553314, + "loss": 0.40478211641311646, + "loss/core": 0.385967880487442, + "loss/preference_sft": 0.27039602398872375, + "loss/reference_anchor": 0.3492446839809418, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.6953542232513428, + "rewards/margins": 1.5257197618484497, + "rewards/rejected": 1.1696348190307617, + "ronpo/logit": 0.15257197618484497, + "ronpo/residual": 0.15629687905311584, + "ronpo/residual_abs": 0.19490864872932434, + "ronpo/target": -0.003724911715835333, + "ronpo/target_abs": 0.07523742318153381, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 815 + }, + { + "drift/chosen_abs": 0.2987804412841797, + "drift/rejected_abs": 0.1348761022090912, + "epoch": 0.26062773142630113, + "grad_norm": 205.0, + "learning_rate": 2.4471741852423233e-09, + "logits/chosen": -1.8530000448226929, + "logits/rejected": -1.8922631740570068, + "logps/chosen": -0.2706072926521301, + "logps/rejected": -0.26072853803634644, + "loss": 0.3187785744667053, + "loss/core": 0.30332642793655396, + "loss/preference_sft": 0.2643357813358307, + "loss/reference_anchor": 0.2826094925403595, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.9860172271728516, + "rewards/margins": 1.6376988887786865, + "rewards/rejected": 1.348318338394165, + "ronpo/logit": 0.1637699007987976, + "ronpo/residual": 0.16065004467964172, + "ronpo/residual_abs": 0.21420709788799286, + "ronpo/target": 0.0031198528595268726, + "ronpo/target_abs": 0.08196929842233658, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 820 + }, + { + "drift/chosen_abs": 0.22662219405174255, + "drift/rejected_abs": 0.09158599376678467, + "epoch": 0.2622169249106079, + "grad_norm": 9.5, + "learning_rate": 2.1565029686999305e-09, + "logits/chosen": -1.7125442028045654, + "logits/rejected": -1.758292555809021, + "logps/chosen": -0.2971535921096802, + "logps/rejected": -0.30583325028419495, + "loss": 0.2096571922302246, + "loss/core": 0.19694137573242188, + "loss/preference_sft": 0.30101680755615234, + "loss/reference_anchor": 0.2242145836353302, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.266221761703491, + "rewards/margins": 1.3520539999008179, + "rewards/rejected": 0.9141677618026733, + "ronpo/logit": 0.13520540297031403, + "ronpo/residual": 0.11602624505758286, + "ronpo/residual_abs": 0.17035934329032898, + "ronpo/target": 0.019179154187440872, + "ronpo/target_abs": 0.07527647167444229, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 825 + }, + { + "drift/chosen_abs": 0.15426619350910187, + "drift/rejected_abs": 0.07910831272602081, + "epoch": 0.2638061183949146, + "grad_norm": 2.59375, + "learning_rate": 1.8838237347353845e-09, + "logits/chosen": -1.9536901712417603, + "logits/rejected": -1.982287049293518, + "logps/chosen": -0.30930250883102417, + "logps/rejected": -0.31085312366485596, + "loss": 0.08624901622533798, + "loss/core": 0.07954810559749603, + "loss/preference_sft": 0.31143519282341003, + "loss/reference_anchor": 0.10287465900182724, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.5410267114639282, + "rewards/margins": 0.749943733215332, + "rewards/rejected": 0.791083037853241, + "ronpo/logit": 0.07499437034130096, + "ronpo/residual": 0.05749637633562088, + "ronpo/residual_abs": 0.13403594493865967, + "ronpo/target": 0.01749800331890583, + "ronpo/target_abs": 0.07980482280254364, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 830 + }, + { + "drift/chosen_abs": 0.34234294295310974, + "drift/rejected_abs": 0.15316882729530334, + "epoch": 0.2653953118792213, + "grad_norm": 4.125, + "learning_rate": 1.6292390268568101e-09, + "logits/chosen": -1.5117677450180054, + "logits/rejected": -1.5660102367401123, + "logps/chosen": -0.27989453077316284, + "logps/rejected": -0.30721816420555115, + "loss": 0.3620285987854004, + "loss/core": 0.34146106243133545, + "loss/preference_sft": 0.30074799060821533, + "loss/reference_anchor": 0.3812769651412964, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.4228203296661377, + "rewards/margins": 1.891760230064392, + "rewards/rejected": 1.5310604572296143, + "ronpo/logit": 0.1891760379076004, + "ronpo/residual": 0.17069289088249207, + "ronpo/residual_abs": 0.2289830446243286, + "ronpo/target": 0.01848311722278595, + "ronpo/target_abs": 0.07700731605291367, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 835 + }, + { + "drift/chosen_abs": 0.3780483305454254, + "drift/rejected_abs": 0.12148091942071915, + "epoch": 0.266984505363528, + "grad_norm": 7.28125, + "learning_rate": 1.392844583961078e-09, + "logits/chosen": -1.8261111974716187, + "logits/rejected": -1.9145349264144897, + "logps/chosen": -0.2696583867073059, + "logps/rejected": -0.26150408387184143, + "loss": 1.2962982654571533, + "loss/core": 1.2566306591033936, + "loss/preference_sft": 0.2642626464366913, + "loss/reference_anchor": 0.7669227123260498, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.7794418334960938, + "rewards/margins": 2.5657799243927, + "rewards/rejected": 1.213661789894104, + "ronpo/logit": 0.2565780282020569, + "ronpo/residual": 0.2539408504962921, + "ronpo/residual_abs": 0.3109070360660553, + "ronpo/target": 0.002637143013998866, + "ronpo/target_abs": 0.0665135607123375, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 840 + }, + { + "drift/chosen_abs": 0.1283138245344162, + "drift/rejected_abs": 0.066379114985466, + "epoch": 0.2685736988478347, + "grad_norm": 17.125, + "learning_rate": 1.1747293043303518e-09, + "logits/chosen": -1.858109474182129, + "logits/rejected": -1.7954213619232178, + "logps/chosen": -0.28292885422706604, + "logps/rejected": -0.2912057936191559, + "loss": 0.03620520606637001, + "loss/core": 0.032505668699741364, + "loss/preference_sft": 0.28797465562820435, + "loss/reference_anchor": 0.04519306868314743, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.2819868326187134, + "rewards/margins": 0.6205378174781799, + "rewards/rejected": 0.6614489555358887, + "ronpo/logit": 0.062053781002759933, + "ronpo/residual": 0.05858888477087021, + "ronpo/residual_abs": 0.11494974046945572, + "ronpo/target": 0.0034649022854864597, + "ronpo/target_abs": 0.08666227012872696, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 845 + }, + { + "drift/chosen_abs": 0.15765869617462158, + "drift/rejected_abs": 0.1053333505988121, + "epoch": 0.2701628923321414, + "grad_norm": 5.5625, + "learning_rate": 9.749752122010346e-10, + "logits/chosen": -1.743290901184082, + "logits/rejected": -1.6410249471664429, + "logps/chosen": -0.3139035701751709, + "logps/rejected": -0.31750422716140747, + "loss": 0.059708237648010254, + "loss/core": 0.047553762793540955, + "loss/preference_sft": 0.3167984187602997, + "loss/reference_anchor": 0.2114095687866211, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.5753216743469238, + "rewards/margins": 0.5241764783859253, + "rewards/rejected": 1.0511451959609985, + "ronpo/logit": 0.05241764709353447, + "ronpo/residual": 0.039160002022981644, + "ronpo/residual_abs": 0.13094462454319, + "ronpo/target": 0.013257652521133423, + "ronpo/target_abs": 0.08758213371038437, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 850 + }, + { + "drift/chosen_abs": 0.20687925815582275, + "drift/rejected_abs": 0.10394798219203949, + "epoch": 0.27175208581644816, + "grad_norm": 9.0625, + "learning_rate": 7.936574269178375e-10, + "logits/chosen": -1.7965866327285767, + "logits/rejected": -1.8572895526885986, + "logps/chosen": -0.2916758358478546, + "logps/rejected": -0.2980891466140747, + "loss": 0.13103294372558594, + "loss/core": 0.12218041718006134, + "loss/preference_sft": 0.30287644267082214, + "loss/reference_anchor": 0.14676304161548615, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.067711353302002, + "rewards/margins": 1.0313072204589844, + "rewards/rejected": 1.0364038944244385, + "ronpo/logit": 0.10313073545694351, + "ronpo/residual": 0.12047480046749115, + "ronpo/residual_abs": 0.19301636517047882, + "ronpo/target": -0.017344076186418533, + "ronpo/target_abs": 0.08581249415874481, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 855 + }, + { + "drift/chosen_abs": 0.23877234756946564, + "drift/rejected_abs": 0.16189390420913696, + "epoch": 0.27334127930075486, + "grad_norm": 14.6875, + "learning_rate": 6.308441346844384e-10, + "logits/chosen": -1.7404086589813232, + "logits/rejected": -1.6830250024795532, + "logps/chosen": -0.2857043445110321, + "logps/rejected": -0.2822682857513428, + "loss": 0.7747726440429688, + "loss/core": 0.7492650747299194, + "loss/preference_sft": 0.2928890585899353, + "loss/reference_anchor": 0.48086386919021606, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.387723445892334, + "rewards/margins": 0.7728492617607117, + "rewards/rejected": 1.614874243736267, + "ronpo/logit": 0.07728491723537445, + "ronpo/residual": 0.09040971845388412, + "ronpo/residual_abs": 0.27997887134552, + "ronpo/target": -0.013124806806445122, + "ronpo/target_abs": 0.07721184939146042, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 860 + }, + { + "drift/chosen_abs": 0.35751527547836304, + "drift/rejected_abs": 0.18375881016254425, + "epoch": 0.2749304727850616, + "grad_norm": 2.03125, + "learning_rate": 4.865965629214819e-10, + "logits/chosen": -1.853266716003418, + "logits/rejected": -1.793908715248108, + "logps/chosen": -0.2659180164337158, + "logps/rejected": -0.2806767523288727, + "loss": 0.9706258773803711, + "loss/core": 0.9136497378349304, + "loss/preference_sft": 0.27038851380348206, + "loss/reference_anchor": 1.1124818325042725, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.574227809906006, + "rewards/margins": 1.7369954586029053, + "rewards/rejected": 1.8372325897216797, + "ronpo/logit": 0.1736995428800583, + "ronpo/residual": 0.18940187990665436, + "ronpo/residual_abs": 0.29147475957870483, + "ronpo/target": -0.015702344477176666, + "ronpo/target_abs": 0.0712803453207016, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 865 + }, + { + "drift/chosen_abs": 0.2339034080505371, + "drift/rejected_abs": 0.11022868007421494, + "epoch": 0.2765196662693683, + "grad_norm": 2.9375, + "learning_rate": 3.609689572414143e-10, + "logits/chosen": -1.794655203819275, + "logits/rejected": -1.8671070337295532, + "logps/chosen": -0.27023810148239136, + "logps/rejected": -0.27533429861068726, + "loss": 0.15046367049217224, + "loss/core": 0.14148980379104614, + "loss/preference_sft": 0.26623964309692383, + "loss/reference_anchor": 0.15285304188728333, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.3387210369110107, + "rewards/margins": 1.2503958940505981, + "rewards/rejected": 1.0883252620697021, + "ronpo/logit": 0.12503957748413086, + "ronpo/residual": 0.12151853740215302, + "ronpo/residual_abs": 0.17508959770202637, + "ronpo/target": 0.0035210344940423965, + "ronpo/target_abs": 0.08289609849452972, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 870 + }, + { + "drift/chosen_abs": 0.2378336638212204, + "drift/rejected_abs": 0.08225777000188828, + "epoch": 0.278108859753675, + "grad_norm": 2.140625, + "learning_rate": 2.5400856104894063e-10, + "logits/chosen": -1.9827836751937866, + "logits/rejected": -2.0595450401306152, + "logps/chosen": -0.2802538275718689, + "logps/rejected": -0.2741009593009949, + "loss": 0.7382990717887878, + "loss/core": 0.7156741619110107, + "loss/preference_sft": 0.2707212567329407, + "loss/reference_anchor": 0.42542606592178345, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.3760526180267334, + "rewards/margins": 1.553762435913086, + "rewards/rejected": 0.8222904205322266, + "ronpo/logit": 0.15537627041339874, + "ronpo/residual": 0.15069438517093658, + "ronpo/residual_abs": 0.20385237038135529, + "ronpo/target": 0.004681874066591263, + "ronpo/target_abs": 0.06999818235635757, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 875 + }, + { + "drift/chosen_abs": 0.2282506674528122, + "drift/rejected_abs": 0.14202216267585754, + "epoch": 0.27969805323798175, + "grad_norm": 11.5625, + "learning_rate": 1.6575559777469716e-10, + "logits/chosen": -1.6084506511688232, + "logits/rejected": -1.652144432067871, + "logps/chosen": -0.28672975301742554, + "logps/rejected": -0.2819688022136688, + "loss": 0.3919622004032135, + "loss/core": 0.37796106934547424, + "loss/preference_sft": 0.28091609477996826, + "loss/reference_anchor": 0.2519303262233734, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.2814536094665527, + "rewards/margins": 0.8872292637825012, + "rewards/rejected": 1.3942242860794067, + "ronpo/logit": 0.08872292190790176, + "ronpo/residual": 0.10213188827037811, + "ronpo/residual_abs": 0.2572005093097687, + "ronpo/target": -0.013408951461315155, + "ronpo/target_abs": 0.07671914249658585, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 880 + }, + { + "drift/chosen_abs": 0.2979094684123993, + "drift/rejected_abs": 0.12212695926427841, + "epoch": 0.28128724672228844, + "grad_norm": 0.765625, + "learning_rate": 9.624325574890125e-11, + "logits/chosen": -1.799660325050354, + "logits/rejected": -1.7860395908355713, + "logps/chosen": -0.2907426357269287, + "logps/rejected": -0.29116326570510864, + "loss": 0.39276254177093506, + "loss/core": 0.3697052299976349, + "loss/preference_sft": 0.2975885272026062, + "loss/reference_anchor": 0.4313863217830658, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.97800874710083, + "rewards/margins": 1.7583093643188477, + "rewards/rejected": 1.2196992635726929, + "ronpo/logit": 0.1758309304714203, + "ronpo/residual": 0.20061039924621582, + "ronpo/residual_abs": 0.22602948546409607, + "ronpo/target": -0.024779465049505234, + "ronpo/target_abs": 0.07615292817354202, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 885 + }, + { + "drift/chosen_abs": 0.13840210437774658, + "drift/rejected_abs": 0.10734204202890396, + "epoch": 0.28287644020659514, + "grad_norm": 8.3125, + "learning_rate": 4.5497675720540527e-11, + "logits/chosen": -1.923963189125061, + "logits/rejected": -1.902496099472046, + "logps/chosen": -0.28208020329475403, + "logps/rejected": -0.2937772274017334, + "loss": 0.12399367988109589, + "loss/core": 0.11801999807357788, + "loss/preference_sft": 0.2783675789833069, + "loss/reference_anchor": 0.0916367918252945, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.3840210437774658, + "rewards/margins": 0.3155991733074188, + "rewards/rejected": 1.0684218406677246, + "ronpo/logit": 0.03155992180109024, + "ronpo/residual": 0.055709730833768845, + "ronpo/residual_abs": 0.15228456258773804, + "ronpo/target": -0.024149809032678604, + "ronpo/target_abs": 0.08036401867866516, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 890 + }, + { + "drift/chosen_abs": 0.1609112173318863, + "drift/rejected_abs": 0.08355027437210083, + "epoch": 0.2844656336909019, + "grad_norm": 12.6875, + "learning_rate": 1.3537941026914302e-11, + "logits/chosen": -1.9804353713989258, + "logits/rejected": -1.975773572921753, + "logps/chosen": -0.30720216035842896, + "logps/rejected": -0.30572324991226196, + "loss": 0.1231997162103653, + "loss/core": 0.11622112989425659, + "loss/preference_sft": 0.3030904531478882, + "loss/reference_anchor": 0.10926266759634018, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.608919382095337, + "rewards/margins": 0.7737261652946472, + "rewards/rejected": 0.8351932764053345, + "ronpo/logit": 0.07737261801958084, + "ronpo/residual": 0.07732744514942169, + "ronpo/residual_abs": 0.12864170968532562, + "ronpo/target": 4.518199784797616e-05, + "ronpo/target_abs": 0.059388406574726105, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 895 + }, + { + "drift/chosen_abs": 0.23922479152679443, + "drift/rejected_abs": 0.13327732682228088, + "epoch": 0.2860548271752086, + "grad_norm": 15.75, + "learning_rate": 3.7607041719622813e-13, + "logits/chosen": -1.8160746097564697, + "logits/rejected": -1.8043482303619385, + "logps/chosen": -0.2887292504310608, + "logps/rejected": -0.28216832876205444, + "loss": 0.0885263979434967, + "loss/core": 0.0769236832857132, + "loss/preference_sft": 0.285482257604599, + "loss/reference_anchor": 0.20350602269172668, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.3910775184631348, + "rewards/margins": 1.061415433883667, + "rewards/rejected": 1.3296619653701782, + "ronpo/logit": 0.10614154487848282, + "ronpo/residual": 0.1308242827653885, + "ronpo/residual_abs": 0.15298447012901306, + "ronpo/target": -0.024682745337486267, + "ronpo/target_abs": 0.07124222069978714, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 900 + }, + { + "epoch": 0.2860548271752086, + "step": 900, + "total_flos": 0.0, + "train_loss": 0.48311558212671013, + "train_runtime": 8488.8129, + "train_samples_per_second": 1.696, + "train_steps_per_second": 0.106 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..121f147 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45ba56b7207518957edb37cbe8f5c875620e2bb5a8acb0cb47382eb55cd80228 +size 7057