From 8443171fe9bba4b07bbf755f9c49161a76a2cfcb Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Thu, 23 Jul 2026 13:22:12 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: promotion/qwen3-8b-aaai27-flagship-ronpo-k-only-s42 Source: Original Platform --- .gitattributes | 36 + README.md | 24 + all_results.json | 9 + chat_template.jinja | 89 + config.json | 71 + config.yaml | 59 + generation_config.json | 12 + job_status.json | 23 + model.safetensors | 3 + objective_protocol.json | 69 + pair_manifest.json | 34 + provenance.json | 13 + resource_profile.json | 21 + run_status.json | 14 + split_manifest.json | 23 + stability_gate.json | 38 + tokenizer.json | 3 + tokenizer_config.json | 30 + train_results.json | 9 + trainer_state.json | 4903 +++++++++++++++++++++++++++++++++++++++ training_args.bin | 3 + 21 files changed, 5486 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 config.yaml create mode 100644 generation_config.json create mode 100644 job_status.json create mode 100644 model.safetensors create mode 100644 objective_protocol.json create mode 100644 pair_manifest.json create mode 100644 provenance.json create mode 100644 resource_profile.json create mode 100644 run_status.json create mode 100644 split_manifest.json create mode 100644 stability_gate.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..168a6ce --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +--- +library_name: transformers +base_model: Qwen/Qwen3-8B +tags: +- ronpo +- mnpo +- preference-optimization +--- + +# qwen3-8b-aaai27-flagship-ronpo-k-only-s42 + +Research checkpoint for the RONPO AAAI revision experiments. + +- Method: ronpo_k_only +- Base model: `Qwen/Qwen3-8B` +- Seed: 42 +- Local source at upload time: `/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/ronpo_k_only/seed42/attempt3` +- Uploaded at UTC: 2026-07-12T22:02:37Z +- Run status metadata: `{"attempt": 3, "effective_batch_size": 16, "method": "ronpo_k_only", "objective_protocol": "objective_protocol.json", "optimizer_steps": 900, "resource_profile": "resource_profile.json", "seed": 42, "split_manifest": "split_manifest.json", "stability_gate": "stability_gate.json", "status": "completed", "wandb_run_id": "89616b47c092", "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/89616b47c092"}` + +AAAI-27 P1 matched budget: 900 optimizer steps, effective batch 16; passed non-thinking and collapse stability gates; sealed-test metrics are not used for checkpoint selection. + +Intended use: reproducibility and evaluation for the RONPO paper. This +checkpoint is not intended as a production assistant. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..aa15823 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.2860548271752086, + "total_flos": 0.0, + "train_loss": 0.42856775469250147, + "train_runtime": 7333.3683, + "train_samples": 50340, + "train_samples_per_second": 1.964, + "train_steps_per_second": 0.123 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..8af5a27 --- /dev/null +++ b/config.yaml @@ -0,0 +1,59 @@ +model_name_or_path: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218 +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + ? /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/ronpo_k_only + : 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: ronpo +eta: 0.0075 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.05 +preference_sft_weight: 0.005 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 2.5e-08 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 42 +gradient_accumulation_steps: 16 +gradient_checkpointing: true +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 1 +per_device_eval_batch_size: 1 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/ronpo_k_only/seed42/attempt3 +run_name: aaai27-flagship-full-ronpo_k_only-s42-a3 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/job_status.json b/job_status.json new file mode 100644 index 0000000..1fc371a --- /dev/null +++ b/job_status.json @@ -0,0 +1,23 @@ +{ + "status": "completed", + "stage": "full", + "method": "ronpo_k_only", + "seed": 42, + "attempt": 3, + "gpu": 0, + "gpus": [ + 0 + ], + "returncode": 0, + "finite_metrics": true, + "oom": false, + "traceback": false, + "reason": "passed", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "89616b47c092", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/89616b47c092", + "output_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/ronpo_k_only/seed42/attempt3", + "log": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/logs/full_ronpo_k_only_s42_a3.log", + "completed_at": "2026-07-12T22:00:08Z" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..da7598b --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80ba53aa1666a2e3813fdde0bc766ac2ced033be7781a0e5d0f3799c9026e493 +size 16381517208 diff --git a/objective_protocol.json b/objective_protocol.json new file mode 100644 index 0000000..35b3b44 --- /dev/null +++ b/objective_protocol.json @@ -0,0 +1,69 @@ +{ + "schema_version": 1, + "frozen_at_kst": "2026-07-12T21:40:00+09:00", + "base_model": "Qwen/Qwen3-8B", + "primary_triple": { + "helpfulness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "ultrafeedback-helpfulness", + "transform": "identity" + }, + "safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "conciseness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "helpsteer-verbosity", + "transform": "negate" + } + }, + "gate_thresholds": { + "raw_score_std_strictly_greater_than": 0.01, + "aggregate_prompt_pair_spearman_median_strictly_less_than": 0.0, + "aggregate_pairwise_top1_mismatch_at_least": 0.5 + }, + "gate_pool": "source-train Qwen3-8B response pool split into deterministic train/validation; sealed source-test excluded", + "training_seeds": [42, 43, 44], + "optimizer_steps": 900, + "effective_batch_size": 16, + "decode": { + "enable_thinking": false, + "temperature": 0.7, + "top_p": 0.9, + "max_new_tokens": 2048, + "decode_seed": 42 + }, + "methods": [ + "ronpo_full_expect", + "ronpo_k_only", + "dpo", + "ipo", + "simpo", + "kto", + "sppo_avg", + "inpo_avg", + "ht_mnpo_helpfulness", + "ht_mnpo_safety", + "ht_mnpo_conciseness" + ], + "selection_policy": "No sealed-test selection. One preregistered configuration per method; failures use only the declared stability retry recipe, at most three attempts.", + "stability_gate": { + "validation_prompts": 128, + "min_candidate_base_mean_word_ratio": 0.33, + "max_candidate_base_mean_word_ratio": 2.0, + "max_consecutive_identical_word_run": 20, + "think_tag_count": 0, + "empty_response_count": 0, + "retry_recipe": "attempt 2 halves learning rate, enables gradient checkpointing, and adds reference=0.02/SFT=0.002 anchoring to unanchored unified losses; attempt 3 quarters learning rate and uses reference=0.05/SFT=0.005; RONPO uses reference=0.05/SFT=0.005 from attempt 1" + }, + "primary_metric": "mean prompt-level minimum of per-prompt min-max-normalized helpfulness, safety, conciseness", + "bootstrap": { + "unit": "prompt", + "paired": true, + "resamples": 2000, + "interval": "percentile_95", + "seed": 42 + } +} diff --git a/pair_manifest.json b/pair_manifest.json new file mode 100644 index 0000000..be034b0 --- /dev/null +++ b/pair_manifest.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "objectives": [ + "helpfulness", + "safety", + "conciseness" + ], + "normalization": "per_prompt_minmax", + "adversary": { + "steps": 25, + "alpha": 1.0, + "kappa": 0.05, + "preference_scale": 8.0 + }, + "ronpo_estimator": "Rao-Blackwellized full sigma expectation with three common response pairs per prompt", + "counts": { + "train": { + "avg": 16746, + "ronpo_full_expect": 50340, + "ronpo_k_only": 50340, + "ht_mnpo_helpfulness": 16743, + "ht_mnpo_safety": 16764, + "ht_mnpo_conciseness": 16755 + }, + "validation": { + "avg": 1859, + "ronpo_full_expect": 5592, + "ronpo_k_only": 5592, + "ht_mnpo_helpfulness": 1864, + "ht_mnpo_safety": 1862, + "ht_mnpo_conciseness": 1858 + } + } +} diff --git a/provenance.json b/provenance.json new file mode 100644 index 0000000..61b0e14 --- /dev/null +++ b/provenance.json @@ -0,0 +1,13 @@ +{ + "method": "ronpo_k_only", + "seed": 42, + "attempt": 3, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "89616b47c092", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/89616b47c092", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json" +} diff --git a/resource_profile.json b/resource_profile.json new file mode 100644 index 0000000..18eec47 --- /dev/null +++ b/resource_profile.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "measured_at_kst": "2026-07-12T22:38:00+09:00", + "hardware": "NVIDIA B200 183359 MiB reported total; 178.35 GiB PyTorch-visible capacity", + "smoke_evidence": { + "attempted_profile": {"per_device_batch_size": 2, "gradient_accumulation_steps": 8, "gradient_checkpointing": false, "effective_batch_size": 16}, + "outcome": "OOM before completing the 20-step smoke across every launched unified baseline; representative process used 176.87 GiB and failed a 1.62 GiB allocation. KTO also OOMed at the optimizer step.", + "paper_metric": false + }, + "selected_full_training_profile": { + "per_device_batch_size": 2, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": false, + "attention_implementation": "PyTorch SDPA with cuDNN SDPA disabled", + "effective_batch_size": 16, + "applies_symmetrically_to_all_unified_pairwise_methods": true, + "kto_implementation_constraint": "TRL KTO requires actual batch size > 1. KTO uses batch 2 / accumulation 8 and identical AdamW with foreach=False. PyTorch SDPA is used with cuDNN SDPA disabled. The fixed base reference model is placed on a second B200 and its exact logits are returned to the policy device; the policy/optimizer remain on the first B200. This avoids a measured ~6-hour reference-precompute path while leaving the KTO loss, actual/effective batch, optimizer, and 900-step budget unchanged." + }, + "retry_semantics": "Resource selection is separated from method-stability attempts. Full attempt 1 uses the smoke-validated SDPA batch-2 profile and retains the preregistered method LR/anchor; attempts 2 and 3 fall back to batch 1 / accumulation 16 / gradient checkpointing and apply the declared LR/anchor stabilization recipe.", + "provenance_note": "No reward or sealed-evaluation result was available when this hardware profile was selected." +} diff --git a/run_status.json b/run_status.json new file mode 100644 index 0000000..fe3a15c --- /dev/null +++ b/run_status.json @@ -0,0 +1,14 @@ +{ + "method": "ronpo_k_only", + "seed": 42, + "attempt": 3, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "89616b47c092", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/89616b47c092", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json", + "status": "completed" +} diff --git a/split_manifest.json b/split_manifest.json new file mode 100644 index 0000000..4fb96bd --- /dev/null +++ b/split_manifest.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "split_rule": "sort source-train prompts by sha256('flagship-v1||'+prompt_id); first round(10%) validation", + "validation_fraction": 0.1, + "source_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/avg_oracle", + "base_model": "Qwen/Qwen3-8B", + "counts": { + "train": 16780, + "validation": 1864, + "sealed_test": 604 + }, + "prompt_id_sha256": { + "train": "fcf6f2ea5a0b7bcd41179e6f5288f4d850b2d04bdb729da91907bd18bf733a69", + "validation": "a87ba6f519192164daa487b109a6547dd8cbde46a27a373d6da201ca2d061b8f", + "sealed_test": "b398ced6af6cf73d731e1a5f32a0631d8060323d51eda1e375695566bf5f1867" + }, + "overlaps": { + "train_validation": 0, + "train_sealed_test": 0, + "validation_sealed_test": 0 + }, + "sealed_test_policy": "prompts only until S4a; no objective scores or model generations used in S0-S3" +} diff --git a/stability_gate.json b/stability_gate.json new file mode 100644 index 0000000..cf81a63 --- /dev/null +++ b/stability_gate.json @@ -0,0 +1,38 @@ +{ + "status": "passed", + "passed": true, + "base": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 388.1484375, + "max_words": 1379, + "max_repeat_run": 4 + }, + "candidate": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 391.8359375, + "max_words": 1411, + "max_repeat_run": 4 + }, + "candidate_base_mean_word_ratio": 1.009500231467504, + "thresholds": { + "min_length_ratio": 0.33, + "max_length_ratio": 2.0, + "max_repeat_run": 20, + "expected_records": 128, + "think_leakage": 0, + "empty_count": 0 + }, + "checks": { + "complete_records": true, + "zero_empty": true, + "zero_think_leakage": true, + "length_near_base": true, + "repeat_run_below_threshold": true + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ffd19d2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..aa15823 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.2860548271752086, + "total_flos": 0.0, + "train_loss": 0.42856775469250147, + "train_runtime": 7333.3683, + "train_samples": 50340, + "train_samples_per_second": 1.964, + "train_steps_per_second": 0.123 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..7e114f7 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,4903 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.2860548271752086, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "drift/chosen_abs": 0.2889288067817688, + "drift/rejected_abs": 0.17525029182434082, + "epoch": 0.0015891934843067143, + "grad_norm": 1.0546875, + "learning_rate": 1.111111111111111e-09, + "logits/chosen": -2.227959632873535, + "logits/rejected": -2.153649091720581, + "logps/chosen": -0.25035396218299866, + "logps/rejected": -0.25459814071655273, + "loss": 0.2694305181503296, + "loss/core": 0.24971142411231995, + "loss/preference_sft": 0.2522751986980438, + "loss/reference_anchor": 0.3691546618938446, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.8892879486083984, + "rewards/margins": 1.1368046998977661, + "rewards/rejected": 1.7524830102920532, + "ronpo/logit": 0.11368048191070557, + "ronpo/residual": 0.11023612320423126, + "ronpo/residual_abs": 0.22715234756469727, + "ronpo/target": 0.0034443773329257965, + "ronpo/target_abs": 0.06975062191486359, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 5 + }, + { + "drift/chosen_abs": 0.2513688802719116, + "drift/rejected_abs": 0.17290709912776947, + "epoch": 0.0031783869686134287, + "grad_norm": 4.5, + "learning_rate": 2.5e-09, + "logits/chosen": -1.9819161891937256, + "logits/rejected": -1.923046350479126, + "logps/chosen": -0.26421818137168884, + "logps/rejected": -0.2721954882144928, + "loss": 0.17001162469387054, + "loss/core": 0.15246111154556274, + "loss/preference_sft": 0.2747360169887543, + "loss/reference_anchor": 0.3235367238521576, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.5129077434539795, + "rewards/margins": 0.78448885679245, + "rewards/rejected": 1.7284189462661743, + "ronpo/logit": 0.07844887673854828, + "ronpo/residual": 0.06979227811098099, + "ronpo/residual_abs": 0.19728776812553406, + "ronpo/target": 0.008656605146825314, + "ronpo/target_abs": 0.09380026161670685, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 10 + }, + { + "drift/chosen_abs": 0.14740121364593506, + "drift/rejected_abs": 0.08022782951593399, + "epoch": 0.004767580452920143, + "grad_norm": 7.3125, + "learning_rate": 3.8888888888888884e-09, + "logits/chosen": -2.2918307781219482, + "logits/rejected": -2.2467479705810547, + "logps/chosen": -0.27619749307632446, + "logps/rejected": -0.2822367548942566, + "loss": 0.05183436721563339, + "loss/core": 0.047430895268917084, + "loss/preference_sft": 0.277311235666275, + "loss/reference_anchor": 0.060338426381349564, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.4728786945343018, + "rewards/margins": 0.6718867421150208, + "rewards/rejected": 0.8009918928146362, + "ronpo/logit": 0.06718868762254715, + "ronpo/residual": 0.102389857172966, + "ronpo/residual_abs": 0.14397867023944855, + "ronpo/target": -0.035201169550418854, + "ronpo/target_abs": 0.08456156402826309, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 15 + }, + { + "drift/chosen_abs": 0.1409628540277481, + "drift/rejected_abs": 0.07020283490419388, + "epoch": 0.006356773937226857, + "grad_norm": 1.4765625, + "learning_rate": 5.277777777777778e-09, + "logits/chosen": -2.291287422180176, + "logits/rejected": -2.2165215015411377, + "logps/chosen": -0.27579012513160706, + "logps/rejected": -0.278958261013031, + "loss": 0.03074510395526886, + "loss/core": 0.026744594797492027, + "loss/preference_sft": 0.2790845036506653, + "loss/reference_anchor": 0.052101779729127884, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.409259557723999, + "rewards/margins": 0.7112404704093933, + "rewards/rejected": 0.6980190277099609, + "ronpo/logit": 0.07112405449151993, + "ronpo/residual": 0.08393178880214691, + "ronpo/residual_abs": 0.11383052915334702, + "ronpo/target": -0.012807758525013924, + "ronpo/target_abs": 0.07244060188531876, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 20 + }, + { + "drift/chosen_abs": 0.2341737300157547, + "drift/rejected_abs": 0.07688029110431671, + "epoch": 0.007945967421533572, + "grad_norm": 1.0625, + "learning_rate": 6.666666666666666e-09, + "logits/chosen": -2.354504346847534, + "logits/rejected": -2.393965721130371, + "logps/chosen": -0.2399778664112091, + "logps/rejected": -0.2485647201538086, + "loss": 0.30489665269851685, + "loss/core": 0.29363757371902466, + "loss/preference_sft": 0.2436579465866089, + "loss/reference_anchor": 0.20081612467765808, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.3414459228515625, + "rewards/margins": 1.5734647512435913, + "rewards/rejected": 0.7679811716079712, + "ronpo/logit": 0.15734648704528809, + "ronpo/residual": 0.15989667177200317, + "ronpo/residual_abs": 0.2039765864610672, + "ronpo/target": -0.0025501928757876158, + "ronpo/target_abs": 0.07882604002952576, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 25 + }, + { + "drift/chosen_abs": 0.20403337478637695, + "drift/rejected_abs": 0.08686282485723495, + "epoch": 0.009535160905840286, + "grad_norm": 344.0, + "learning_rate": 8.055555555555556e-09, + "logits/chosen": -2.2312631607055664, + "logits/rejected": -2.1263251304626465, + "logps/chosen": -0.28779473900794983, + "logps/rejected": -0.28351494669914246, + "loss": 0.18434959650039673, + "loss/core": 0.1724001169204712, + "loss/preference_sft": 0.29089125990867615, + "loss/reference_anchor": 0.20990124344825745, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.0403337478637695, + "rewards/margins": 1.1722705364227295, + "rewards/rejected": 0.8680631518363953, + "ronpo/logit": 0.11722707748413086, + "ronpo/residual": 0.10378406196832657, + "ronpo/residual_abs": 0.1859159767627716, + "ronpo/target": 0.013443012721836567, + "ronpo/target_abs": 0.09407193213701248, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 30 + }, + { + "drift/chosen_abs": 0.20877280831336975, + "drift/rejected_abs": 0.13062386214733124, + "epoch": 0.011124354390147001, + "grad_norm": 4.0, + "learning_rate": 9.444444444444444e-09, + "logits/chosen": -2.1255409717559814, + "logits/rejected": -2.06231951713562, + "logps/chosen": -0.2710401117801666, + "logps/rejected": -0.28147274255752563, + "loss": 0.05778823047876358, + "loss/core": 0.04806242510676384, + "loss/preference_sft": 0.27798718214035034, + "loss/reference_anchor": 0.16671733558177948, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.0877280235290527, + "rewards/margins": 0.7836453318595886, + "rewards/rejected": 1.3040828704833984, + "ronpo/logit": 0.0783645361661911, + "ronpo/residual": 0.06768552213907242, + "ronpo/residual_abs": 0.1325724571943283, + "ronpo/target": 0.01067900937050581, + "ronpo/target_abs": 0.08033877611160278, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 35 + }, + { + "drift/chosen_abs": 0.1339804232120514, + "drift/rejected_abs": 0.1121961921453476, + "epoch": 0.012713547874453715, + "grad_norm": 444.0, + "learning_rate": 1.0833333333333333e-08, + "logits/chosen": -2.0807948112487793, + "logits/rejected": -2.042069435119629, + "logps/chosen": -0.31901049613952637, + "logps/rejected": -0.3213141858577728, + "loss": 0.2600531280040741, + "loss/core": 0.2503146529197693, + "loss/preference_sft": 0.31382906436920166, + "loss/reference_anchor": 0.16338738799095154, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.3386170864105225, + "rewards/margins": 0.21692340075969696, + "rewards/rejected": 1.1216938495635986, + "ronpo/logit": 0.021692335605621338, + "ronpo/residual": 0.011326847597956657, + "ronpo/residual_abs": 0.17393314838409424, + "ronpo/target": 0.010365481488406658, + "ronpo/target_abs": 0.08013413846492767, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 40 + }, + { + "drift/chosen_abs": 0.2002873420715332, + "drift/rejected_abs": 0.10398294776678085, + "epoch": 0.014302741358760428, + "grad_norm": 4.0, + "learning_rate": 1.2222222222222222e-08, + "logits/chosen": -2.149164915084839, + "logits/rejected": -2.077511787414551, + "logps/chosen": -0.30421438813209534, + "logps/rejected": -0.29620206356048584, + "loss": 0.09899889677762985, + "loss/core": 0.09022404253482819, + "loss/preference_sft": 0.2990511357784271, + "loss/reference_anchor": 0.14559200406074524, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.0019478797912598, + "rewards/margins": 0.9650747179985046, + "rewards/rejected": 1.0368731021881104, + "ronpo/logit": 0.0965074822306633, + "ronpo/residual": 0.09062016755342484, + "ronpo/residual_abs": 0.1583360731601715, + "ronpo/target": 0.005887320265173912, + "ronpo/target_abs": 0.08980895578861237, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 45 + }, + { + "drift/chosen_abs": 0.21013769507408142, + "drift/rejected_abs": 0.1717991977930069, + "epoch": 0.015891934843067144, + "grad_norm": 7.65625, + "learning_rate": 1.361111111111111e-08, + "logits/chosen": -2.2315473556518555, + "logits/rejected": -2.2804598808288574, + "logps/chosen": -0.38569122552871704, + "logps/rejected": -0.37548166513442993, + "loss": 0.49833402037620544, + "loss/core": 0.47320446372032166, + "loss/preference_sft": 0.3764234185218811, + "loss/reference_anchor": 0.4649490416049957, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.10087251663208, + "rewards/margins": 1.4145400524139404, + "rewards/rejected": 0.6863324642181396, + "ronpo/logit": 0.14145398139953613, + "ronpo/residual": 0.14585429430007935, + "ronpo/residual_abs": 0.2349962443113327, + "ronpo/target": -0.004400297999382019, + "ronpo/target_abs": 0.0763171911239624, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 50 + }, + { + "drift/chosen_abs": 0.24356035888195038, + "drift/rejected_abs": 0.1136392131447792, + "epoch": 0.01748112832737386, + "grad_norm": 1.1015625, + "learning_rate": 1.5e-08, + "logits/chosen": -2.132157802581787, + "logits/rejected": -2.104531764984131, + "logps/chosen": -0.30124813318252563, + "logps/rejected": -0.29356569051742554, + "loss": 0.29936105012893677, + "loss/core": 0.28574955463409424, + "loss/preference_sft": 0.28952398896217346, + "loss/reference_anchor": 0.2432773858308792, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.4335732460021973, + "rewards/margins": 1.3003393411636353, + "rewards/rejected": 1.133233666419983, + "ronpo/logit": 0.130033940076828, + "ronpo/residual": 0.13970455527305603, + "ronpo/residual_abs": 0.19125238060951233, + "ronpo/target": -0.009670635685324669, + "ronpo/target_abs": 0.07653583586215973, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 55 + }, + { + "drift/chosen_abs": 0.2864144444465637, + "drift/rejected_abs": 0.0761997327208519, + "epoch": 0.01907032181168057, + "grad_norm": 7.1875, + "learning_rate": 1.6388888888888888e-08, + "logits/chosen": -2.0191774368286133, + "logits/rejected": -2.0377559661865234, + "logps/chosen": -0.2580411732196808, + "logps/rejected": -0.2605469822883606, + "loss": 0.6949197053909302, + "loss/core": 0.6738607883453369, + "loss/preference_sft": 0.2634747624397278, + "loss/reference_anchor": 0.39483022689819336, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 2.863614559173584, + "rewards/margins": 2.115543842315674, + "rewards/rejected": 0.7480708360671997, + "ronpo/logit": 0.2115544080734253, + "ronpo/residual": 0.21217253804206848, + "ronpo/residual_abs": 0.25770795345306396, + "ronpo/target": -0.0006181513890624046, + "ronpo/target_abs": 0.07183288037776947, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 60 + }, + { + "drift/chosen_abs": 0.23456306755542755, + "drift/rejected_abs": 0.0846903845667839, + "epoch": 0.020659515295987287, + "grad_norm": 168.0, + "learning_rate": 1.7777777777777777e-08, + "logits/chosen": -2.3910279273986816, + "logits/rejected": -2.4622697830200195, + "logps/chosen": -0.25069719552993774, + "logps/rejected": -0.24407532811164856, + "loss": 0.2673589289188385, + "loss/core": 0.2575306296348572, + "loss/preference_sft": 0.24895858764648438, + "loss/reference_anchor": 0.17166933417320251, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.345244884490967, + "rewards/margins": 1.499186635017395, + "rewards/rejected": 0.846058189868927, + "ronpo/logit": 0.14991866052150726, + "ronpo/residual": 0.12659938633441925, + "ronpo/residual_abs": 0.19241955876350403, + "ronpo/target": 0.02331927977502346, + "ronpo/target_abs": 0.07524405419826508, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 65 + }, + { + "drift/chosen_abs": 0.21925799548625946, + "drift/rejected_abs": 0.11180920898914337, + "epoch": 0.022248708780294002, + "grad_norm": 2.71875, + "learning_rate": 1.9166666666666666e-08, + "logits/chosen": -2.1304917335510254, + "logits/rejected": -2.13004994392395, + "logps/chosen": -0.26294752955436707, + "logps/rejected": -0.26812273263931274, + "loss": 0.0758952647447586, + "loss/core": 0.06884925067424774, + "loss/preference_sft": 0.26684775948524475, + "loss/reference_anchor": 0.11423534154891968, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.1923234462738037, + "rewards/margins": 1.074231505393982, + "rewards/rejected": 1.1180920600891113, + "ronpo/logit": 0.10742314904928207, + "ronpo/residual": 0.09753496944904327, + "ronpo/residual_abs": 0.14825595915317535, + "ronpo/target": 0.00988816749304533, + "ronpo/target_abs": 0.08186240494251251, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 70 + }, + { + "drift/chosen_abs": 0.2804720997810364, + "drift/rejected_abs": 0.1483515202999115, + "epoch": 0.023837902264600714, + "grad_norm": 5.84375, + "learning_rate": 2.0555555555555552e-08, + "logits/chosen": -2.107518434524536, + "logits/rejected": -2.0409445762634277, + "logps/chosen": -0.2804359197616577, + "logps/rejected": -0.27236875891685486, + "loss": 0.681021511554718, + "loss/core": 0.6588231921195984, + "loss/preference_sft": 0.271370530128479, + "loss/reference_anchor": 0.41682863235473633, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.8040549755096436, + "rewards/margins": 1.3575153350830078, + "rewards/rejected": 1.4465398788452148, + "ronpo/logit": 0.13575153052806854, + "ronpo/residual": 0.14553678035736084, + "ronpo/residual_abs": 0.30461931228637695, + "ronpo/target": -0.009785245172679424, + "ronpo/target_abs": 0.08345939964056015, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 75 + }, + { + "drift/chosen_abs": 0.32475045323371887, + "drift/rejected_abs": 0.13860228657722473, + "epoch": 0.02542709574890743, + "grad_norm": 42.5, + "learning_rate": 2.1944444444444445e-08, + "logits/chosen": -2.056220531463623, + "logits/rejected": -2.0234427452087402, + "logps/chosen": -0.27260422706604004, + "logps/rejected": -0.27340400218963623, + "loss": 0.372364342212677, + "loss/core": 0.35138756036758423, + "loss/preference_sft": 0.2743496894836426, + "loss/reference_anchor": 0.39210090041160583, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.2468693256378174, + "rewards/margins": 1.8645988702774048, + "rewards/rejected": 1.3822702169418335, + "ronpo/logit": 0.18645988404750824, + "ronpo/residual": 0.18637283146381378, + "ronpo/residual_abs": 0.2369922697544098, + "ronpo/target": 8.704736683284864e-05, + "ronpo/target_abs": 0.08356542885303497, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 80 + }, + { + "drift/chosen_abs": 0.22833116352558136, + "drift/rejected_abs": 0.07853401452302933, + "epoch": 0.027016289233214145, + "grad_norm": 988.0, + "learning_rate": 2.3333333333333334e-08, + "logits/chosen": -2.3675479888916016, + "logits/rejected": -2.3091373443603516, + "logps/chosen": -0.2940252423286438, + "logps/rejected": -0.2664296329021454, + "loss": 0.5738587975502014, + "loss/core": 0.5565118789672852, + "loss/preference_sft": 0.2698623538017273, + "loss/reference_anchor": 0.31994983553886414, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.281959056854248, + "rewards/margins": 1.5006506443023682, + "rewards/rejected": 0.7813081741333008, + "ronpo/logit": 0.1500650942325592, + "ronpo/residual": 0.13329221308231354, + "ronpo/residual_abs": 0.20770840346813202, + "ronpo/target": 0.01677287369966507, + "ronpo/target_abs": 0.08873967826366425, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 85 + }, + { + "drift/chosen_abs": 0.1896965652704239, + "drift/rejected_abs": 0.16110625863075256, + "epoch": 0.028605482717520857, + "grad_norm": 1.7578125, + "learning_rate": 2.4722222222222223e-08, + "logits/chosen": -2.261641025543213, + "logits/rejected": -2.271144390106201, + "logps/chosen": -0.2831428349018097, + "logps/rejected": -0.2776196002960205, + "loss": 0.27627962827682495, + "loss/core": 0.2642122805118561, + "loss/preference_sft": 0.27972379326820374, + "loss/reference_anchor": 0.21337434649467468, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.896965742111206, + "rewards/margins": 0.2912505865097046, + "rewards/rejected": 1.6057151556015015, + "ronpo/logit": 0.029125045984983444, + "ronpo/residual": 0.029582571238279343, + "ronpo/residual_abs": 0.18990905582904816, + "ronpo/target": -0.0004575133207254112, + "ronpo/target_abs": 0.08104494959115982, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 90 + }, + { + "drift/chosen_abs": 0.22530770301818848, + "drift/rejected_abs": 0.10128001868724823, + "epoch": 0.030194676201827572, + "grad_norm": 3.734375, + "learning_rate": 2.4998495746616843e-08, + "logits/chosen": -2.339174747467041, + "logits/rejected": -2.257371425628662, + "logps/chosen": -0.27327409386634827, + "logps/rejected": -0.26891353726387024, + "loss": 0.345085084438324, + "loss/core": 0.33327919244766235, + "loss/preference_sft": 0.26861560344696045, + "loss/reference_anchor": 0.20925641059875488, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.2530770301818848, + "rewards/margins": 1.2411054372787476, + "rewards/rejected": 1.0119714736938477, + "ronpo/logit": 0.12411054223775864, + "ronpo/residual": 0.1508522927761078, + "ronpo/residual_abs": 0.2290908843278885, + "ronpo/target": -0.0267417524009943, + "ronpo/target_abs": 0.08890058100223541, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 95 + }, + { + "drift/chosen_abs": 0.2532444894313812, + "drift/rejected_abs": 0.09977379441261292, + "epoch": 0.03178386968613429, + "grad_norm": 8.625, + "learning_rate": 2.4992385337738698e-08, + "logits/chosen": -1.9698337316513062, + "logits/rejected": -2.0415987968444824, + "logps/chosen": -0.29794543981552124, + "logps/rejected": -0.28950685262680054, + "loss": 0.24947309494018555, + "loss/core": 0.23802785575389862, + "loss/preference_sft": 0.29647988080978394, + "loss/reference_anchor": 0.1992568075656891, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.531574249267578, + "rewards/margins": 1.5353649854660034, + "rewards/rejected": 0.9962092638015747, + "ronpo/logit": 0.15353651344776154, + "ronpo/residual": 0.15389743447303772, + "ronpo/residual_abs": 0.2095203399658203, + "ronpo/target": -0.00036093126982450485, + "ronpo/target_abs": 0.08967583626508713, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 100 + }, + { + "drift/chosen_abs": 0.3917911648750305, + "drift/rejected_abs": 0.127794548869133, + "epoch": 0.033373063170441, + "grad_norm": 93.5, + "learning_rate": 2.4981577053636144e-08, + "logits/chosen": -1.9294116497039795, + "logits/rejected": -1.936621904373169, + "logps/chosen": -0.2786913514137268, + "logps/rejected": -0.2831571400165558, + "loss": 0.699501633644104, + "loss/core": 0.6670955419540405, + "loss/preference_sft": 0.2841565012931824, + "loss/reference_anchor": 0.6197081208229065, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.9174468517303467, + "rewards/margins": 2.6591506004333496, + "rewards/rejected": 1.258296251296997, + "ronpo/logit": 0.26591506600379944, + "ronpo/residual": 0.2671971917152405, + "ronpo/residual_abs": 0.31767672300338745, + "ronpo/target": -0.0012821194250136614, + "ronpo/target_abs": 0.08636602014303207, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 105 + }, + { + "drift/chosen_abs": 0.2426718771457672, + "drift/rejected_abs": 0.12293334305286407, + "epoch": 0.03496225665474772, + "grad_norm": 8.375, + "learning_rate": 2.4966074958862813e-08, + "logits/chosen": -2.245562791824341, + "logits/rejected": -2.205705165863037, + "logps/chosen": -0.29507607221603394, + "logps/rejected": -0.2873607277870178, + "loss": 0.12289254367351532, + "loss/core": 0.1113683208823204, + "loss/preference_sft": 0.29041385650634766, + "loss/reference_anchor": 0.20144303143024445, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.424837589263916, + "rewards/margins": 1.2106255292892456, + "rewards/rejected": 1.21421217918396, + "ronpo/logit": 0.12106256186962128, + "ronpo/residual": 0.11163101345300674, + "ronpo/residual_abs": 0.1918853521347046, + "ronpo/target": 0.009431545622646809, + "ronpo/target_abs": 0.10175038874149323, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 110 + }, + { + "drift/chosen_abs": 0.27447623014450073, + "drift/rejected_abs": 0.13834591209888458, + "epoch": 0.03655145013905443, + "grad_norm": 1.7265625, + "learning_rate": 2.4945884883122553e-08, + "logits/chosen": -1.8843482732772827, + "logits/rejected": -1.775320053100586, + "logps/chosen": -0.2798824906349182, + "logps/rejected": -0.2814459502696991, + "loss": 0.1541265845298767, + "loss/core": 0.14217844605445862, + "loss/preference_sft": 0.27655261754989624, + "loss/reference_anchor": 0.21130748093128204, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.742548942565918, + "rewards/margins": 1.366241693496704, + "rewards/rejected": 1.376307725906372, + "ronpo/logit": 0.13662415742874146, + "ronpo/residual": 0.14770525693893433, + "ronpo/residual_abs": 0.18340587615966797, + "ronpo/target": -0.011081096716225147, + "ronpo/target_abs": 0.07495449483394623, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 115 + }, + { + "drift/chosen_abs": 0.25327441096305847, + "drift/rejected_abs": 0.11703983694314957, + "epoch": 0.03814064362336114, + "grad_norm": 0.9609375, + "learning_rate": 2.492101441907714e-08, + "logits/chosen": -2.0191433429718018, + "logits/rejected": -2.067561626434326, + "logps/chosen": -0.30190160870552063, + "logps/rejected": -0.30017122626304626, + "loss": 0.30577465891838074, + "loss/core": 0.28840842843055725, + "loss/preference_sft": 0.2985996901988983, + "loss/reference_anchor": 0.31746405363082886, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.5322983264923096, + "rewards/margins": 1.3638817071914673, + "rewards/rejected": 1.1684163808822632, + "ronpo/logit": 0.1363881528377533, + "ronpo/residual": 0.11740622669458389, + "ronpo/residual_abs": 0.22061511874198914, + "ronpo/target": 0.0189819298684597, + "ronpo/target_abs": 0.08912766724824905, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 120 + }, + { + "drift/chosen_abs": 0.12704667448997498, + "drift/rejected_abs": 0.07079822570085526, + "epoch": 0.03972983710766786, + "grad_norm": 1.2265625, + "learning_rate": 2.4891472919490977e-08, + "logits/chosen": -2.272683620452881, + "logits/rejected": -2.193810224533081, + "logps/chosen": -0.2744141221046448, + "logps/rejected": -0.2814023196697235, + "loss": 0.04335567355155945, + "loss/core": 0.03851154074072838, + "loss/preference_sft": 0.26887914538383484, + "loss/reference_anchor": 0.06999474763870239, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.2701770067214966, + "rewards/margins": 0.5637924671173096, + "rewards/rejected": 0.7063844799995422, + "ronpo/logit": 0.05637924745678902, + "ronpo/residual": 0.0646718442440033, + "ronpo/residual_abs": 0.11568011343479156, + "ronpo/target": -0.008292593993246555, + "ronpo/target_abs": 0.08368505537509918, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 125 + }, + { + "drift/chosen_abs": 0.3699837327003479, + "drift/rejected_abs": 0.1531117707490921, + "epoch": 0.04131903059197457, + "grad_norm": 15.5, + "learning_rate": 2.4857271493713895e-08, + "logits/chosen": -2.0499277114868164, + "logits/rejected": -1.9798452854156494, + "logps/chosen": -0.2802344560623169, + "logps/rejected": -0.3006633222103119, + "loss": 0.4787680506706238, + "loss/core": 0.4418778419494629, + "loss/preference_sft": 0.2814800441265106, + "loss/reference_anchor": 0.7096576690673828, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.6996917724609375, + "rewards/margins": 2.1705682277679443, + "rewards/rejected": 1.529123306274414, + "ronpo/logit": 0.21705684065818787, + "ronpo/residual": 0.2116149216890335, + "ronpo/residual_abs": 0.26738792657852173, + "ronpo/target": 0.005441917572170496, + "ronpo/target_abs": 0.07667852938175201, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 130 + }, + { + "drift/chosen_abs": 0.2286110669374466, + "drift/rejected_abs": 0.17728818953037262, + "epoch": 0.04290822407628129, + "grad_norm": 99.5, + "learning_rate": 2.481842300350339e-08, + "logits/chosen": -2.156745195388794, + "logits/rejected": -2.1426777839660645, + "logps/chosen": -0.2545807361602783, + "logps/rejected": -0.2720378339290619, + "loss": 0.4073014259338379, + "loss/core": 0.39105960726737976, + "loss/preference_sft": 0.26137062907218933, + "loss/reference_anchor": 0.2986990809440613, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.285310745239258, + "rewards/margins": 0.5148512125015259, + "rewards/rejected": 1.770459532737732, + "ronpo/logit": 0.05148511379957199, + "ronpo/residual": 0.06029368191957474, + "ronpo/residual_abs": 0.22104015946388245, + "ronpo/target": -0.00880857277661562, + "ronpo/target_abs": 0.08236870914697647, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 135 + }, + { + "drift/chosen_abs": 0.21387775242328644, + "drift/rejected_abs": 0.1563216596841812, + "epoch": 0.044497417560588004, + "grad_norm": 41.75, + "learning_rate": 2.4774942058187855e-08, + "logits/chosen": -2.0665993690490723, + "logits/rejected": -2.035794734954834, + "logps/chosen": -0.28517797589302063, + "logps/rejected": -0.28532570600509644, + "loss": 0.26362624764442444, + "loss/core": 0.2508805990219116, + "loss/preference_sft": 0.28584787249565125, + "loss/reference_anchor": 0.22632861137390137, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.1381399631500244, + "rewards/margins": 0.5762137174606323, + "rewards/rejected": 1.561926245689392, + "ronpo/logit": 0.05762135982513428, + "ronpo/residual": 0.059470035135746, + "ronpo/residual_abs": 0.2042684555053711, + "ronpo/target": -0.0018486687913537025, + "ronpo/target_abs": 0.08461553603410721, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 140 + }, + { + "drift/chosen_abs": 0.3112896680831909, + "drift/rejected_abs": 0.13714732229709625, + "epoch": 0.04608661104489471, + "grad_norm": 226.0, + "learning_rate": 2.472684500917257e-08, + "logits/chosen": -2.104536771774292, + "logits/rejected": -2.14062237739563, + "logps/chosen": -0.2743372917175293, + "logps/rejected": -0.2655439078807831, + "loss": 0.2187369167804718, + "loss/core": 0.1985674947500229, + "loss/preference_sft": 0.2756105661392212, + "loss/reference_anchor": 0.3758276104927063, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.1124117374420166, + "rewards/margins": 1.7423970699310303, + "rewards/rejected": 1.3700147867202759, + "ronpo/logit": 0.17423972487449646, + "ronpo/residual": 0.18899571895599365, + "ronpo/residual_abs": 0.2230168879032135, + "ronpo/target": -0.014756004326045513, + "ronpo/target_abs": 0.08157791197299957, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 145 + }, + { + "drift/chosen_abs": 0.4021173417568207, + "drift/rejected_abs": 0.11387000232934952, + "epoch": 0.04767580452920143, + "grad_norm": 2.390625, + "learning_rate": 2.467414994379065e-08, + "logits/chosen": -2.0298092365264893, + "logits/rejected": -2.1323561668395996, + "logps/chosen": -0.29558226466178894, + "logps/rejected": -0.27471041679382324, + "loss": 1.039489507675171, + "loss/core": 1.0037477016448975, + "loss/preference_sft": 0.286141037940979, + "loss/reference_anchor": 0.6862256526947021, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 4.021132469177246, + "rewards/margins": 2.884185314178467, + "rewards/rejected": 1.136947751045227, + "ronpo/logit": 0.28841856122016907, + "ronpo/residual": 0.2740480601787567, + "ronpo/residual_abs": 0.33974286913871765, + "ronpo/target": 0.014370448887348175, + "ronpo/target_abs": 0.08442968875169754, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 150 + }, + { + "drift/chosen_abs": 0.3120030462741852, + "drift/rejected_abs": 0.10412098467350006, + "epoch": 0.04926499801350814, + "grad_norm": 3.84375, + "learning_rate": 2.4616876678501113e-08, + "logits/chosen": -2.170553207397461, + "logits/rejected": -2.1755318641662598, + "logps/chosen": -0.265367329120636, + "logps/rejected": -0.26216381788253784, + "loss": 0.7199813723564148, + "loss/core": 0.6925104260444641, + "loss/preference_sft": 0.2665371000766754, + "loss/reference_anchor": 0.522765040397644, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.119339942932129, + "rewards/margins": 2.079453229904175, + "rewards/rejected": 1.0398868322372437, + "ronpo/logit": 0.207945317029953, + "ronpo/residual": 0.22640779614448547, + "ronpo/residual_abs": 0.26298147439956665, + "ronpo/target": -0.018462475389242172, + "ronpo/target_abs": 0.07875766605138779, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 155 + }, + { + "drift/chosen_abs": 0.1994825154542923, + "drift/rejected_abs": 0.06770407408475876, + "epoch": 0.05085419149781486, + "grad_norm": 856.0, + "learning_rate": 2.4555046751436738e-08, + "logits/chosen": -2.207547426223755, + "logits/rejected": -2.225343942642212, + "logps/chosen": -0.32042816281318665, + "logps/rejected": -0.2738822400569916, + "loss": 0.559381365776062, + "loss/core": 0.5423927903175354, + "loss/preference_sft": 0.2767319679260254, + "loss/reference_anchor": 0.3120982050895691, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.9948251247406006, + "rewards/margins": 1.318454384803772, + "rewards/rejected": 0.6763707399368286, + "ronpo/logit": 0.13184544444084167, + "ronpo/residual": 0.1333637535572052, + "ronpo/residual_abs": 0.19957098364830017, + "ronpo/target": -0.0015183339128270745, + "ronpo/target_abs": 0.09095166623592377, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 160 + }, + { + "drift/chosen_abs": 0.13100005686283112, + "drift/rejected_abs": 0.05130317807197571, + "epoch": 0.052443384982121574, + "grad_norm": 4.15625, + "learning_rate": 2.4488683414304425e-08, + "logits/chosen": -2.1318647861480713, + "logits/rejected": -2.010211229324341, + "logps/chosen": -0.28451108932495117, + "logps/rejected": -0.29346317052841187, + "loss": 0.1088658794760704, + "loss/core": 0.10401229560375214, + "loss/preference_sft": 0.2892284393310547, + "loss/reference_anchor": 0.06814897060394287, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.3097032308578491, + "rewards/margins": 0.8015567660331726, + "rewards/rejected": 0.5081464052200317, + "ronpo/logit": 0.08015568554401398, + "ronpo/residual": 0.07226266711950302, + "ronpo/residual_abs": 0.1403239220380783, + "ronpo/target": 0.007893010973930359, + "ronpo/target_abs": 0.07677415758371353, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 165 + }, + { + "drift/chosen_abs": 0.317328542470932, + "drift/rejected_abs": 0.16766056418418884, + "epoch": 0.05403257846642829, + "grad_norm": 1.078125, + "learning_rate": 2.4417811623641206e-08, + "logits/chosen": -2.027057647705078, + "logits/rejected": -2.019274950027466, + "logps/chosen": -0.2885628640651703, + "logps/rejected": -0.2752649486064911, + "loss": 0.4365611672401428, + "loss/core": 0.4138968586921692, + "loss/preference_sft": 0.28210192918777466, + "loss/reference_anchor": 0.4250755310058594, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.172837734222412, + "rewards/margins": 1.4988738298416138, + "rewards/rejected": 1.6739639043807983, + "ronpo/logit": 0.14988741278648376, + "ronpo/residual": 0.13528050482273102, + "ronpo/residual_abs": 0.2508905827999115, + "ronpo/target": 0.014606885612010956, + "ronpo/target_abs": 0.07879899442195892, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 170 + }, + { + "drift/chosen_abs": 0.19914527237415314, + "drift/rejected_abs": 0.11187348514795303, + "epoch": 0.055621771950735005, + "grad_norm": 30.125, + "learning_rate": 2.4342458031429115e-08, + "logits/chosen": -2.262404680252075, + "logits/rejected": -2.1995086669921875, + "logps/chosen": -0.2932138741016388, + "logps/rejected": -0.30438342690467834, + "loss": 0.06648578494787216, + "loss/core": 0.05691850930452347, + "loss/preference_sft": 0.29610422253608704, + "loss/reference_anchor": 0.16173511743545532, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9914528131484985, + "rewards/margins": 0.8727973103523254, + "rewards/rejected": 1.1186553239822388, + "ronpo/logit": 0.08727972954511642, + "ronpo/residual": 0.0997995063662529, + "ronpo/residual_abs": 0.1461952179670334, + "ronpo/target": -0.012519779615104198, + "ronpo/target_abs": 0.08836988359689713, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 175 + }, + { + "drift/chosen_abs": 0.4732588231563568, + "drift/rejected_abs": 0.22694897651672363, + "epoch": 0.057210965435041714, + "grad_norm": 37.25, + "learning_rate": 2.4262650975072444e-08, + "logits/chosen": -2.074978828430176, + "logits/rejected": -2.051633358001709, + "logps/chosen": -0.2896614074707031, + "logps/rejected": -0.28108644485473633, + "loss": 0.9334848523139954, + "loss/core": 0.8751218914985657, + "loss/preference_sft": 0.2792913615703583, + "loss/reference_anchor": 1.1393296718597412, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.732588291168213, + "rewards/margins": 2.463618755340576, + "rewards/rejected": 2.2689692974090576, + "ronpo/logit": 0.24636192619800568, + "ronpo/residual": 0.2580645680427551, + "ronpo/residual_abs": 0.31068843603134155, + "ronpo/target": -0.011702699586749077, + "ronpo/target_abs": 0.07820998132228851, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 180 + }, + { + "drift/chosen_abs": 0.12275053560733795, + "drift/rejected_abs": 0.06439492851495743, + "epoch": 0.05880015891934843, + "grad_norm": 5.625, + "learning_rate": 2.4178420466741218e-08, + "logits/chosen": -2.226714849472046, + "logits/rejected": -2.3291454315185547, + "logps/chosen": -0.29459166526794434, + "logps/rejected": -0.29199695587158203, + "loss": 0.03509552404284477, + "loss/core": 0.03178117424249649, + "loss/preference_sft": 0.2933792173862457, + "loss/reference_anchor": 0.036949075758457184, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.227303385734558, + "rewards/margins": 0.5856597423553467, + "rewards/rejected": 0.6416436433792114, + "ronpo/logit": 0.058565981686115265, + "ronpo/residual": 0.07432404160499573, + "ronpo/residual_abs": 0.13376492261886597, + "ronpo/target": -0.01575806364417076, + "ronpo/target_abs": 0.10001115500926971, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 185 + }, + { + "drift/chosen_abs": 0.12490232288837433, + "drift/rejected_abs": 0.055704616010189056, + "epoch": 0.060389352403655144, + "grad_norm": 0.9296875, + "learning_rate": 2.408979818208484e-08, + "logits/chosen": -2.3149032592773438, + "logits/rejected": -2.411414861679077, + "logps/chosen": -0.30525779724121094, + "logps/rejected": -0.3026205003261566, + "loss": 0.052305638790130615, + "loss/core": 0.04764842614531517, + "loss/preference_sft": 0.3053823411464691, + "loss/reference_anchor": 0.06260595470666885, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.247715950012207, + "rewards/margins": 0.6911423802375793, + "rewards/rejected": 0.5565735697746277, + "ronpo/logit": 0.06911423802375793, + "ronpo/residual": 0.0638674646615982, + "ronpo/residual_abs": 0.12851160764694214, + "ronpo/target": 0.005246774293482304, + "ronpo/target_abs": 0.08141742646694183, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 190 + }, + { + "drift/chosen_abs": 0.21187469363212585, + "drift/rejected_abs": 0.11845757067203522, + "epoch": 0.06197854588796186, + "grad_norm": 6.75, + "learning_rate": 2.3996817448320193e-08, + "logits/chosen": -2.055734872817993, + "logits/rejected": -2.031132698059082, + "logps/chosen": -0.27869275212287903, + "logps/rejected": -0.27192652225494385, + "loss": 0.06286276876926422, + "loss/core": 0.055714428424835205, + "loss/preference_sft": 0.26679545640945435, + "loss/reference_anchor": 0.11628717184066772, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.1181392669677734, + "rewards/margins": 0.9341791868209839, + "rewards/rejected": 1.1839600801467896, + "ronpo/logit": 0.0934179276227951, + "ronpo/residual": 0.09014447033405304, + "ronpo/residual_abs": 0.14351728558540344, + "ronpo/target": 0.0032734572887420654, + "ronpo/target_abs": 0.08329368382692337, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 195 + }, + { + "drift/chosen_abs": 0.2836345136165619, + "drift/rejected_abs": 0.08946871757507324, + "epoch": 0.06356773937226858, + "grad_norm": 8.1875, + "learning_rate": 2.389951323169861e-08, + "logits/chosen": -2.040783166885376, + "logits/rejected": -2.161843776702881, + "logps/chosen": -0.2890108823776245, + "logps/rejected": -0.2789885699748993, + "loss": 0.4890855848789215, + "loss/core": 0.4716506600379944, + "loss/preference_sft": 0.28799134492874146, + "loss/reference_anchor": 0.31989890336990356, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.835172176361084, + "rewards/margins": 1.9416320323944092, + "rewards/rejected": 0.8935402035713196, + "ronpo/logit": 0.1941632330417633, + "ronpo/residual": 0.18651123344898224, + "ronpo/residual_abs": 0.23668105900287628, + "ronpo/target": 0.007652005646377802, + "ronpo/target_abs": 0.07496481388807297, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 200 + }, + { + "drift/chosen_abs": 0.2938988208770752, + "drift/rejected_abs": 0.10776060819625854, + "epoch": 0.06515693285657528, + "grad_norm": 94.0, + "learning_rate": 2.3797922124356508e-08, + "logits/chosen": -2.003150463104248, + "logits/rejected": -2.0846657752990723, + "logps/chosen": -0.28210049867630005, + "logps/rejected": -0.28749504685401917, + "loss": 0.755161464214325, + "loss/core": 0.7267810106277466, + "loss/preference_sft": 0.2828027009963989, + "loss/reference_anchor": 0.5393306016921997, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.935922861099243, + "rewards/margins": 1.859521508216858, + "rewards/rejected": 1.0764009952545166, + "ronpo/logit": 0.18595215678215027, + "ronpo/residual": 0.21726493537425995, + "ronpo/residual_abs": 0.2416853904724121, + "ronpo/target": -0.03131275624036789, + "ronpo/target_abs": 0.07854778319597244, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 205 + }, + { + "drift/chosen_abs": 0.21909186244010925, + "drift/rejected_abs": 0.05819239094853401, + "epoch": 0.066746126340882, + "grad_norm": 2.984375, + "learning_rate": 2.3692082330554585e-08, + "logits/chosen": -2.2509193420410156, + "logits/rejected": -2.169978141784668, + "logps/chosen": -0.307937890291214, + "logps/rejected": -0.31472259759902954, + "loss": 1.0144634246826172, + "loss/core": 0.9857050776481628, + "loss/preference_sft": 0.31031107902526855, + "loss/reference_anchor": 0.544138491153717, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.1909189224243164, + "rewards/margins": 1.6100225448608398, + "rewards/rejected": 0.5808963179588318, + "ronpo/logit": 0.1610022485256195, + "ronpo/residual": 0.17502841353416443, + "ronpo/residual_abs": 0.22845324873924255, + "ronpo/target": -0.01402618270367384, + "ronpo/target_abs": 0.08586090058088303, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 210 + }, + { + "drift/chosen_abs": 0.32701677083969116, + "drift/rejected_abs": 0.16467604041099548, + "epoch": 0.06833531982518871, + "grad_norm": 1.0234375, + "learning_rate": 2.3582033652310766e-08, + "logits/chosen": -2.1693928241729736, + "logits/rejected": -2.157107353210449, + "logps/chosen": -0.26543909311294556, + "logps/rejected": -0.2721554636955261, + "loss": 0.30226364731788635, + "loss/core": 0.27254778146743774, + "loss/preference_sft": 0.2698844075202942, + "loss/reference_anchor": 0.5673290491104126, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 3.270167827606201, + "rewards/margins": 1.624214768409729, + "rewards/rejected": 1.6459531784057617, + "ronpo/logit": 0.16242146492004395, + "ronpo/residual": 0.15607237815856934, + "ronpo/residual_abs": 0.21735577285289764, + "ronpo/target": 0.006349098868668079, + "ronpo/target_abs": 0.0811850056052208, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 215 + }, + { + "drift/chosen_abs": 0.1198434829711914, + "drift/rejected_abs": 0.08144821226596832, + "epoch": 0.06992451330949544, + "grad_norm": 0.71875, + "learning_rate": 2.3467817474432273e-08, + "logits/chosen": -2.216386079788208, + "logits/rejected": -2.084725856781006, + "logps/chosen": -0.30442219972610474, + "logps/rejected": -0.29973161220550537, + "loss": 0.05327620357275009, + "loss/core": 0.04937787353992462, + "loss/preference_sft": 0.3065362572669983, + "loss/reference_anchor": 0.04731312766671181, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.1957999467849731, + "rewards/margins": 0.38699817657470703, + "rewards/rejected": 0.8088018298149109, + "ronpo/logit": 0.038699813187122345, + "ronpo/residual": 0.05509736388921738, + "ronpo/residual_abs": 0.13401086628437042, + "ronpo/target": -0.016397548839449883, + "ronpo/target_abs": 0.08902524411678314, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 220 + }, + { + "drift/chosen_abs": 0.23227110505104065, + "drift/rejected_abs": 0.10326921939849854, + "epoch": 0.07151370679380215, + "grad_norm": 6.21875, + "learning_rate": 2.3349476748952507e-08, + "logits/chosen": -2.1769394874572754, + "logits/rejected": -2.194044351577759, + "logps/chosen": -0.28651732206344604, + "logps/rejected": -0.28518351912498474, + "loss": 0.1896631419658661, + "loss/core": 0.17952075600624084, + "loss/preference_sft": 0.27889543771743774, + "loss/reference_anchor": 0.1749579906463623, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.32271146774292, + "rewards/margins": 1.2900190353393555, + "rewards/rejected": 1.0326921939849854, + "ronpo/logit": 0.1290019154548645, + "ronpo/residual": 0.11321617662906647, + "ronpo/residual_abs": 0.17566923797130585, + "ronpo/target": 0.015785735100507736, + "ronpo/target_abs": 0.07616916298866272, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 225 + }, + { + "drift/chosen_abs": 0.26629695296287537, + "drift/rejected_abs": 0.09605975449085236, + "epoch": 0.07310290027810885, + "grad_norm": 2.546875, + "learning_rate": 2.3227055978978543e-08, + "logits/chosen": -2.242928981781006, + "logits/rejected": -2.2605459690093994, + "logps/chosen": -0.26110613346099854, + "logps/rejected": -0.2610822319984436, + "loss": 0.419999897480011, + "loss/core": 0.40420547127723694, + "loss/preference_sft": 0.2564738988876343, + "loss/reference_anchor": 0.2902412414550781, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.662400960922241, + "rewards/margins": 1.7032798528671265, + "rewards/rejected": 0.9591208696365356, + "ronpo/logit": 0.17032800614833832, + "ronpo/residual": 0.1783141791820526, + "ronpo/residual_abs": 0.22243869304656982, + "ronpo/target": -0.00798617023974657, + "ronpo/target_abs": 0.09001145511865616, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 230 + }, + { + "drift/chosen_abs": 0.46215638518333435, + "drift/rejected_abs": 0.23638899624347687, + "epoch": 0.07469209376241558, + "grad_norm": 50.5, + "learning_rate": 2.310060120195532e-08, + "logits/chosen": -2.113866090774536, + "logits/rejected": -2.116377115249634, + "logps/chosen": -0.2798251509666443, + "logps/rejected": -0.2868059575557709, + "loss": 1.1930959224700928, + "loss/core": 1.1315306425094604, + "loss/preference_sft": 0.27541080117225647, + "loss/reference_anchor": 1.20376455783844, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.620245933532715, + "rewards/margins": 2.3461127281188965, + "rewards/rejected": 2.2741329669952393, + "ronpo/logit": 0.23461131751537323, + "ronpo/residual": 0.237228661775589, + "ronpo/residual_abs": 0.40296974778175354, + "ronpo/target": -0.0026173379737883806, + "ronpo/target_abs": 0.08310168236494064, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 235 + }, + { + "drift/chosen_abs": 0.2902994751930237, + "drift/rejected_abs": 0.16377928853034973, + "epoch": 0.07628128724672228, + "grad_norm": 204.0, + "learning_rate": 2.2970159972352866e-08, + "logits/chosen": -2.1415343284606934, + "logits/rejected": -2.2179665565490723, + "logps/chosen": -0.2673754096031189, + "logps/rejected": -0.2882601320743561, + "loss": 0.22219261527061462, + "loss/core": 0.19527706503868103, + "loss/preference_sft": 0.28320834040641785, + "loss/reference_anchor": 0.5099908113479614, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.9026601314544678, + "rewards/margins": 1.265976905822754, + "rewards/rejected": 1.636683464050293, + "ronpo/logit": 0.12659768760204315, + "ronpo/residual": 0.11025957018136978, + "ronpo/residual_abs": 0.18036124110221863, + "ronpo/target": 0.016338128596544266, + "ronpo/target_abs": 0.087813600897789, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 240 + }, + { + "drift/chosen_abs": 0.18804895877838135, + "drift/rejected_abs": 0.07305173575878143, + "epoch": 0.07787048073102901, + "grad_norm": 2.1875, + "learning_rate": 2.2835781343782966e-08, + "logits/chosen": -2.182067394256592, + "logits/rejected": -2.2002573013305664, + "logps/chosen": -0.29913002252578735, + "logps/rejected": -0.3051024079322815, + "loss": 0.2962071895599365, + "loss/core": 0.28634169697761536, + "loss/preference_sft": 0.3018152117729187, + "loss/reference_anchor": 0.1671280860900879, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.8793338537216187, + "rewards/margins": 1.174026370048523, + "rewards/rejected": 0.7053073644638062, + "ronpo/logit": 0.11740265041589737, + "ronpo/residual": 0.12060768902301788, + "ronpo/residual_abs": 0.1809166967868805, + "ronpo/target": -0.0032050427980720997, + "ronpo/target_abs": 0.09324783831834793, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 245 + }, + { + "drift/chosen_abs": 0.362274706363678, + "drift/rejected_abs": 0.08525294065475464, + "epoch": 0.07945967421533572, + "grad_norm": 7.25, + "learning_rate": 2.269751585055215e-08, + "logits/chosen": -2.1129937171936035, + "logits/rejected": -2.090263843536377, + "logps/chosen": -0.3094167709350586, + "logps/rejected": -0.31022873520851135, + "loss": 1.225620985031128, + "loss/core": 1.189970850944519, + "loss/preference_sft": 0.3118210434913635, + "loss/reference_anchor": 0.6818209886550903, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 3.6219310760498047, + "rewards/margins": 2.770771026611328, + "rewards/rejected": 0.851159930229187, + "ronpo/logit": 0.2770771086215973, + "ronpo/residual": 0.2786293029785156, + "ronpo/residual_abs": 0.33625951409339905, + "ronpo/target": -0.0015522129833698273, + "ronpo/target_abs": 0.08379620313644409, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 250 + }, + { + "drift/chosen_abs": 0.4456920623779297, + "drift/rejected_abs": 0.19766589999198914, + "epoch": 0.08104886769964244, + "grad_norm": 7.0625, + "learning_rate": 2.2555415488657776e-08, + "logits/chosen": -2.1046249866485596, + "logits/rejected": -2.107532262802124, + "logps/chosen": -0.2630499303340912, + "logps/rejected": -0.27011799812316895, + "loss": 0.7377499341964722, + "loss/core": 0.6888483166694641, + "loss/preference_sft": 0.27254000306129456, + "loss/reference_anchor": 0.9507792592048645, + "rewards/accuracies": 0.8125, + "rewards/chosen": 4.456816673278809, + "rewards/margins": 2.533341407775879, + "rewards/rejected": 1.9234750270843506, + "ronpo/logit": 0.2533341646194458, + "ronpo/residual": 0.25843900442123413, + "ronpo/residual_abs": 0.31028133630752563, + "ronpo/target": -0.005104861222207546, + "ronpo/target_abs": 0.08672384172677994, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 255 + }, + { + "drift/chosen_abs": 0.32755011320114136, + "drift/rejected_abs": 0.08800020813941956, + "epoch": 0.08263806118394915, + "grad_norm": 972.0, + "learning_rate": 2.240953369623447e-08, + "logits/chosen": -2.193181276321411, + "logits/rejected": -2.168120861053467, + "logps/chosen": -0.2840411067008972, + "logps/rejected": -0.28209537267684937, + "loss": 0.8339856266975403, + "loss/core": 0.8076542615890503, + "loss/preference_sft": 0.29532259702682495, + "loss/reference_anchor": 0.4970928728580475, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.2736563682556152, + "rewards/margins": 2.4421114921569824, + "rewards/rejected": 0.8315451741218567, + "ronpo/logit": 0.24421115219593048, + "ronpo/residual": 0.2369440346956253, + "ronpo/residual_abs": 0.29261597990989685, + "ronpo/target": 0.007267133332788944, + "ronpo/target_abs": 0.08349494636058807, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 260 + }, + { + "drift/chosen_abs": 0.18214836716651917, + "drift/rejected_abs": 0.0814029648900032, + "epoch": 0.08422725466825585, + "grad_norm": 16.0, + "learning_rate": 2.225992533345824e-08, + "logits/chosen": -2.155879497528076, + "logits/rejected": -2.1550204753875732, + "logps/chosen": -0.284726619720459, + "logps/rejected": -0.29086941480636597, + "loss": 0.08231814950704575, + "loss/core": 0.07635300606489182, + "loss/preference_sft": 0.2914847135543823, + "loss/reference_anchor": 0.0901542603969574, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8214836120605469, + "rewards/margins": 1.0148638486862183, + "rewards/rejected": 0.8066196441650391, + "ronpo/logit": 0.10148639976978302, + "ronpo/residual": 0.09669841825962067, + "ronpo/residual_abs": 0.16233278810977936, + "ronpo/target": 0.004787982441484928, + "ronpo/target_abs": 0.09343020617961884, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 265 + }, + { + "drift/chosen_abs": 0.2922074496746063, + "drift/rejected_abs": 0.16019122302532196, + "epoch": 0.08581644815256258, + "grad_norm": 2.734375, + "learning_rate": 2.210664666191579e-08, + "logits/chosen": -2.2221102714538574, + "logits/rejected": -2.1809840202331543, + "logps/chosen": -0.2800453305244446, + "logps/rejected": -0.28999465703964233, + "loss": 0.357001394033432, + "loss/core": 0.3204185366630554, + "loss/preference_sft": 0.28067368268966675, + "loss/reference_anchor": 0.7035894393920898, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.9200611114501953, + "rewards/margins": 1.3199583292007446, + "rewards/rejected": 1.6001027822494507, + "ronpo/logit": 0.13199582695960999, + "ronpo/residual": 0.11801192909479141, + "ronpo/residual_abs": 0.18810518085956573, + "ronpo/target": 0.013983900658786297, + "ronpo/target_abs": 0.08693692088127136, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 270 + }, + { + "drift/chosen_abs": 0.17948992550373077, + "drift/rejected_abs": 0.10400255769491196, + "epoch": 0.08740564163686929, + "grad_norm": 6.875, + "learning_rate": 2.1949755323446846e-08, + "logits/chosen": -2.275407314300537, + "logits/rejected": -2.3497166633605957, + "logps/chosen": -0.27847951650619507, + "logps/rejected": -0.2680722773075104, + "loss": 0.048314549028873444, + "loss/core": 0.041888050734996796, + "loss/preference_sft": 0.27859556674957275, + "loss/reference_anchor": 0.1006704568862915, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.794899582862854, + "rewards/margins": 0.7595475912094116, + "rewards/rejected": 1.0353518724441528, + "ronpo/logit": 0.07595475763082504, + "ronpo/residual": 0.05474218726158142, + "ronpo/residual_abs": 0.13210758566856384, + "ronpo/target": 0.02121257223188877, + "ronpo/target_abs": 0.0803799256682396, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 275 + }, + { + "drift/chosen_abs": 0.1700095236301422, + "drift/rejected_abs": 0.11016277223825455, + "epoch": 0.08899483512117601, + "grad_norm": 52.0, + "learning_rate": 2.1789310318467428e-08, + "logits/chosen": -2.180828094482422, + "logits/rejected": -2.182821750640869, + "logps/chosen": -0.2840616703033447, + "logps/rejected": -0.2911655306816101, + "loss": 0.10237391293048859, + "loss/core": 0.09611751139163971, + "loss/preference_sft": 0.28340545296669006, + "loss/reference_anchor": 0.09678752720355988, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.6990578174591064, + "rewards/margins": 0.5979472398757935, + "rewards/rejected": 1.1011104583740234, + "ronpo/logit": 0.059794723987579346, + "ronpo/residual": 0.052797090262174606, + "ronpo/residual_abs": 0.1553640067577362, + "ronpo/target": 0.006997637450695038, + "ronpo/target_abs": 0.07295690476894379, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 280 + }, + { + "drift/chosen_abs": 0.43757256865501404, + "drift/rejected_abs": 0.14037099480628967, + "epoch": 0.09058402860548272, + "grad_norm": 10.9375, + "learning_rate": 2.162537198378218e-08, + "logits/chosen": -2.2235183715820312, + "logits/rejected": -2.194262981414795, + "logps/chosen": -0.2805050313472748, + "logps/rejected": -0.2911986708641052, + "loss": 1.5213605165481567, + "loss/core": 1.4580514430999756, + "loss/preference_sft": 0.28713876008987427, + "loss/reference_anchor": 1.237465739250183, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 4.375100135803223, + "rewards/margins": 2.9718849658966064, + "rewards/rejected": 1.4032156467437744, + "ronpo/logit": 0.29718852043151855, + "ronpo/residual": 0.30155670642852783, + "ronpo/residual_abs": 0.34738093614578247, + "ronpo/target": -0.004368200898170471, + "ronpo/target_abs": 0.0792548879981041, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 285 + }, + { + "drift/chosen_abs": 0.2998009920120239, + "drift/rejected_abs": 0.13739660382270813, + "epoch": 0.09217322208978942, + "grad_norm": 133.0, + "learning_rate": 2.14580019698942e-08, + "logits/chosen": -2.0153133869171143, + "logits/rejected": -2.089466094970703, + "logps/chosen": -0.27183207869529724, + "logps/rejected": -0.2848559617996216, + "loss": 0.26796045899391174, + "loss/core": 0.25352850556373596, + "loss/preference_sft": 0.2748119533061981, + "loss/reference_anchor": 0.261157751083374, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.99642276763916, + "rewards/margins": 1.623225212097168, + "rewards/rejected": 1.3731976747512817, + "ronpo/logit": 0.162322536110878, + "ronpo/residual": 0.1448330283164978, + "ronpo/residual_abs": 0.23569996654987335, + "ronpo/target": 0.01748950406908989, + "ronpo/target_abs": 0.0773894339799881, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 290 + }, + { + "drift/chosen_abs": 0.19523122906684875, + "drift/rejected_abs": 0.07123854756355286, + "epoch": 0.09376241557409615, + "grad_norm": 178.0, + "learning_rate": 2.128726321782077e-08, + "logits/chosen": -2.1933460235595703, + "logits/rejected": -2.1863884925842285, + "logps/chosen": -0.30095577239990234, + "logps/rejected": -0.3060711622238159, + "loss": 0.24075086414813995, + "loss/core": 0.2300082892179489, + "loss/preference_sft": 0.30148637294769287, + "loss/reference_anchor": 0.18470288813114166, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.951626181602478, + "rewards/margins": 1.244182825088501, + "rewards/rejected": 0.7074434161186218, + "ronpo/logit": 0.12441827356815338, + "ronpo/residual": 0.1209135428071022, + "ronpo/residual_abs": 0.17452454566955566, + "ronpo/target": 0.003504746360704303, + "ronpo/target_abs": 0.07728487253189087, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 295 + }, + { + "drift/chosen_abs": 0.13675720989704132, + "drift/rejected_abs": 0.0812956839799881, + "epoch": 0.09535160905840286, + "grad_norm": 5.84375, + "learning_rate": 2.111321993542385e-08, + "logits/chosen": -2.407482862472534, + "logits/rejected": -2.4131016731262207, + "logps/chosen": -0.29553696513175964, + "logps/rejected": -0.28668391704559326, + "loss": 0.03661810979247093, + "loss/core": 0.03264369070529938, + "loss/preference_sft": 0.2879699468612671, + "loss/reference_anchor": 0.05069132521748543, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.3664600849151611, + "rewards/margins": 0.5552068948745728, + "rewards/rejected": 0.8112531900405884, + "ronpo/logit": 0.055520690977573395, + "ronpo/residual": 0.053940463811159134, + "ronpo/residual_abs": 0.12251658737659454, + "ronpo/target": 0.0015802305424585938, + "ronpo/target_abs": 0.09203118085861206, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 300 + }, + { + "drift/chosen_abs": 0.2224307805299759, + "drift/rejected_abs": 0.10239239037036896, + "epoch": 0.09694080254270958, + "grad_norm": 3.28125, + "learning_rate": 2.0935937573264096e-08, + "logits/chosen": -2.0765604972839355, + "logits/rejected": -2.104783296585083, + "logps/chosen": -0.30479732155799866, + "logps/rejected": -0.29314523935317993, + "loss": 0.1927420198917389, + "loss/core": 0.17845560610294342, + "loss/preference_sft": 0.3036041855812073, + "loss/reference_anchor": 0.25536784529685974, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.221867322921753, + "rewards/margins": 1.2011425495147705, + "rewards/rejected": 1.0207250118255615, + "ronpo/logit": 0.12011425197124481, + "ronpo/residual": 0.12593664228916168, + "ronpo/residual_abs": 0.18047745525836945, + "ronpo/target": -0.005822394508868456, + "ronpo/target_abs": 0.0825614184141159, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 305 + }, + { + "drift/chosen_abs": 0.2303655445575714, + "drift/rejected_abs": 0.14673031866550446, + "epoch": 0.09852999602701629, + "grad_norm": 278.0, + "learning_rate": 2.0755482799987594e-08, + "logits/chosen": -2.0974299907684326, + "logits/rejected": -1.9897010326385498, + "logps/chosen": -0.30457696318626404, + "logps/rejected": -0.3037092089653015, + "loss": 0.09838415682315826, + "loss/core": 0.08123552054166794, + "loss/preference_sft": 0.31207767128944397, + "loss/reference_anchor": 0.3117648661136627, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.3016207218170166, + "rewards/margins": 0.8351632952690125, + "rewards/rejected": 1.4664572477340698, + "ronpo/logit": 0.08351632952690125, + "ronpo/residual": 0.08266756683588028, + "ronpo/residual_abs": 0.16260148584842682, + "ronpo/target": 0.0008487681043334305, + "ronpo/target_abs": 0.08333858102560043, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 310 + }, + { + "drift/chosen_abs": 0.21899929642677307, + "drift/rejected_abs": 0.17108279466629028, + "epoch": 0.10011918951132301, + "grad_norm": 1.8046875, + "learning_rate": 2.0571923477254526e-08, + "logits/chosen": -2.210271120071411, + "logits/rejected": -2.2432236671447754, + "logps/chosen": -0.28812751173973083, + "logps/rejected": -0.2839018404483795, + "loss": 0.12934687733650208, + "loss/core": 0.08804286271333694, + "loss/preference_sft": 0.28392842411994934, + "loss/reference_anchor": 0.7976871728897095, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.1899399757385254, + "rewards/margins": 0.48040738701820374, + "rewards/rejected": 1.7095324993133545, + "ronpo/logit": 0.04804074019193649, + "ronpo/residual": 0.03702558949589729, + "ronpo/residual_abs": 0.14168190956115723, + "ronpo/target": 0.011015147902071476, + "ronpo/target_abs": 0.08301273733377457, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 315 + }, + { + "drift/chosen_abs": 0.30637550354003906, + "drift/rejected_abs": 0.15101422369480133, + "epoch": 0.10170838299562972, + "grad_norm": 7.90625, + "learning_rate": 2.038532863421914e-08, + "logits/chosen": -2.0474765300750732, + "logits/rejected": -2.046475648880005, + "logps/chosen": -0.2672887146472931, + "logps/rejected": -0.2681429982185364, + "loss": 0.43184512853622437, + "loss/core": 0.39570650458335876, + "loss/preference_sft": 0.26500943303108215, + "loss/reference_anchor": 0.6962720155715942, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.0628886222839355, + "rewards/margins": 1.5545833110809326, + "rewards/rejected": 1.5083054304122925, + "ronpo/logit": 0.15545836091041565, + "ronpo/residual": 0.15610265731811523, + "ronpo/residual_abs": 0.2172074019908905, + "ronpo/target": -0.000644296407699585, + "ronpo/target_abs": 0.08082158118486404, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 320 + }, + { + "drift/chosen_abs": 0.2718430161476135, + "drift/rejected_abs": 0.06493407487869263, + "epoch": 0.10329757647993643, + "grad_norm": 23.875, + "learning_rate": 2.0195768441570726e-08, + "logits/chosen": -2.213127374649048, + "logits/rejected": -2.2380900382995605, + "logps/chosen": -0.3036438524723053, + "logps/rejected": -0.2892650365829468, + "loss": 1.196415901184082, + "loss/core": 1.1635918617248535, + "loss/preference_sft": 0.2940658628940582, + "loss/reference_anchor": 0.6270740032196045, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.717806100845337, + "rewards/margins": 2.0749688148498535, + "rewards/rejected": 0.642837643623352, + "ronpo/logit": 0.20749685168266296, + "ronpo/residual": 0.22607414424419403, + "ronpo/residual_abs": 0.2697368264198303, + "ronpo/target": -0.018577273935079575, + "ronpo/target_abs": 0.09064265340566635, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 325 + }, + { + "drift/chosen_abs": 0.2503616511821747, + "drift/rejected_abs": 0.12431076914072037, + "epoch": 0.10488676996424315, + "grad_norm": 10.875, + "learning_rate": 2.0003314185145307e-08, + "logits/chosen": -2.3221628665924072, + "logits/rejected": -2.322690010070801, + "logps/chosen": -0.26718512177467346, + "logps/rejected": -0.26268628239631653, + "loss": 0.566504180431366, + "loss/core": 0.5484728813171387, + "loss/preference_sft": 0.255176305770874, + "loss/reference_anchor": 0.3351089656352997, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.5033228397369385, + "rewards/margins": 1.2658904790878296, + "rewards/rejected": 1.2374324798583984, + "ronpo/logit": 0.12658903002738953, + "ronpo/residual": 0.1411885768175125, + "ronpo/residual_abs": 0.2634466588497162, + "ronpo/target": -0.014599531888961792, + "ronpo/target_abs": 0.09172377735376358, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 330 + }, + { + "drift/chosen_abs": 0.445903480052948, + "drift/rejected_abs": 0.21055002510547638, + "epoch": 0.10647596344854986, + "grad_norm": 12.875, + "learning_rate": 1.9808038239117914e-08, + "logits/chosen": -1.991874098777771, + "logits/rejected": -1.9406774044036865, + "logps/chosen": -0.2943147122859955, + "logps/rejected": -0.294437974691391, + "loss": 0.7760679721832275, + "loss/core": 0.7043066024780273, + "loss/preference_sft": 0.2942696809768677, + "loss/reference_anchor": 1.4057997465133667, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.458679676055908, + "rewards/margins": 2.356698513031006, + "rewards/rejected": 2.1019809246063232, + "ronpo/logit": 0.23566989600658417, + "ronpo/residual": 0.2460823804140091, + "ronpo/residual_abs": 0.30592185258865356, + "ronpo/target": -0.010412474162876606, + "ronpo/target_abs": 0.07370901107788086, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 335 + }, + { + "drift/chosen_abs": 0.45900607109069824, + "drift/rejected_abs": 0.16360488533973694, + "epoch": 0.10806515693285658, + "grad_norm": 82.0, + "learning_rate": 1.9610014038785648e-08, + "logits/chosen": -1.8805198669433594, + "logits/rejected": -1.9393765926361084, + "logps/chosen": -0.28708821535110474, + "logps/rejected": -0.2747197151184082, + "loss": 1.0118316411972046, + "loss/core": 0.9642781019210815, + "loss/preference_sft": 0.27820974588394165, + "loss/reference_anchor": 0.9232500791549683, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.589595794677734, + "rewards/margins": 2.960404396057129, + "rewards/rejected": 1.6291908025741577, + "ronpo/logit": 0.29604044556617737, + "ronpo/residual": 0.3044920861721039, + "ronpo/residual_abs": 0.3554214537143707, + "ronpo/target": -0.008451665751636028, + "ronpo/target_abs": 0.08010601997375488, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 340 + }, + { + "drift/chosen_abs": 0.24650216102600098, + "drift/rejected_abs": 0.1208873763680458, + "epoch": 0.10965435041716329, + "grad_norm": 58.5, + "learning_rate": 1.9409316052951656e-08, + "logits/chosen": -2.178605556488037, + "logits/rejected": -2.2515740394592285, + "logps/chosen": -0.2723388671875, + "logps/rejected": -0.2781441807746887, + "loss": 0.12823966145515442, + "loss/core": 0.11076222360134125, + "loss/preference_sft": 0.28036752343177795, + "loss/reference_anchor": 0.32151204347610474, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.4640796184539795, + "rewards/margins": 1.2876027822494507, + "rewards/rejected": 1.1764768362045288, + "ronpo/logit": 0.12876029312610626, + "ronpo/residual": 0.12763699889183044, + "ronpo/residual_abs": 0.17462731897830963, + "ronpo/target": 0.0011232979595661163, + "ronpo/target_abs": 0.07203267514705658, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 345 + }, + { + "drift/chosen_abs": 0.4136660099029541, + "drift/rejected_abs": 0.16482646763324738, + "epoch": 0.11124354390147001, + "grad_norm": 1.8359375, + "learning_rate": 1.920601975592047e-08, + "logits/chosen": -2.267277240753174, + "logits/rejected": -2.2952818870544434, + "logps/chosen": -0.23398800194263458, + "logps/rejected": -0.2486116588115692, + "loss": 1.0056898593902588, + "loss/core": 0.9502155184745789, + "loss/preference_sft": 0.24102886021137238, + "loss/reference_anchor": 1.0853848457336426, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.136660099029541, + "rewards/margins": 2.506661891937256, + "rewards/rejected": 1.629997968673706, + "ronpo/logit": 0.25066617131233215, + "ronpo/residual": 0.24287991225719452, + "ronpo/residual_abs": 0.3027934730052948, + "ronpo/target": 0.007786286063492298, + "ronpo/target_abs": 0.08268611133098602, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 350 + }, + { + "drift/chosen_abs": 0.28172412514686584, + "drift/rejected_abs": 0.10272540897130966, + "epoch": 0.11283273738577672, + "grad_norm": 14.875, + "learning_rate": 1.900020159911519e-08, + "logits/chosen": -2.1245241165161133, + "logits/rejected": -2.0237369537353516, + "logps/chosen": -0.2827402949333191, + "logps/rejected": -0.2766692042350769, + "loss": 0.41723641753196716, + "loss/core": 0.4000471234321594, + "loss/preference_sft": 0.2682899832725525, + "loss/reference_anchor": 0.31695735454559326, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.8172414302825928, + "rewards/margins": 1.7916021347045898, + "rewards/rejected": 1.025639533996582, + "ronpo/logit": 0.1791602075099945, + "ronpo/residual": 0.16192427277565002, + "ronpo/residual_abs": 0.23042872548103333, + "ronpo/target": 0.01723591983318329, + "ronpo/target_abs": 0.08204632252454758, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 355 + }, + { + "drift/chosen_abs": 0.21779580414295197, + "drift/rejected_abs": 0.08848261833190918, + "epoch": 0.11442193087008343, + "grad_norm": 11.5, + "learning_rate": 1.8791938982327247e-08, + "logits/chosen": -2.206521511077881, + "logits/rejected": -2.2727904319763184, + "logps/chosen": -0.2671610713005066, + "logps/rejected": -0.2585466206073761, + "loss": 0.09771557152271271, + "loss/core": 0.09170512855052948, + "loss/preference_sft": 0.2634662985801697, + "loss/reference_anchor": 0.09386207908391953, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.1769795417785645, + "rewards/margins": 1.2925890684127808, + "rewards/rejected": 0.8843904733657837, + "ronpo/logit": 0.1292589008808136, + "ronpo/residual": 0.1271723061800003, + "ronpo/residual_abs": 0.18293924629688263, + "ronpo/target": 0.002086599823087454, + "ronpo/target_abs": 0.07021833956241608, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 360 + }, + { + "drift/chosen_abs": 0.20787136256694794, + "drift/rejected_abs": 0.13100872933864594, + "epoch": 0.11601112435439015, + "grad_norm": 2.125, + "learning_rate": 1.85813102246095e-08, + "logits/chosen": -2.0922367572784424, + "logits/rejected": -2.1322176456451416, + "logps/chosen": -0.2858452796936035, + "logps/rejected": -0.3246702551841736, + "loss": 0.06760017573833466, + "loss/core": 0.047362055629491806, + "loss/preference_sft": 0.28588929772377014, + "loss/reference_anchor": 0.37617334723472595, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.0768673419952393, + "rewards/margins": 0.791536271572113, + "rewards/rejected": 1.2853310108184814, + "ronpo/logit": 0.0791536197066307, + "ronpo/residual": 0.07149665057659149, + "ronpo/residual_abs": 0.13622984290122986, + "ronpo/target": 0.007656970527023077, + "ronpo/target_abs": 0.09360074996948242, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 365 + }, + { + "drift/chosen_abs": 0.31859681010246277, + "drift/rejected_abs": 0.21548397839069366, + "epoch": 0.11760031783869686, + "grad_norm": 20.375, + "learning_rate": 1.8368394534823633e-08, + "logits/chosen": -2.136131763458252, + "logits/rejected": -2.168311595916748, + "logps/chosen": -0.27841588854789734, + "logps/rejected": -0.28144991397857666, + "loss": 0.16637016832828522, + "loss/core": 0.10749348253011703, + "loss/preference_sft": 0.2804015874862671, + "loss/reference_anchor": 1.1494934558868408, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.1855576038360596, + "rewards/margins": 1.0312860012054443, + "rewards/rejected": 2.1542716026306152, + "ronpo/logit": 0.10312861204147339, + "ronpo/residual": 0.0984123945236206, + "ronpo/residual_abs": 0.17065349221229553, + "ronpo/target": 0.004716214723885059, + "ronpo/target_abs": 0.08422146737575531, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 370 + }, + { + "drift/chosen_abs": 0.21396997570991516, + "drift/rejected_abs": 0.09890774637460709, + "epoch": 0.11918951132300358, + "grad_norm": 19.5, + "learning_rate": 1.815327198185297e-08, + "logits/chosen": -2.278458833694458, + "logits/rejected": -2.3031888008117676, + "logps/chosen": -0.29324546456336975, + "logps/rejected": -0.28806382417678833, + "loss": 0.13843554258346558, + "loss/core": 0.12961502373218536, + "loss/preference_sft": 0.28800949454307556, + "loss/reference_anchor": 0.14760896563529968, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.1395039558410645, + "rewards/margins": 1.1516611576080322, + "rewards/rejected": 0.9878427386283875, + "ronpo/logit": 0.11516612768173218, + "ronpo/residual": 0.10511883348226547, + "ronpo/residual_abs": 0.1740320473909378, + "ronpo/target": 0.01004729326814413, + "ronpo/target_abs": 0.08315954357385635, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 375 + }, + { + "drift/chosen_abs": 0.2743528187274933, + "drift/rejected_abs": 0.09948831051588058, + "epoch": 0.12077870480731029, + "grad_norm": 0.81640625, + "learning_rate": 1.793602346449181e-08, + "logits/chosen": -2.158848524093628, + "logits/rejected": -2.1761603355407715, + "logps/chosen": -0.2898809313774109, + "logps/rejected": -0.28347843885421753, + "loss": 0.5179681181907654, + "loss/core": 0.49899888038635254, + "loss/preference_sft": 0.28954416513442993, + "loss/reference_anchor": 0.35042867064476013, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.7419450283050537, + "rewards/margins": 1.7553457021713257, + "rewards/rejected": 0.9865990877151489, + "ronpo/logit": 0.17553459107875824, + "ronpo/residual": 0.17026254534721375, + "ronpo/residual_abs": 0.2293941080570221, + "ronpo/target": 0.0052720606327056885, + "ronpo/target_abs": 0.0783097967505455, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 380 + }, + { + "drift/chosen_abs": 0.2517983317375183, + "drift/rejected_abs": 0.14854536950588226, + "epoch": 0.12236789829161701, + "grad_norm": 63.5, + "learning_rate": 1.7716730681022724e-08, + "logits/chosen": -2.2751753330230713, + "logits/rejected": -2.2461843490600586, + "logps/chosen": -0.2717970013618469, + "logps/rejected": -0.2640511393547058, + "loss": 0.1832820177078247, + "loss/core": 0.17082270979881287, + "loss/preference_sft": 0.26608261466026306, + "loss/reference_anchor": 0.22257855534553528, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.516340494155884, + "rewards/margins": 1.0328803062438965, + "rewards/rejected": 1.4834601879119873, + "ronpo/logit": 0.10328805446624756, + "ronpo/residual": 0.0827520564198494, + "ronpo/residual_abs": 0.15645729005336761, + "ronpo/target": 0.020535994321107864, + "ronpo/target_abs": 0.08894232660531998, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 385 + }, + { + "drift/chosen_abs": 0.21122761070728302, + "drift/rejected_abs": 0.12187887728214264, + "epoch": 0.12395709177592372, + "grad_norm": 2.46875, + "learning_rate": 1.749547609849315e-08, + "logits/chosen": -2.0779099464416504, + "logits/rejected": -2.118682384490967, + "logps/chosen": -0.3044072091579437, + "logps/rejected": -0.2954100966453552, + "loss": 0.09059147536754608, + "loss/core": 0.07689197361469269, + "loss/preference_sft": 0.3024619221687317, + "loss/reference_anchor": 0.24374382197856903, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.1094882488250732, + "rewards/margins": 0.8933243751525879, + "rewards/rejected": 1.2161636352539062, + "ronpo/logit": 0.0893324464559555, + "ronpo/residual": 0.0991659015417099, + "ronpo/residual_abs": 0.1529773473739624, + "ronpo/target": -0.00983346439898014, + "ronpo/target_abs": 0.09204504638910294, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 390 + }, + { + "drift/chosen_abs": 0.21260306239128113, + "drift/rejected_abs": 0.0686182901263237, + "epoch": 0.12554628526023043, + "grad_norm": 2.84375, + "learning_rate": 1.7272342921702935e-08, + "logits/chosen": -2.158278465270996, + "logits/rejected": -2.2224063873291016, + "logps/chosen": -0.31105631589889526, + "logps/rejected": -0.2972649335861206, + "loss": 0.35496413707733154, + "loss/core": 0.34252816438674927, + "loss/preference_sft": 0.3063923716545105, + "loss/reference_anchor": 0.2180800884962082, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.1244759559631348, + "rewards/margins": 1.4389787912368774, + "rewards/rejected": 0.6854972839355469, + "ronpo/logit": 0.1438978910446167, + "ronpo/residual": 0.12823660671710968, + "ronpo/residual_abs": 0.1940845549106598, + "ronpo/target": 0.015661273151636124, + "ronpo/target_abs": 0.0806753933429718, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 395 + }, + { + "drift/chosen_abs": 0.19167861342430115, + "drift/rejected_abs": 0.08439207822084427, + "epoch": 0.12713547874453715, + "grad_norm": 8.3125, + "learning_rate": 1.7047415061914393e-08, + "logits/chosen": -2.2295565605163574, + "logits/rejected": -2.3195853233337402, + "logps/chosen": -0.25695428252220154, + "logps/rejected": -0.2609720826148987, + "loss": 0.07849901169538498, + "loss/core": 0.07116608321666718, + "loss/preference_sft": 0.2507200539112091, + "loss/reference_anchor": 0.12158646434545517, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 1.9159801006317139, + "rewards/margins": 1.0806057453155518, + "rewards/rejected": 0.8353743553161621, + "ronpo/logit": 0.1080605760216713, + "ronpo/residual": 0.11492305994033813, + "ronpo/residual_abs": 0.1632971316576004, + "ronpo/target": -0.0068624913692474365, + "ronpo/target_abs": 0.08553292602300644, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 400 + }, + { + "drift/chosen_abs": 0.17035509645938873, + "drift/rejected_abs": 0.14627881348133087, + "epoch": 0.12872467222884387, + "grad_norm": 2.34375, + "learning_rate": 1.6820777105296708e-08, + "logits/chosen": -2.2909557819366455, + "logits/rejected": -2.2994208335876465, + "logps/chosen": -0.29540935158729553, + "logps/rejected": -0.2884915769100189, + "loss": 0.29835981130599976, + "loss/core": 0.2854141592979431, + "loss/preference_sft": 0.2857155203819275, + "loss/reference_anchor": 0.230341836810112, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.7012271881103516, + "rewards/margins": 0.239228293299675, + "rewards/rejected": 1.4619989395141602, + "ronpo/logit": 0.023922838270664215, + "ronpo/residual": 0.02246231772005558, + "ronpo/residual_abs": 0.18744370341300964, + "ronpo/target": 0.001460521249100566, + "ronpo/target_abs": 0.08477913588285446, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 405 + }, + { + "drift/chosen_abs": 0.18731415271759033, + "drift/rejected_abs": 0.05840852111577988, + "epoch": 0.13031386571315057, + "grad_norm": 0.6875, + "learning_rate": 1.6592514281116553e-08, + "logits/chosen": -2.229135751724243, + "logits/rejected": -2.1434459686279297, + "logps/chosen": -0.28917819261550903, + "logps/rejected": -0.2873764634132385, + "loss": 0.35508978366851807, + "loss/core": 0.344736784696579, + "loss/preference_sft": 0.28428682684898376, + "loss/reference_anchor": 0.17863091826438904, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.8712348937988281, + "rewards/margins": 1.3082423210144043, + "rewards/rejected": 0.5629926919937134, + "ronpo/logit": 0.1308242380619049, + "ronpo/residual": 0.14936430752277374, + "ronpo/residual_abs": 0.17734959721565247, + "ronpo/target": -0.01854007877409458, + "ronpo/target_abs": 0.06559576839208603, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 410 + }, + { + "drift/chosen_abs": 0.12775962054729462, + "drift/rejected_abs": 0.05946607142686844, + "epoch": 0.1319030591974573, + "grad_norm": 18.5, + "learning_rate": 1.6362712429686844e-08, + "logits/chosen": -2.2688026428222656, + "logits/rejected": -2.2994446754455566, + "logps/chosen": -0.2852744460105896, + "logps/rejected": -0.2741158604621887, + "loss": 0.04457860440015793, + "loss/core": 0.04096058011054993, + "loss/preference_sft": 0.27498331665992737, + "loss/reference_anchor": 0.04486214369535446, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.2775468826293945, + "rewards/margins": 0.6865289807319641, + "rewards/rejected": 0.5910179018974304, + "ronpo/logit": 0.06865289807319641, + "ronpo/residual": 0.061678577214479446, + "ronpo/residual_abs": 0.13630971312522888, + "ronpo/target": 0.006974323187023401, + "ronpo/target_abs": 0.08561506867408752, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 415 + }, + { + "drift/chosen_abs": 0.3900828957557678, + "drift/rejected_abs": 0.12692275643348694, + "epoch": 0.133492252681764, + "grad_norm": 2.71875, + "learning_rate": 1.6131457970085686e-08, + "logits/chosen": -2.1172633171081543, + "logits/rejected": -2.235016345977783, + "logps/chosen": -0.2711094319820404, + "logps/rejected": -0.2714807391166687, + "loss": 0.8514937162399292, + "loss/core": 0.8164560198783875, + "loss/preference_sft": 0.2708297073841095, + "loss/reference_anchor": 0.6736721992492676, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.9008281230926514, + "rewards/margins": 2.7281997203826904, + "rewards/rejected": 1.17262864112854, + "ronpo/logit": 0.27282002568244934, + "ronpo/residual": 0.25294801592826843, + "ronpo/residual_abs": 0.3054654002189636, + "ronpo/target": 0.019871985539793968, + "ronpo/target_abs": 0.07743876427412033, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 420 + }, + { + "drift/chosen_abs": 0.4248512387275696, + "drift/rejected_abs": 0.13988152146339417, + "epoch": 0.1350814461660707, + "grad_norm": 310.0, + "learning_rate": 1.5898837867657726e-08, + "logits/chosen": -2.166339159011841, + "logits/rejected": -2.1603424549102783, + "logps/chosen": -0.2978591322898865, + "logps/rejected": -0.2953459620475769, + "loss": 1.2260562181472778, + "loss/core": 1.1734199523925781, + "loss/preference_sft": 0.29464253783226013, + "loss/reference_anchor": 1.0232583284378052, + "rewards/accuracies": 0.875, + "rewards/chosen": 4.247471332550049, + "rewards/margins": 2.849874258041382, + "rewards/rejected": 1.3975974321365356, + "ronpo/logit": 0.2849874496459961, + "ronpo/residual": 0.2766841650009155, + "ronpo/residual_abs": 0.3479006886482239, + "ronpo/target": 0.008303266949951649, + "ronpo/target_abs": 0.09610433876514435, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 425 + }, + { + "drift/chosen_abs": 0.18739613890647888, + "drift/rejected_abs": 0.09084701538085938, + "epoch": 0.13667063965037743, + "grad_norm": 10.5, + "learning_rate": 1.5664939601310025e-08, + "logits/chosen": -2.4331743717193604, + "logits/rejected": -2.4204342365264893, + "logps/chosen": -0.2795220911502838, + "logps/rejected": -0.2753976583480835, + "loss": 0.10062910616397858, + "loss/core": 0.09226439893245697, + "loss/preference_sft": 0.2744489312171936, + "loss/reference_anchor": 0.13984903693199158, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.8734142780303955, + "rewards/margins": 0.9657185673713684, + "rewards/rejected": 0.9076957702636719, + "ronpo/logit": 0.09657186269760132, + "ronpo/residual": 0.07882784307003021, + "ronpo/residual_abs": 0.14449390769004822, + "ronpo/target": 0.017744004726409912, + "ronpo/target_abs": 0.07996980845928192, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 430 + }, + { + "drift/chosen_abs": 0.17314964532852173, + "drift/rejected_abs": 0.09341616928577423, + "epoch": 0.13825983313468415, + "grad_norm": 12.5, + "learning_rate": 1.542985113061481e-08, + "logits/chosen": -2.1819088459014893, + "logits/rejected": -2.1771626472473145, + "logps/chosen": -0.2796979546546936, + "logps/rejected": -0.2803123891353607, + "loss": 0.04070793837308884, + "loss/core": 0.03540237620472908, + "loss/preference_sft": 0.2738858759403229, + "loss/reference_anchor": 0.07872271537780762, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.730942964553833, + "rewards/margins": 0.7972149848937988, + "rewards/rejected": 0.9337277412414551, + "ronpo/logit": 0.07972150295972824, + "ronpo/residual": 0.054669272154569626, + "ronpo/residual_abs": 0.12845924496650696, + "ronpo/target": 0.025052234530448914, + "ronpo/target_abs": 0.07804572582244873, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 435 + }, + { + "drift/chosen_abs": 0.318293035030365, + "drift/rejected_abs": 0.13249650597572327, + "epoch": 0.13984902661899087, + "grad_norm": 4.15625, + "learning_rate": 1.51936608627315e-08, + "logits/chosen": -2.273724317550659, + "logits/rejected": -2.299027919769287, + "logps/chosen": -0.2951945960521698, + "logps/rejected": -0.29159849882125854, + "loss": 0.7339528203010559, + "loss/core": 0.6881879568099976, + "loss/preference_sft": 0.2977384924888611, + "loss/reference_anchor": 0.885523796081543, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.182380437850952, + "rewards/margins": 1.8633613586425781, + "rewards/rejected": 1.319018840789795, + "ronpo/logit": 0.18633611500263214, + "ronpo/residual": 0.19329491257667542, + "ronpo/residual_abs": 0.24592044949531555, + "ronpo/target": -0.006958787329494953, + "ronpo/target_abs": 0.0852695107460022, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 440 + }, + { + "drift/chosen_abs": 0.18986327946186066, + "drift/rejected_abs": 0.16925686597824097, + "epoch": 0.14143822010329757, + "grad_norm": 260.0, + "learning_rate": 1.4956457619160377e-08, + "logits/chosen": -2.2318367958068848, + "logits/rejected": -2.154519557952881, + "logps/chosen": -0.28089621663093567, + "logps/rejected": -0.2713889479637146, + "loss": 0.25093281269073486, + "loss/core": 0.2370346039533615, + "loss/preference_sft": 0.27891331911087036, + "loss/reference_anchor": 0.25007274746894836, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.8986326456069946, + "rewards/margins": 0.20648033916950226, + "rewards/rejected": 1.6921522617340088, + "ronpo/logit": 0.020648030564188957, + "ronpo/residual": 0.02750701829791069, + "ronpo/residual_abs": 0.20267800986766815, + "ronpo/target": -0.006858985871076584, + "ronpo/target_abs": 0.08476842194795609, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 445 + }, + { + "drift/chosen_abs": 0.3151724934577942, + "drift/rejected_abs": 0.1027737632393837, + "epoch": 0.1430274135876043, + "grad_norm": 41.0, + "learning_rate": 1.4718330602340439e-08, + "logits/chosen": -2.264042615890503, + "logits/rejected": -2.327329397201538, + "logps/chosen": -0.2729445695877075, + "logps/rejected": -0.28299084305763245, + "loss": 0.7172930836677551, + "loss/core": 0.6890506148338318, + "loss/preference_sft": 0.2784006595611572, + "loss/reference_anchor": 0.5370103120803833, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.150481939315796, + "rewards/margins": 2.1262738704681396, + "rewards/rejected": 1.0242083072662354, + "ronpo/logit": 0.2126273810863495, + "ronpo/residual": 0.21831850707530975, + "ronpo/residual_abs": 0.26258429884910583, + "ronpo/target": -0.005691120866686106, + "ronpo/target_abs": 0.0731092318892479, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 450 + }, + { + "drift/chosen_abs": 0.20744338631629944, + "drift/rejected_abs": 0.09908546507358551, + "epoch": 0.144616607071911, + "grad_norm": 10.25, + "learning_rate": 1.4479369362104036e-08, + "logits/chosen": -2.3172354698181152, + "logits/rejected": -2.3511035442352295, + "logps/chosen": -0.284721314907074, + "logps/rejected": -0.28263527154922485, + "loss": 0.10904005914926529, + "loss/core": 0.10059742629528046, + "loss/preference_sft": 0.2903629243373871, + "loss/reference_anchor": 0.13981646299362183, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.0744338035583496, + "rewards/margins": 1.083627462387085, + "rewards/rejected": 0.9908064603805542, + "ronpo/logit": 0.10836274921894073, + "ronpo/residual": 0.107516810297966, + "ronpo/residual_abs": 0.15838299691677094, + "ronpo/target": 0.0008459277451038361, + "ronpo/target_abs": 0.0736476257443428, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 455 + }, + { + "drift/chosen_abs": 0.37808623909950256, + "drift/rejected_abs": 0.1648140251636505, + "epoch": 0.1462058005562177, + "grad_norm": 121.5, + "learning_rate": 1.4239663762000817e-08, + "logits/chosen": -1.8078231811523438, + "logits/rejected": -1.6678974628448486, + "logps/chosen": -0.2803362011909485, + "logps/rejected": -0.2994075417518616, + "loss": 0.551475465297699, + "loss/core": 0.5203845500946045, + "loss/preference_sft": 0.28328341245651245, + "loss/reference_anchor": 0.5934896469116211, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.7770161628723145, + "rewards/margins": 2.1362390518188477, + "rewards/rejected": 1.6407768726348877, + "ronpo/logit": 0.21362391114234924, + "ronpo/residual": 0.21224097907543182, + "ronpo/residual_abs": 0.2707666754722595, + "ronpo/target": 0.0013829537201672792, + "ronpo/target_abs": 0.08258362114429474, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 460 + }, + { + "drift/chosen_abs": 0.34338995814323425, + "drift/rejected_abs": 0.23440727591514587, + "epoch": 0.14779499404052443, + "grad_norm": 400.0, + "learning_rate": 1.3999303945503746e-08, + "logits/chosen": -2.0976834297180176, + "logits/rejected": -2.116819381713867, + "logps/chosen": -0.2706809341907501, + "logps/rejected": -0.27307775616645813, + "loss": 0.8269802331924438, + "loss/core": 0.7959731221199036, + "loss/preference_sft": 0.2685784101486206, + "loss/reference_anchor": 0.5932824611663818, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.4336636066436768, + "rewards/margins": 1.0909364223480225, + "rewards/rejected": 2.3427271842956543, + "ronpo/logit": 0.10909364372491837, + "ronpo/residual": 0.10874764621257782, + "ronpo/residual_abs": 0.3142366111278534, + "ronpo/target": 0.00034598278580233455, + "ronpo/target_abs": 0.07308773696422577, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 465 + }, + { + "drift/chosen_abs": 0.29909831285476685, + "drift/rejected_abs": 0.11643582582473755, + "epoch": 0.14938418752483115, + "grad_norm": 1.9375, + "learning_rate": 1.3758380302109807e-08, + "logits/chosen": -2.075563430786133, + "logits/rejected": -2.044642210006714, + "logps/chosen": -0.254794180393219, + "logps/rejected": -0.2602981626987457, + "loss": 0.7801807522773743, + "loss/core": 0.7529562711715698, + "loss/preference_sft": 0.26095765829086304, + "loss/reference_anchor": 0.5183919668197632, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.989849090576172, + "rewards/margins": 1.8269249200820923, + "rewards/rejected": 1.1629244089126587, + "ronpo/logit": 0.1826924830675125, + "ronpo/residual": 0.19307713210582733, + "ronpo/residual_abs": 0.2564256191253662, + "ronpo/target": -0.010384641587734222, + "ronpo/target_abs": 0.08509234338998795, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 470 + }, + { + "drift/chosen_abs": 0.2505967915058136, + "drift/rejected_abs": 0.09651152044534683, + "epoch": 0.15097338100913787, + "grad_norm": 9.1875, + "learning_rate": 1.3516983433348227e-08, + "logits/chosen": -2.19514536857605, + "logits/rejected": -2.130986213684082, + "logps/chosen": -0.3052103817462921, + "logps/rejected": -0.29925405979156494, + "loss": 0.22467193007469177, + "loss/core": 0.21474584937095642, + "loss/preference_sft": 0.3062250018119812, + "loss/reference_anchor": 0.16789942979812622, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.505239486694336, + "rewards/margins": 1.5401643514633179, + "rewards/rejected": 0.9650753140449524, + "ronpo/logit": 0.15401646494865417, + "ronpo/residual": 0.15226852893829346, + "ronpo/residual_abs": 0.21335072815418243, + "ronpo/target": 0.0017479099333286285, + "ronpo/target_abs": 0.07254009693861008, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 475 + }, + { + "drift/chosen_abs": 0.22144432365894318, + "drift/rejected_abs": 0.10890810191631317, + "epoch": 0.15256257449344457, + "grad_norm": 1.0859375, + "learning_rate": 1.3275204118708941e-08, + "logits/chosen": -2.119616746902466, + "logits/rejected": -2.1420140266418457, + "logps/chosen": -0.273566871881485, + "logps/rejected": -0.2686625123023987, + "loss": 0.08422975242137909, + "loss/core": 0.07651440799236298, + "loss/preference_sft": 0.2724953591823578, + "loss/reference_anchor": 0.12705738842487335, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.21227765083313, + "rewards/margins": 1.1317625045776367, + "rewards/rejected": 1.0805152654647827, + "ronpo/logit": 0.11317624896764755, + "ronpo/residual": 0.1295435130596161, + "ronpo/residual_abs": 0.1753176897764206, + "ronpo/target": -0.016367264091968536, + "ronpo/target_abs": 0.08774258196353912, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 480 + }, + { + "drift/chosen_abs": 0.2984105944633484, + "drift/rejected_abs": 0.1611432582139969, + "epoch": 0.1541517679777513, + "grad_norm": 5.5, + "learning_rate": 1.3033133281504133e-08, + "logits/chosen": -1.902937889099121, + "logits/rejected": -2.0238137245178223, + "logps/chosen": -0.28192150592803955, + "logps/rejected": -0.27930665016174316, + "loss": 0.13234123587608337, + "loss/core": 0.11475740373134613, + "loss/preference_sft": 0.2823740839958191, + "loss/reference_anchor": 0.3234391510486603, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.983555793762207, + "rewards/margins": 1.3728954792022705, + "rewards/rejected": 1.6106599569320679, + "ronpo/logit": 0.13728956878185272, + "ronpo/residual": 0.1287764012813568, + "ronpo/residual_abs": 0.18728485703468323, + "ronpo/target": 0.008513162843883038, + "ronpo/target_abs": 0.08558246493339539, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 485 + }, + { + "drift/chosen_abs": 0.3685315251350403, + "drift/rejected_abs": 0.150931254029274, + "epoch": 0.15574096146205801, + "grad_norm": 290.0, + "learning_rate": 1.2790861954675703e-08, + "logits/chosen": -2.10400390625, + "logits/rejected": -2.0856916904449463, + "logps/chosen": -0.28186899423599243, + "logps/rejected": -0.28309863805770874, + "loss": 0.6792070269584656, + "loss/core": 0.643970787525177, + "loss/preference_sft": 0.28243130445480347, + "loss/reference_anchor": 0.676482617855072, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.6826233863830566, + "rewards/margins": 2.1746814250946045, + "rewards/rejected": 1.5079416036605835, + "ronpo/logit": 0.21746817231178284, + "ronpo/residual": 0.21048131585121155, + "ronpo/residual_abs": 0.2691686451435089, + "ronpo/target": 0.006986844353377819, + "ronpo/target_abs": 0.09098772704601288, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 490 + }, + { + "drift/chosen_abs": 0.22006180882453918, + "drift/rejected_abs": 0.18455646932125092, + "epoch": 0.1573301549463647, + "grad_norm": 396.0, + "learning_rate": 1.2548481246561504e-08, + "logits/chosen": -2.1262259483337402, + "logits/rejected": -2.0967445373535156, + "logps/chosen": -0.2868593633174896, + "logps/rejected": -0.27408647537231445, + "loss": 1.1514532566070557, + "loss/core": 1.1166924238204956, + "loss/preference_sft": 0.28249964118003845, + "loss/reference_anchor": 0.6669629812240601, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.1990761756896973, + "rewards/margins": 0.357932448387146, + "rewards/rejected": 1.8411436080932617, + "ronpo/logit": 0.03579322621226311, + "ronpo/residual": 0.049376267939805984, + "ronpo/residual_abs": 0.3220534324645996, + "ronpo/target": -0.01358303613960743, + "ronpo/target_abs": 0.09055624902248383, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 495 + }, + { + "drift/chosen_abs": 0.2433398962020874, + "drift/rejected_abs": 0.14733143150806427, + "epoch": 0.15891934843067143, + "grad_norm": 4.125, + "learning_rate": 1.2306082306633209e-08, + "logits/chosen": -2.231499195098877, + "logits/rejected": -2.0890326499938965, + "logps/chosen": -0.2816493809223175, + "logps/rejected": -0.27643585205078125, + "loss": 0.23166105151176453, + "loss/core": 0.21609747409820557, + "loss/preference_sft": 0.2753569185733795, + "loss/reference_anchor": 0.28373581171035767, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.4331910610198975, + "rewards/margins": 0.9780365228652954, + "rewards/rejected": 1.455154538154602, + "ronpo/logit": 0.09780365228652954, + "ronpo/residual": 0.10129328817129135, + "ronpo/residual_abs": 0.21842603385448456, + "ronpo/target": -0.0034896410070359707, + "ronpo/target_abs": 0.08457918465137482, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 500 + }, + { + "drift/chosen_abs": 0.16049498319625854, + "drift/rejected_abs": 0.10690776258707047, + "epoch": 0.16050854191497815, + "grad_norm": 4.5625, + "learning_rate": 1.2063756291218741e-08, + "logits/chosen": -2.284985065460205, + "logits/rejected": -2.205040454864502, + "logps/chosen": -0.29578253626823425, + "logps/rejected": -0.2804624140262604, + "loss": 0.03737588971853256, + "loss/core": 0.0324200838804245, + "loss/preference_sft": 0.2899005115032196, + "loss/reference_anchor": 0.070126011967659, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.6018108129501343, + "rewards/margins": 0.5337648987770081, + "rewards/rejected": 1.0680458545684814, + "ronpo/logit": 0.05337648466229439, + "ronpo/residual": 0.051874689757823944, + "ronpo/residual_abs": 0.12874646484851837, + "ronpo/target": 0.0015017971163615584, + "ronpo/target_abs": 0.08839758485555649, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 505 + }, + { + "drift/chosen_abs": 0.31590768694877625, + "drift/rejected_abs": 0.1603766679763794, + "epoch": 0.16209773539928488, + "grad_norm": 2.671875, + "learning_rate": 1.1821594329222079e-08, + "logits/chosen": -1.9426066875457764, + "logits/rejected": -1.9659500122070312, + "logps/chosen": -0.26106467843055725, + "logps/rejected": -0.2636715769767761, + "loss": 0.4681622087955475, + "loss/core": 0.435793936252594, + "loss/preference_sft": 0.26362866163253784, + "loss/reference_anchor": 0.6210030317306519, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.1584575176239014, + "rewards/margins": 1.5580565929412842, + "rewards/rejected": 1.6004009246826172, + "ronpo/logit": 0.15580564737319946, + "ronpo/residual": 0.15252608060836792, + "ronpo/residual_abs": 0.2079734355211258, + "ronpo/target": 0.0032795709557831287, + "ronpo/target_abs": 0.07548246532678604, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 510 + }, + { + "drift/chosen_abs": 0.2593478560447693, + "drift/rejected_abs": 0.15457691252231598, + "epoch": 0.16368692888359157, + "grad_norm": 5.03125, + "learning_rate": 1.157968748785344e-08, + "logits/chosen": -2.1415348052978516, + "logits/rejected": -2.2097504138946533, + "logps/chosen": -0.28786981105804443, + "logps/rejected": -0.27476245164871216, + "loss": 0.3538331389427185, + "loss/core": 0.33824825286865234, + "loss/preference_sft": 0.2842520475387573, + "loss/reference_anchor": 0.2832731306552887, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.5934786796569824, + "rewards/margins": 1.050806999206543, + "rewards/rejected": 1.542672038078308, + "ronpo/logit": 0.10508068650960922, + "ronpo/residual": 0.1014178991317749, + "ronpo/residual_abs": 0.22226110100746155, + "ronpo/target": 0.0036627918016165495, + "ronpo/target_abs": 0.07639802247285843, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 515 + }, + { + "drift/chosen_abs": 0.2165084183216095, + "drift/rejected_abs": 0.10440798103809357, + "epoch": 0.1652761223678983, + "grad_norm": 3.078125, + "learning_rate": 1.1338126738382598e-08, + "logits/chosen": -2.1498093605041504, + "logits/rejected": -2.1602375507354736, + "logps/chosen": -0.2931273579597473, + "logps/rejected": -0.30199140310287476, + "loss": 0.261959433555603, + "loss/core": 0.2352810651063919, + "loss/preference_sft": 0.293254554271698, + "loss/reference_anchor": 0.5042413473129272, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.16508412361145, + "rewards/margins": 1.1286433935165405, + "rewards/rejected": 1.0364407300949097, + "ronpo/logit": 0.11286433786153793, + "ronpo/residual": 0.10544689744710922, + "ronpo/residual_abs": 0.17088401317596436, + "ronpo/target": 0.007417450193315744, + "ronpo/target_abs": 0.08988238126039505, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 520 + }, + { + "drift/chosen_abs": 0.22024352848529816, + "drift/rejected_abs": 0.06529395282268524, + "epoch": 0.16686531585220502, + "grad_norm": 7.9375, + "learning_rate": 1.1097002921928317e-08, + "logits/chosen": -1.9056131839752197, + "logits/rejected": -2.050123929977417, + "logps/chosen": -0.3115862309932709, + "logps/rejected": -0.3198958933353424, + "loss": 0.2833040654659271, + "loss/core": 0.2724967300891876, + "loss/preference_sft": 0.32389670610427856, + "loss/reference_anchor": 0.18375670909881592, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.201336145401001, + "rewards/margins": 1.5499281883239746, + "rewards/rejected": 0.6514078378677368, + "ronpo/logit": 0.15499281883239746, + "ronpo/residual": 0.1652340143918991, + "ronpo/residual_abs": 0.22205249965190887, + "ronpo/target": -0.010241201147437096, + "ronpo/target_abs": 0.09030992537736893, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 525 + }, + { + "drift/chosen_abs": 0.15349416434764862, + "drift/rejected_abs": 0.06507018208503723, + "epoch": 0.1684545093365117, + "grad_norm": 1.2734375, + "learning_rate": 1.0856406715296717e-08, + "logits/chosen": -2.3231589794158936, + "logits/rejected": -2.317765712738037, + "logps/chosen": -0.2665853500366211, + "logps/rejected": -0.2610515356063843, + "loss": 0.06234220415353775, + "loss/core": 0.05808652564883232, + "loss/preference_sft": 0.26546263694763184, + "loss/reference_anchor": 0.05856741592288017, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 1.534586787223816, + "rewards/margins": 0.8877205848693848, + "rewards/rejected": 0.6468662023544312, + "ronpo/logit": 0.08877205848693848, + "ronpo/residual": 0.09845416247844696, + "ronpo/residual_abs": 0.13878455758094788, + "ronpo/target": -0.009682102128863335, + "ronpo/target_abs": 0.08094774186611176, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 530 + }, + { + "drift/chosen_abs": 0.3475359380245209, + "drift/rejected_abs": 0.1439954936504364, + "epoch": 0.17004370282081843, + "grad_norm": 3.171875, + "learning_rate": 1.0616428596881461e-08, + "logits/chosen": -2.2414088249206543, + "logits/rejected": -2.255117893218994, + "logps/chosen": -0.31528395414352417, + "logps/rejected": -0.3214128613471985, + "loss": 1.5130736827850342, + "loss/core": 1.4584028720855713, + "loss/preference_sft": 0.31637319922447205, + "loss/reference_anchor": 1.0617750883102417, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.473903179168701, + "rewards/margins": 2.195761203765869, + "rewards/rejected": 1.2781422138214111, + "ronpo/logit": 0.21957612037658691, + "ronpo/residual": 0.2536545991897583, + "ronpo/residual_abs": 0.3511970341205597, + "ronpo/target": -0.034078482538461685, + "ronpo/target_abs": 0.09332413971424103, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 535 + }, + { + "drift/chosen_abs": 0.24628369510173798, + "drift/rejected_abs": 0.1078454852104187, + "epoch": 0.17163289630512515, + "grad_norm": 4.6875, + "learning_rate": 1.0377158812638491e-08, + "logits/chosen": -2.202691078186035, + "logits/rejected": -2.2384159564971924, + "logps/chosen": -0.27798351645469666, + "logps/rejected": -0.29006585478782654, + "loss": 0.22769439220428467, + "loss/core": 0.20758168399333954, + "loss/preference_sft": 0.28700748085975647, + "loss/reference_anchor": 0.37355345487594604, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.4620206356048584, + "rewards/margins": 1.38406240940094, + "rewards/rejected": 1.0779582262039185, + "ronpo/logit": 0.13840624690055847, + "ronpo/residual": 0.1439955234527588, + "ronpo/residual_abs": 0.1996123194694519, + "ronpo/target": -0.005589271429926157, + "ronpo/target_abs": 0.07924790680408478, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 540 + }, + { + "drift/chosen_abs": 0.3985271155834198, + "drift/rejected_abs": 0.24583449959754944, + "epoch": 0.17322208978943188, + "grad_norm": 3.984375, + "learning_rate": 1.0138687342148249e-08, + "logits/chosen": -1.992592453956604, + "logits/rejected": -2.0166046619415283, + "logps/chosen": -0.2606894373893738, + "logps/rejected": -0.27958837151527405, + "loss": 0.46023693680763245, + "loss/core": 0.41777506470680237, + "loss/preference_sft": 0.27719753980636597, + "loss/reference_anchor": 0.8215174674987793, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.9839775562286377, + "rewards/margins": 1.532893419265747, + "rewards/rejected": 2.4510836601257324, + "ronpo/logit": 0.15328934788703918, + "ronpo/residual": 0.15502643585205078, + "ronpo/residual_abs": 0.2659107744693756, + "ronpo/target": -0.0017371115973219275, + "ronpo/target_abs": 0.0876903235912323, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 545 + }, + { + "drift/chosen_abs": 0.2757337689399719, + "drift/rejected_abs": 0.13784222304821014, + "epoch": 0.17481128327373857, + "grad_norm": 0.8828125, + "learning_rate": 9.90110386477801e-09, + "logits/chosen": -2.144867420196533, + "logits/rejected": -2.1006226539611816, + "logps/chosen": -0.2549815773963928, + "logps/rejected": -0.26524803042411804, + "loss": 0.307639479637146, + "loss/core": 0.28966981172561646, + "loss/preference_sft": 0.26307356357574463, + "loss/reference_anchor": 0.3330860733985901, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.7563281059265137, + "rewards/margins": 1.3787453174591064, + "rewards/rejected": 1.3775826692581177, + "ronpo/logit": 0.1378745287656784, + "ronpo/residual": 0.14565442502498627, + "ronpo/residual_abs": 0.20090845227241516, + "ronpo/target": -0.007779897656291723, + "ronpo/target_abs": 0.0823056623339653, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 550 + }, + { + "drift/chosen_abs": 0.12533283233642578, + "drift/rejected_abs": 0.05521406978368759, + "epoch": 0.1764004767580453, + "grad_norm": 0.640625, + "learning_rate": 9.664497725957164e-09, + "logits/chosen": -2.216212511062622, + "logits/rejected": -2.258756160736084, + "logps/chosen": -0.311422199010849, + "logps/rejected": -0.30880364775657654, + "loss": 0.07220657169818878, + "loss/core": 0.06694497168064117, + "loss/preference_sft": 0.3110686242580414, + "loss/reference_anchor": 0.07412528246641159, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.252854347229004, + "rewards/margins": 0.7011260390281677, + "rewards/rejected": 0.551728367805481, + "ronpo/logit": 0.07011260092258453, + "ronpo/residual": 0.0683169811964035, + "ronpo/residual_abs": 0.1411377489566803, + "ronpo/target": 0.0017956193769350648, + "ronpo/target_abs": 0.09536027908325195, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 555 + }, + { + "drift/chosen_abs": 0.2655150294303894, + "drift/rejected_abs": 0.12211163341999054, + "epoch": 0.17798967024235202, + "grad_norm": 16.5, + "learning_rate": 9.428957903578044e-09, + "logits/chosen": -2.08587646484375, + "logits/rejected": -2.091303586959839, + "logps/chosen": -0.2719891667366028, + "logps/rejected": -0.27926185727119446, + "loss": 0.254443883895874, + "loss/core": 0.24059149622917175, + "loss/preference_sft": 0.2721792161464691, + "loss/reference_anchor": 0.24982962012290955, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.653385639190674, + "rewards/margins": 1.4334917068481445, + "rewards/rejected": 1.2198941707611084, + "ronpo/logit": 0.14334917068481445, + "ronpo/residual": 0.13729219138622284, + "ronpo/residual_abs": 0.18650557100772858, + "ronpo/target": 0.0060569727793335915, + "ronpo/target_abs": 0.0743868425488472, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 560 + }, + { + "drift/chosen_abs": 0.2600178122520447, + "drift/rejected_abs": 0.11504121869802475, + "epoch": 0.1795788637266587, + "grad_norm": 28.25, + "learning_rate": 9.194572974534975e-09, + "logits/chosen": -2.041768789291382, + "logits/rejected": -2.0671210289001465, + "logps/chosen": -0.34080713987350464, + "logps/rejected": -0.3354662358760834, + "loss": 0.36407026648521423, + "loss/core": 0.34170204401016235, + "loss/preference_sft": 0.34413477778434753, + "loss/reference_anchor": 0.4129505157470703, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.597466230392456, + "rewards/margins": 1.4497805833816528, + "rewards/rejected": 1.1476857662200928, + "ronpo/logit": 0.14497807621955872, + "ronpo/residual": 0.144731804728508, + "ronpo/residual_abs": 0.20529747009277344, + "ronpo/target": 0.0002462614211253822, + "ronpo/target_abs": 0.08707638084888458, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 565 + }, + { + "drift/chosen_abs": 0.41662201285362244, + "drift/rejected_abs": 0.09962382912635803, + "epoch": 0.18116805721096543, + "grad_norm": 9.9375, + "learning_rate": 8.96143108141412e-09, + "logits/chosen": -2.1486594676971436, + "logits/rejected": -2.119309902191162, + "logps/chosen": -0.27083221077919006, + "logps/rejected": -0.2647841274738312, + "loss": 1.456986904144287, + "loss/core": 1.414247989654541, + "loss/preference_sft": 0.2618175148963928, + "loss/reference_anchor": 0.8285951614379883, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 4.165776252746582, + "rewards/margins": 3.1706364154815674, + "rewards/rejected": 0.9951398968696594, + "ronpo/logit": 0.31706368923187256, + "ronpo/residual": 0.30529677867889404, + "ronpo/residual_abs": 0.3695726692676544, + "ronpo/target": 0.011766904965043068, + "ronpo/target_abs": 0.0811844915151596, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 570 + }, + { + "drift/chosen_abs": 0.19189342856407166, + "drift/rejected_abs": 0.09652575105428696, + "epoch": 0.18275725069527216, + "grad_norm": 3.328125, + "learning_rate": 8.72961989934668e-09, + "logits/chosen": -2.0099151134490967, + "logits/rejected": -2.0976462364196777, + "logps/chosen": -0.28342169523239136, + "logps/rejected": -0.27922505140304565, + "loss": 0.05769554525613785, + "loss/core": 0.052024852484464645, + "loss/preference_sft": 0.28891006112098694, + "loss/reference_anchor": 0.08452272415161133, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.9166200160980225, + "rewards/margins": 0.9536026120185852, + "rewards/rejected": 0.963017463684082, + "ronpo/logit": 0.09536024928092957, + "ronpo/residual": 0.10312417894601822, + "ronpo/residual_abs": 0.14294111728668213, + "ronpo/target": -0.00776391988620162, + "ronpo/target_abs": 0.07904698699712753, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 575 + }, + { + "drift/chosen_abs": 0.36418384313583374, + "drift/rejected_abs": 0.1662895232439041, + "epoch": 0.18434644417957885, + "grad_norm": 1.7734375, + "learning_rate": 8.499226603037854e-09, + "logits/chosen": -2.0011439323425293, + "logits/rejected": -2.031702756881714, + "logps/chosen": -0.28904083371162415, + "logps/rejected": -0.2685038447380066, + "loss": 0.5859975814819336, + "loss/core": 0.5528149008750916, + "loss/preference_sft": 0.28503909707069397, + "loss/reference_anchor": 0.6351504921913147, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.640242099761963, + "rewards/margins": 1.9781475067138672, + "rewards/rejected": 1.6620944738388062, + "ronpo/logit": 0.19781477749347687, + "ronpo/residual": 0.19819863140583038, + "ronpo/residual_abs": 0.2739180326461792, + "ronpo/target": -0.0003838575503323227, + "ronpo/target_abs": 0.08456491678953171, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 580 + }, + { + "drift/chosen_abs": 0.23182368278503418, + "drift/rejected_abs": 0.09531630575656891, + "epoch": 0.18593563766388557, + "grad_norm": 1.9375, + "learning_rate": 8.270337833983987e-09, + "logits/chosen": -1.9567636251449585, + "logits/rejected": -1.9816787242889404, + "logps/chosen": -0.28648510575294495, + "logps/rejected": -0.28638434410095215, + "loss": 0.15469267964363098, + "loss/core": 0.14479199051856995, + "loss/preference_sft": 0.29087167978286743, + "loss/reference_anchor": 0.16892696917057037, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.317553997039795, + "rewards/margins": 1.3662830591201782, + "rewards/rejected": 0.9512712359428406, + "ronpo/logit": 0.13662829995155334, + "ronpo/residual": 0.1309715062379837, + "ronpo/residual_abs": 0.17593583464622498, + "ronpo/target": 0.00565680768340826, + "ronpo/target_abs": 0.0699964389204979, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 585 + }, + { + "drift/chosen_abs": 0.25730934739112854, + "drift/rejected_abs": 0.1230391412973404, + "epoch": 0.1875248311481923, + "grad_norm": 1.375, + "learning_rate": 8.04303966789025e-09, + "logits/chosen": -2.118440866470337, + "logits/rejected": -2.0865204334259033, + "logps/chosen": -0.3093511760234833, + "logps/rejected": -0.3141013979911804, + "loss": 0.5824509263038635, + "loss/core": 0.5587186217308044, + "loss/preference_sft": 0.3041399419307709, + "loss/reference_anchor": 0.4442318379878998, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.570599317550659, + "rewards/margins": 1.3402273654937744, + "rewards/rejected": 1.2303721904754639, + "ronpo/logit": 0.1340227574110031, + "ronpo/residual": 0.14667926728725433, + "ronpo/residual_abs": 0.22119927406311035, + "ronpo/target": -0.012656529434025288, + "ronpo/target_abs": 0.08529551327228546, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 590 + }, + { + "drift/chosen_abs": 0.23181328177452087, + "drift/rejected_abs": 0.11188916116952896, + "epoch": 0.18911402463249902, + "grad_norm": 6.5, + "learning_rate": 7.817417582301098e-09, + "logits/chosen": -1.9782168865203857, + "logits/rejected": -1.9370613098144531, + "logps/chosen": -0.2896624207496643, + "logps/rejected": -0.29374343156814575, + "loss": 0.1985751986503601, + "loss/core": 0.18759621679782867, + "loss/preference_sft": 0.29568907618522644, + "loss/reference_anchor": 0.1900101602077484, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.3166379928588867, + "rewards/margins": 1.202042818069458, + "rewards/rejected": 1.1145952939987183, + "ronpo/logit": 0.12020426988601685, + "ronpo/residual": 0.11585943400859833, + "ronpo/residual_abs": 0.1900613158941269, + "ronpo/target": 0.00434483215212822, + "ronpo/target_abs": 0.08699861913919449, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 595 + }, + { + "drift/chosen_abs": 0.2037758082151413, + "drift/rejected_abs": 0.14159931242465973, + "epoch": 0.1907032181168057, + "grad_norm": 152.0, + "learning_rate": 7.59355642445566e-09, + "logits/chosen": -2.1320338249206543, + "logits/rejected": -2.1046671867370605, + "logps/chosen": -0.29946523904800415, + "logps/rejected": -0.3209082782268524, + "loss": 0.4402243494987488, + "loss/core": 0.4250958561897278, + "loss/preference_sft": 0.3151097893714905, + "loss/reference_anchor": 0.27105993032455444, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.0345118045806885, + "rewards/margins": 0.6213451623916626, + "rewards/rejected": 1.4131665229797363, + "ronpo/logit": 0.0621345154941082, + "ronpo/residual": 0.06725134700536728, + "ronpo/residual_abs": 0.2370101511478424, + "ronpo/target": -0.005116832908242941, + "ronpo/target_abs": 0.07861164957284927, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 600 + }, + { + "drift/chosen_abs": 0.34658682346343994, + "drift/rejected_abs": 0.1579512059688568, + "epoch": 0.19229241160111243, + "grad_norm": 1.8359375, + "learning_rate": 7.371540379380149e-09, + "logits/chosen": -2.0286879539489746, + "logits/rejected": -1.953050971031189, + "logps/chosen": -0.283130407333374, + "logps/rejected": -0.27908605337142944, + "loss": 0.4860695004463196, + "loss/core": 0.45568352937698364, + "loss/preference_sft": 0.28036171197891235, + "loss/reference_anchor": 0.5796834826469421, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.456225633621216, + "rewards/margins": 1.8863617181777954, + "rewards/rejected": 1.5698635578155518, + "ronpo/logit": 0.1886361837387085, + "ronpo/residual": 0.18362456560134888, + "ronpo/residual_abs": 0.25072842836380005, + "ronpo/target": 0.005011610221117735, + "ronpo/target_abs": 0.09113539010286331, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 605 + }, + { + "drift/chosen_abs": 0.33414921164512634, + "drift/rejected_abs": 0.15187159180641174, + "epoch": 0.19388160508541916, + "grad_norm": 87.5, + "learning_rate": 7.151452938229324e-09, + "logits/chosen": -2.0221667289733887, + "logits/rejected": -1.9719922542572021, + "logps/chosen": -0.31065306067466736, + "logps/rejected": -0.294253408908844, + "loss": 0.34279918670654297, + "loss/core": 0.31768709421157837, + "loss/preference_sft": 0.3136529326438904, + "loss/reference_anchor": 0.470877081155777, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.3398075103759766, + "rewards/margins": 1.8233705759048462, + "rewards/rejected": 1.5164368152618408, + "ronpo/logit": 0.18233706057071686, + "ronpo/residual": 0.1946430653333664, + "ronpo/residual_abs": 0.24323885142803192, + "ronpo/target": -0.012306014075875282, + "ronpo/target_abs": 0.0871143639087677, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 610 + }, + { + "drift/chosen_abs": 0.2550937533378601, + "drift/rejected_abs": 0.11376242339611053, + "epoch": 0.19547079856972585, + "grad_norm": 125.5, + "learning_rate": 6.933376866888882e-09, + "logits/chosen": -2.2011666297912598, + "logits/rejected": -2.162236452102661, + "logps/chosen": -0.28771674633026123, + "logps/rejected": -0.28812357783317566, + "loss": 0.3027460277080536, + "loss/core": 0.28452661633491516, + "loss/preference_sft": 0.2859073579311371, + "loss/reference_anchor": 0.3357972800731659, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.5509376525878906, + "rewards/margins": 1.4323103427886963, + "rewards/rejected": 1.1186271905899048, + "ronpo/logit": 0.14323106408119202, + "ronpo/residual": 0.14810332655906677, + "ronpo/residual_abs": 0.1831142008304596, + "ronpo/target": -0.004872290883213282, + "ronpo/target_abs": 0.07001273334026337, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 615 + }, + { + "drift/chosen_abs": 0.22261254489421844, + "drift/rejected_abs": 0.10966954380273819, + "epoch": 0.19705999205403257, + "grad_norm": 12.0, + "learning_rate": 6.717394174850605e-09, + "logits/chosen": -2.0854969024658203, + "logits/rejected": -2.0211100578308105, + "logps/chosen": -0.2912364602088928, + "logps/rejected": -0.2868305444717407, + "loss": 0.10874249786138535, + "loss/core": 0.1000494584441185, + "loss/preference_sft": 0.28769445419311523, + "loss/reference_anchor": 0.14509165287017822, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.2254793643951416, + "rewards/margins": 1.1289622783660889, + "rewards/rejected": 1.0965168476104736, + "ronpo/logit": 0.11289622634649277, + "ronpo/residual": 0.11616779863834381, + "ronpo/residual_abs": 0.1724853515625, + "ronpo/target": -0.003271549940109253, + "ronpo/target_abs": 0.07885955274105072, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 620 + }, + { + "drift/chosen_abs": 0.24589157104492188, + "drift/rejected_abs": 0.09249149262905121, + "epoch": 0.1986491855383393, + "grad_norm": 49.0, + "learning_rate": 6.503586084371926e-09, + "logits/chosen": -2.0363283157348633, + "logits/rejected": -2.0404984951019287, + "logps/chosen": -0.2965640425682068, + "logps/rejected": -0.2985779047012329, + "loss": 0.34146690368652344, + "loss/core": 0.32861843705177307, + "loss/preference_sft": 0.29958051443099976, + "loss/reference_anchor": 0.22701041400432587, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.457526445388794, + "rewards/margins": 1.536529302597046, + "rewards/rejected": 0.920997142791748, + "ronpo/logit": 0.15365295112133026, + "ronpo/residual": 0.15617410838603973, + "ronpo/residual_abs": 0.213802307844162, + "ronpo/target": -0.0025211633183062077, + "ronpo/target_abs": 0.08408491313457489, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 625 + }, + { + "drift/chosen_abs": 0.23458974063396454, + "drift/rejected_abs": 0.08947785198688507, + "epoch": 0.20023837902264602, + "grad_norm": 2.875, + "learning_rate": 6.29203299993155e-09, + "logits/chosen": -2.325148105621338, + "logits/rejected": -2.414926052093506, + "logps/chosen": -0.2667160928249359, + "logps/rejected": -0.26361387968063354, + "loss": 0.3130393922328949, + "loss/core": 0.2962197959423065, + "loss/preference_sft": 0.2673772871494293, + "loss/reference_anchor": 0.3096539378166199, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.345794439315796, + "rewards/margins": 1.4539344310760498, + "rewards/rejected": 0.8918598890304565, + "ronpo/logit": 0.14539344608783722, + "ronpo/residual": 0.13748937845230103, + "ronpo/residual_abs": 0.20279105007648468, + "ronpo/target": 0.007904065772891045, + "ronpo/target_abs": 0.08239643275737762, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 630 + }, + { + "drift/chosen_abs": 0.28986817598342896, + "drift/rejected_abs": 0.10853831470012665, + "epoch": 0.2018275725069527, + "grad_norm": 15.375, + "learning_rate": 6.0828144779926565e-09, + "logits/chosen": -2.111020565032959, + "logits/rejected": -2.1508452892303467, + "logps/chosen": -0.2651686668395996, + "logps/rejected": -0.2719859182834625, + "loss": 0.3829234540462494, + "loss/core": 0.36607077717781067, + "loss/preference_sft": 0.269239604473114, + "loss/reference_anchor": 0.31012919545173645, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.8979406356811523, + "rewards/margins": 1.8216516971588135, + "rewards/rejected": 1.0762889385223389, + "ronpo/logit": 0.18216517567634583, + "ronpo/residual": 0.1788938343524933, + "ronpo/residual_abs": 0.22935160994529724, + "ronpo/target": 0.003271352034062147, + "ronpo/target_abs": 0.07819235324859619, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 635 + }, + { + "drift/chosen_abs": 0.3245066702365875, + "drift/rejected_abs": 0.09787418693304062, + "epoch": 0.20341676599125944, + "grad_norm": 1.3046875, + "learning_rate": 5.87600919708494e-09, + "logits/chosen": -2.0167291164398193, + "logits/rejected": -2.0083279609680176, + "logps/chosen": -0.3133408725261688, + "logps/rejected": -0.3174097239971161, + "loss": 0.7297495603561401, + "loss/core": 0.7002626657485962, + "loss/preference_sft": 0.3104119300842285, + "loss/reference_anchor": 0.5586963891983032, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.2447280883789062, + "rewards/margins": 2.300731897354126, + "rewards/rejected": 0.9439964294433594, + "ronpo/logit": 0.23007316887378693, + "ronpo/residual": 0.23132848739624023, + "ronpo/residual_abs": 0.27878475189208984, + "ronpo/target": -0.0012552805710583925, + "ronpo/target_abs": 0.08154843747615814, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 640 + }, + { + "drift/chosen_abs": 0.2655944526195526, + "drift/rejected_abs": 0.11456499993801117, + "epoch": 0.20500595947556616, + "grad_norm": 7.34375, + "learning_rate": 5.671694928216829e-09, + "logits/chosen": -2.1220450401306152, + "logits/rejected": -2.127779722213745, + "logps/chosen": -0.3085785508155823, + "logps/rejected": -0.29471153020858765, + "loss": 0.3396216034889221, + "loss/core": 0.32221999764442444, + "loss/preference_sft": 0.3059234619140625, + "loss/reference_anchor": 0.3174397945404053, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.6536717414855957, + "rewards/margins": 1.5116037130355835, + "rewards/rejected": 1.1420681476593018, + "ronpo/logit": 0.15116038918495178, + "ronpo/residual": 0.14593927562236786, + "ronpo/residual_abs": 0.2119034230709076, + "ronpo/target": 0.005221105180680752, + "ronpo/target_abs": 0.07934162765741348, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 645 + }, + { + "drift/chosen_abs": 0.15132535994052887, + "drift/rejected_abs": 0.09072579443454742, + "epoch": 0.20659515295987285, + "grad_norm": 1.859375, + "learning_rate": 5.469948505629e-09, + "logits/chosen": -2.2047553062438965, + "logits/rejected": -2.1337642669677734, + "logps/chosen": -0.30132001638412476, + "logps/rejected": -0.2931322455406189, + "loss": 0.06630890816450119, + "loss/core": 0.06135622784495354, + "loss/preference_sft": 0.3019540309906006, + "loss/reference_anchor": 0.06885839998722076, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.5123118162155151, + "rewards/margins": 0.6547757983207703, + "rewards/rejected": 0.8575360178947449, + "ronpo/logit": 0.06547758728265762, + "ronpo/residual": 0.06879720091819763, + "ronpo/residual_abs": 0.14413335919380188, + "ronpo/target": -0.003319614799693227, + "ronpo/target_abs": 0.08891183882951736, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 650 + }, + { + "drift/chosen_abs": 0.20433752238750458, + "drift/rejected_abs": 0.06546036899089813, + "epoch": 0.20818434644417957, + "grad_norm": 160.0, + "learning_rate": 5.270845797900168e-09, + "logits/chosen": -2.2784745693206787, + "logits/rejected": -2.2848639488220215, + "logps/chosen": -0.28383541107177734, + "logps/rejected": -0.29783403873443604, + "loss": 0.3861464858055115, + "loss/core": 0.3738086223602295, + "loss/preference_sft": 0.2895219326019287, + "loss/reference_anchor": 0.21780526638031006, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.042762517929077, + "rewards/margins": 1.3915107250213623, + "rewards/rejected": 0.6512519121170044, + "ronpo/logit": 0.13915108144283295, + "ronpo/residual": 0.16224148869514465, + "ronpo/residual_abs": 0.21403923630714417, + "ronpo/target": -0.02309040166437626, + "ronpo/target_abs": 0.0899852067232132, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 655 + }, + { + "drift/chosen_abs": 0.1753084361553192, + "drift/rejected_abs": 0.07402169704437256, + "epoch": 0.2097735399284863, + "grad_norm": 6.65625, + "learning_rate": 5.07446167941601e-09, + "logits/chosen": -2.2544825077056885, + "logits/rejected": -2.2972869873046875, + "logps/chosen": -0.2956244647502899, + "logps/rejected": -0.28930968046188354, + "loss": 0.1263386309146881, + "loss/core": 0.11955497413873672, + "loss/preference_sft": 0.29426223039627075, + "loss/reference_anchor": 0.10624738782644272, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 1.7521049976348877, + "rewards/margins": 1.014744520187378, + "rewards/rejected": 0.7373603582382202, + "ronpo/logit": 0.10147446393966675, + "ronpo/residual": 0.09581993520259857, + "ronpo/residual_abs": 0.14587488770484924, + "ronpo/target": 0.005654525477439165, + "ronpo/target_abs": 0.07976684719324112, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 660 + }, + { + "drift/chosen_abs": 0.23486240208148956, + "drift/rejected_abs": 0.13577520847320557, + "epoch": 0.21136273341279302, + "grad_norm": 0.6953125, + "learning_rate": 4.880870002211964e-09, + "logits/chosen": -2.0763564109802246, + "logits/rejected": -2.0201761722564697, + "logps/chosen": -0.2823403477668762, + "logps/rejected": -0.2865101993083954, + "loss": 0.24251846969127655, + "loss/core": 0.22713109850883484, + "loss/preference_sft": 0.284621000289917, + "loss/reference_anchor": 0.2792838513851166, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.3464529514312744, + "rewards/margins": 0.9889312982559204, + "rewards/rejected": 1.3575217723846436, + "ronpo/logit": 0.09889312833547592, + "ronpo/residual": 0.10298037528991699, + "ronpo/residual_abs": 0.1839233934879303, + "ronpo/target": -0.004087253473699093, + "ronpo/target_abs": 0.09008091688156128, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 665 + }, + { + "drift/chosen_abs": 0.20962342619895935, + "drift/rejected_abs": 0.07850834727287292, + "epoch": 0.2129519268970997, + "grad_norm": 5.40625, + "learning_rate": 4.6901435682005e-09, + "logits/chosen": -2.3460006713867188, + "logits/rejected": -2.3546290397644043, + "logps/chosen": -0.26797959208488464, + "logps/rejected": -0.27196618914604187, + "loss": 0.2039221227169037, + "loss/core": 0.1944776475429535, + "loss/preference_sft": 0.2721707820892334, + "loss/reference_anchor": 0.16167283058166504, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.094529390335083, + "rewards/margins": 1.3098270893096924, + "rewards/rejected": 0.7847023010253906, + "ronpo/logit": 0.13098271191120148, + "ronpo/residual": 0.15417954325675964, + "ronpo/residual_abs": 0.19339290261268616, + "ronpo/target": -0.02319684997200966, + "ronpo/target_abs": 0.08436106145381927, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 670 + }, + { + "drift/chosen_abs": 0.40202194452285767, + "drift/rejected_abs": 0.08130165189504623, + "epoch": 0.21454112038140644, + "grad_norm": 1.90625, + "learning_rate": 4.502354101793314e-09, + "logits/chosen": -1.9860864877700806, + "logits/rejected": -2.099212646484375, + "logps/chosen": -0.2779615521430969, + "logps/rejected": -0.26965683698654175, + "loss": 1.4497228860855103, + "loss/core": 1.408434271812439, + "loss/preference_sft": 0.2684791684150696, + "loss/reference_anchor": 0.7989233732223511, + "rewards/accuracies": 0.9375, + "rewards/chosen": 4.020218849182129, + "rewards/margins": 3.2108778953552246, + "rewards/rejected": 0.8093411326408386, + "ronpo/logit": 0.3210877776145935, + "ronpo/residual": 0.3173868954181671, + "ronpo/residual_abs": 0.3734723925590515, + "ronpo/target": 0.003700877772644162, + "ronpo/target_abs": 0.09122522175312042, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 675 + }, + { + "drift/chosen_abs": 0.19475491344928741, + "drift/rejected_abs": 0.1171645075082779, + "epoch": 0.21613031386571316, + "grad_norm": 3.171875, + "learning_rate": 4.317572222928703e-09, + "logits/chosen": -1.9540669918060303, + "logits/rejected": -1.9495131969451904, + "logps/chosen": -0.2881017029285431, + "logps/rejected": -0.28433549404144287, + "loss": 0.06940118968486786, + "loss/core": 0.05890883132815361, + "loss/preference_sft": 0.285450279712677, + "loss/reference_anchor": 0.18130208551883698, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9457314014434814, + "rewards/margins": 0.7747111320495605, + "rewards/rejected": 1.1710201501846313, + "ronpo/logit": 0.07747111469507217, + "ronpo/residual": 0.08135292679071426, + "ronpo/residual_abs": 0.13843822479248047, + "ronpo/target": -0.0038818151224404573, + "ronpo/target_abs": 0.08158716559410095, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 680 + }, + { + "drift/chosen_abs": 0.18266239762306213, + "drift/rejected_abs": 0.09462437778711319, + "epoch": 0.21771950735001985, + "grad_norm": 3.421875, + "learning_rate": 4.135867420514276e-09, + "logits/chosen": -1.9716804027557373, + "logits/rejected": -2.060253620147705, + "logps/chosen": -0.2921956181526184, + "logps/rejected": -0.2996564209461212, + "loss": 0.05632435530424118, + "loss/core": 0.05092400312423706, + "loss/preference_sft": 0.29816001653671265, + "loss/reference_anchor": 0.07819100469350815, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.825413465499878, + "rewards/margins": 0.8813973665237427, + "rewards/rejected": 0.9440159797668457, + "ronpo/logit": 0.08813974261283875, + "ronpo/residual": 0.08088794350624084, + "ronpo/residual_abs": 0.14101745188236237, + "ronpo/target": 0.007251805160194635, + "ronpo/target_abs": 0.08236397802829742, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 685 + }, + { + "drift/chosen_abs": 0.4524809420108795, + "drift/rejected_abs": 0.1857810914516449, + "epoch": 0.21930870083432658, + "grad_norm": 434.0, + "learning_rate": 3.957308026295035e-09, + "logits/chosen": -1.9309346675872803, + "logits/rejected": -1.9748626947402954, + "logps/chosen": -0.27637702226638794, + "logps/rejected": -0.271381676197052, + "loss": 0.8734847903251648, + "loss/core": 0.8166624903678894, + "loss/preference_sft": 0.2828080356121063, + "loss/reference_anchor": 1.1081647872924805, + "rewards/accuracies": 0.9375, + "rewards/chosen": 4.524248123168945, + "rewards/margins": 2.668072462081909, + "rewards/rejected": 1.856175422668457, + "ronpo/logit": 0.2668072581291199, + "ronpo/residual": 0.25683221220970154, + "ronpo/residual_abs": 0.3273332417011261, + "ronpo/target": 0.009975056163966656, + "ronpo/target_abs": 0.09136839956045151, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 690 + }, + { + "drift/chosen_abs": 0.18365684151649475, + "drift/rejected_abs": 0.11019404232501984, + "epoch": 0.2208978943186333, + "grad_norm": 12.5, + "learning_rate": 3.781961189156609e-09, + "logits/chosen": -2.160966634750366, + "logits/rejected": -2.1480095386505127, + "logps/chosen": -0.298018217086792, + "logps/rejected": -0.30044853687286377, + "loss": 0.10467197746038437, + "loss/core": 0.0970427542924881, + "loss/preference_sft": 0.2983938455581665, + "loss/reference_anchor": 0.12274493277072906, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.8359683752059937, + "rewards/margins": 0.7349220514297485, + "rewards/rejected": 1.1010462045669556, + "ronpo/logit": 0.07349221408367157, + "ronpo/residual": 0.05487983301281929, + "ronpo/residual_abs": 0.16237375140190125, + "ronpo/target": 0.01861238107085228, + "ronpo/target_abs": 0.07778690755367279, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 695 + }, + { + "drift/chosen_abs": 0.22338958084583282, + "drift/rejected_abs": 0.13935190439224243, + "epoch": 0.22248708780294002, + "grad_norm": 6.15625, + "learning_rate": 3.609892849873286e-09, + "logits/chosen": -2.2161097526550293, + "logits/rejected": -2.176314115524292, + "logps/chosen": -0.321241557598114, + "logps/rejected": -0.3120388686656952, + "loss": 0.4451649785041809, + "loss/core": 0.4236728250980377, + "loss/preference_sft": 0.3101545572280884, + "loss/reference_anchor": 0.398827463388443, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.233896017074585, + "rewards/margins": 0.842010498046875, + "rewards/rejected": 1.39188551902771, + "ronpo/logit": 0.08420103788375854, + "ronpo/residual": 0.0672958642244339, + "ronpo/residual_abs": 0.19979240000247955, + "ronpo/target": 0.016905177384614944, + "ronpo/target_abs": 0.06755395233631134, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 700 + }, + { + "drift/chosen_abs": 0.1843368113040924, + "drift/rejected_abs": 0.08406703919172287, + "epoch": 0.22407628128724671, + "grad_norm": 3.265625, + "learning_rate": 3.441167716310389e-09, + "logits/chosen": -2.2613587379455566, + "logits/rejected": -2.366189479827881, + "logps/chosen": -0.2671636939048767, + "logps/rejected": -0.2705652117729187, + "loss": 0.07335400581359863, + "loss/core": 0.06695351749658585, + "loss/preference_sft": 0.27716490626335144, + "loss/reference_anchor": 0.10029320418834686, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.8426601886749268, + "rewards/margins": 1.0054227113723755, + "rewards/rejected": 0.8372377157211304, + "ronpo/logit": 0.10054226219654083, + "ronpo/residual": 0.08631283044815063, + "ronpo/residual_abs": 0.14657628536224365, + "ronpo/target": 0.014229427091777325, + "ronpo/target_abs": 0.08256835490465164, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 705 + }, + { + "drift/chosen_abs": 0.28539392352104187, + "drift/rejected_abs": 0.10650794208049774, + "epoch": 0.22566547477155344, + "grad_norm": 108.0, + "learning_rate": 3.275849239090295e-09, + "logits/chosen": -2.166898727416992, + "logits/rejected": -2.178861141204834, + "logps/chosen": -0.30302557349205017, + "logps/rejected": -0.3004283905029297, + "loss": 0.707269549369812, + "loss/core": 0.6864758729934692, + "loss/preference_sft": 0.3012581765651703, + "loss/reference_anchor": 0.38574856519699097, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.8538074493408203, + "rewards/margins": 1.9227615594863892, + "rewards/rejected": 0.9310457110404968, + "ronpo/logit": 0.19227614998817444, + "ronpo/residual": 0.18570926785469055, + "ronpo/residual_abs": 0.25992774963378906, + "ronpo/target": 0.006566896103322506, + "ronpo/target_abs": 0.08580425381660461, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 710 + }, + { + "drift/chosen_abs": 0.16650691628456116, + "drift/rejected_abs": 0.09241025149822235, + "epoch": 0.22725466825586016, + "grad_norm": 0.92578125, + "learning_rate": 3.11399958773126e-09, + "logits/chosen": -2.047321081161499, + "logits/rejected": -1.9679555892944336, + "logps/chosen": -0.30258211493492126, + "logps/rejected": -0.29988473653793335, + "loss": 0.07320668548345566, + "loss/core": 0.06755281984806061, + "loss/preference_sft": 0.30704405903816223, + "loss/reference_anchor": 0.08237285912036896, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.6642707586288452, + "rewards/margins": 0.740811824798584, + "rewards/rejected": 0.9234589338302612, + "ronpo/logit": 0.07408119738101959, + "ronpo/residual": 0.07377193868160248, + "ronpo/residual_abs": 0.150286003947258, + "ronpo/target": 0.0003092583210673183, + "ronpo/target_abs": 0.08682587742805481, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 715 + }, + { + "drift/chosen_abs": 0.3417677581310272, + "drift/rejected_abs": 0.1455438882112503, + "epoch": 0.22884386174016685, + "grad_norm": 508.0, + "learning_rate": 2.955679627267997e-09, + "logits/chosen": -1.951354742050171, + "logits/rejected": -2.0309109687805176, + "logps/chosen": -0.2951735854148865, + "logps/rejected": -0.28592047095298767, + "loss": 0.5634924173355103, + "loss/core": 0.5344976782798767, + "loss/preference_sft": 0.2908739447593689, + "loss/reference_anchor": 0.5508082509040833, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.4174652099609375, + "rewards/margins": 2.024085283279419, + "rewards/rejected": 1.393379807472229, + "ronpo/logit": 0.20240852236747742, + "ronpo/residual": 0.18394221365451813, + "ronpo/residual_abs": 0.25016313791275024, + "ronpo/target": 0.018466297537088394, + "ronpo/target_abs": 0.07852284610271454, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 720 + }, + { + "drift/chosen_abs": 0.295247346162796, + "drift/rejected_abs": 0.06837625056505203, + "epoch": 0.23043305522447358, + "grad_norm": 4.125, + "learning_rate": 2.8009488953628218e-09, + "logits/chosen": -2.0851798057556152, + "logits/rejected": -2.0262653827667236, + "logps/chosen": -0.2910119891166687, + "logps/rejected": -0.2917248606681824, + "loss": 1.1185815334320068, + "loss/core": 1.0883275270462036, + "loss/preference_sft": 0.2941739857196808, + "loss/reference_anchor": 0.5756635069847107, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.9508135318756104, + "rewards/margins": 2.2699127197265625, + "rewards/rejected": 0.6809011101722717, + "ronpo/logit": 0.22699125111103058, + "ronpo/residual": 0.23236696422100067, + "ronpo/residual_abs": 0.28527194261550903, + "ronpo/target": -0.005375738255679607, + "ronpo/target_abs": 0.08742313086986542, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 725 + }, + { + "drift/chosen_abs": 0.30962568521499634, + "drift/rejected_abs": 0.09282008558511734, + "epoch": 0.2320222487087803, + "grad_norm": 1712.0, + "learning_rate": 2.649865579915976e-09, + "logits/chosen": -2.178410291671753, + "logits/rejected": -2.213578462600708, + "logps/chosen": -0.30459070205688477, + "logps/rejected": -0.2987954616546631, + "loss": 0.8561824560165405, + "loss/core": 0.8262929916381836, + "loss/preference_sft": 0.3033561408519745, + "loss/reference_anchor": 0.5674565434455872, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.0953526496887207, + "rewards/margins": 2.171635389328003, + "rewards/rejected": 0.9237171411514282, + "ronpo/logit": 0.21716353297233582, + "ronpo/residual": 0.20029985904693604, + "ronpo/residual_abs": 0.2601030468940735, + "ronpo/target": 0.016863666474819183, + "ronpo/target_abs": 0.07844550907611847, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 730 + }, + { + "drift/chosen_abs": 0.41004911065101624, + "drift/rejected_abs": 0.15294381976127625, + "epoch": 0.23361144219308702, + "grad_norm": 11.3125, + "learning_rate": 2.5024864971835508e-09, + "logits/chosen": -1.8713624477386475, + "logits/rejected": -1.9245052337646484, + "logps/chosen": -0.31250444054603577, + "logps/rejected": -0.2822606861591339, + "loss": 1.3640409708023071, + "loss/core": 1.3203461170196533, + "loss/preference_sft": 0.28716808557510376, + "loss/reference_anchor": 0.8451792597770691, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 4.098076343536377, + "rewards/margins": 2.5743887424468994, + "rewards/rejected": 1.5236876010894775, + "ronpo/logit": 0.25743886828422546, + "ronpo/residual": 0.2717561721801758, + "ronpo/residual_abs": 0.36912673711776733, + "ronpo/target": -0.014317306689918041, + "ronpo/target_abs": 0.08681128919124603, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 735 + }, + { + "drift/chosen_abs": 0.40702804923057556, + "drift/rejected_abs": 0.17471705377101898, + "epoch": 0.23520063567739372, + "grad_norm": 8.0625, + "learning_rate": 2.3588670704111995e-09, + "logits/chosen": -2.0632808208465576, + "logits/rejected": -2.0540781021118164, + "logps/chosen": -0.27535516023635864, + "logps/rejected": -0.27248817682266235, + "loss": 0.8561609983444214, + "loss/core": 0.8036273717880249, + "loss/preference_sft": 0.2769920825958252, + "loss/reference_anchor": 1.0229724645614624, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 4.0702805519104, + "rewards/margins": 2.323690414428711, + "rewards/rejected": 1.7465903759002686, + "ronpo/logit": 0.232369065284729, + "ronpo/residual": 0.21827132999897003, + "ronpo/residual_abs": 0.3096594512462616, + "ronpo/target": 0.014097737148404121, + "ronpo/target_abs": 0.09412308037281036, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 740 + }, + { + "drift/chosen_abs": 0.27846285700798035, + "drift/rejected_abs": 0.13982626795768738, + "epoch": 0.23678982916170044, + "grad_norm": 1.015625, + "learning_rate": 2.219061308991721e-09, + "logits/chosen": -1.9662797451019287, + "logits/rejected": -1.9410368204116821, + "logps/chosen": -0.31107696890830994, + "logps/rejected": -0.29558494687080383, + "loss": 0.4125809669494629, + "loss/core": 0.39409929513931274, + "loss/preference_sft": 0.3031148314476013, + "loss/reference_anchor": 0.3393229842185974, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.7828688621520996, + "rewards/margins": 1.401062250137329, + "rewards/rejected": 1.3818066120147705, + "ronpo/logit": 0.14010624587535858, + "ronpo/residual": 0.1506578028202057, + "ronpo/residual_abs": 0.22105178236961365, + "ronpo/target": -0.010551569052040577, + "ronpo/target_abs": 0.07253938913345337, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 745 + }, + { + "drift/chosen_abs": 0.493083655834198, + "drift/rejected_abs": 0.08905399590730667, + "epoch": 0.23837902264600716, + "grad_norm": 35.0, + "learning_rate": 2.0831217881543558e-09, + "logits/chosen": -1.8739421367645264, + "logits/rejected": -2.0240061283111572, + "logps/chosen": -0.32642191648483276, + "logps/rejected": -0.3034181594848633, + "loss": 1.7241404056549072, + "loss/core": 1.6729081869125366, + "loss/preference_sft": 0.30714109539985657, + "loss/reference_anchor": 0.9939321279525757, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.929997444152832, + "rewards/margins": 4.04317045211792, + "rewards/rejected": 0.8868273496627808, + "ronpo/logit": 0.40431705117225647, + "ronpo/residual": 0.39883852005004883, + "ronpo/residual_abs": 0.4577086567878723, + "ronpo/target": 0.00547849852591753, + "ronpo/target_abs": 0.09150009602308273, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 750 + }, + { + "drift/chosen_abs": 0.5546950101852417, + "drift/rejected_abs": 0.2534758150577545, + "epoch": 0.23996821613031386, + "grad_norm": 173.0, + "learning_rate": 1.951099629193366e-09, + "logits/chosen": -1.7793035507202148, + "logits/rejected": -1.708032250404358, + "logps/chosen": -0.26937034726142883, + "logps/rejected": -0.2777889668941498, + "loss": 1.7983328104019165, + "loss/core": 1.7351124286651611, + "loss/preference_sft": 0.264911413192749, + "loss/reference_anchor": 1.2379202842712402, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 5.546171188354492, + "rewards/margins": 3.0143425464630127, + "rewards/rejected": 2.5318284034729004, + "ronpo/logit": 0.3014342784881592, + "ronpo/residual": 0.2820184528827667, + "ronpo/residual_abs": 0.4880169928073883, + "ronpo/target": 0.019415846094489098, + "ronpo/target_abs": 0.08249348402023315, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 755 + }, + { + "drift/chosen_abs": 0.25574034452438354, + "drift/rejected_abs": 0.10070861876010895, + "epoch": 0.24155740961462058, + "grad_norm": 1.3828125, + "learning_rate": 1.823044480243431e-09, + "logits/chosen": -2.2029078006744385, + "logits/rejected": -2.2537450790405273, + "logps/chosen": -0.2618194818496704, + "logps/rejected": -0.26269036531448364, + "loss": 0.4067454934120178, + "loss/core": 0.3905092477798462, + "loss/preference_sft": 0.270187646150589, + "loss/reference_anchor": 0.2977067232131958, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.5571861267089844, + "rewards/margins": 1.5527894496917725, + "rewards/rejected": 1.004396677017212, + "ronpo/logit": 0.15527895092964172, + "ronpo/residual": 0.13664746284484863, + "ronpo/residual_abs": 0.21219086647033691, + "ronpo/target": 0.01863149181008339, + "ronpo/target_abs": 0.07967917621135712, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 760 + }, + { + "drift/chosen_abs": 0.16783472895622253, + "drift/rejected_abs": 0.09149236977100372, + "epoch": 0.2431466030989273, + "grad_norm": 1.359375, + "learning_rate": 1.699004497608994e-09, + "logits/chosen": -2.3972268104553223, + "logits/rejected": -2.3360962867736816, + "logps/chosen": -0.2857986092567444, + "logps/rejected": -0.2897695004940033, + "loss": 0.04794660210609436, + "loss/core": 0.04115251451730728, + "loss/preference_sft": 0.28739649057388306, + "loss/reference_anchor": 0.10714216530323029, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.677700400352478, + "rewards/margins": 0.7631028890609741, + "rewards/rejected": 0.9145975112915039, + "ronpo/logit": 0.07631029188632965, + "ronpo/residual": 0.07975935935974121, + "ronpo/residual_abs": 0.14109909534454346, + "ronpo/target": -0.0034490670077502728, + "ronpo/target_abs": 0.0860501304268837, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 765 + }, + { + "drift/chosen_abs": 0.3305698037147522, + "drift/rejected_abs": 0.10161750018596649, + "epoch": 0.24473579658323402, + "grad_norm": 57.0, + "learning_rate": 1.579026327654666e-09, + "logits/chosen": -1.9631493091583252, + "logits/rejected": -2.093764543533325, + "logps/chosen": -0.2555374801158905, + "logps/rejected": -0.27383852005004883, + "loss": 0.6095373630523682, + "loss/core": 0.5895863771438599, + "loss/preference_sft": 0.27055805921554565, + "loss/reference_anchor": 0.3719633221626282, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.3039238452911377, + "rewards/margins": 2.3924434185028076, + "rewards/rejected": 0.9114801287651062, + "ronpo/logit": 0.23924437165260315, + "ronpo/residual": 0.251014769077301, + "ronpo/residual_abs": 0.2923506796360016, + "ronpo/target": -0.011770381592214108, + "ronpo/target_abs": 0.08334879577159882, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 770 + }, + { + "drift/chosen_abs": 0.34415099024772644, + "drift/rejected_abs": 0.14330972731113434, + "epoch": 0.24632499006754072, + "grad_norm": 1.3828125, + "learning_rate": 1.4631550892634126e-09, + "logits/chosen": -1.82931387424469, + "logits/rejected": -1.8169645071029663, + "logps/chosen": -0.2950168550014496, + "logps/rejected": -0.2965241074562073, + "loss": 0.551666259765625, + "loss/core": 0.5294795036315918, + "loss/preference_sft": 0.2889275848865509, + "loss/reference_anchor": 0.4148431420326233, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.439793825149536, + "rewards/margins": 2.046867847442627, + "rewards/rejected": 1.3929258584976196, + "ronpo/logit": 0.20468676090240479, + "ronpo/residual": 0.21019069850444794, + "ronpo/residual_abs": 0.25614383816719055, + "ronpo/target": -0.00550392922013998, + "ronpo/target_abs": 0.09071238338947296, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 775 + }, + { + "drift/chosen_abs": 0.3496606945991516, + "drift/rejected_abs": 0.15519936382770538, + "epoch": 0.24791418355184744, + "grad_norm": 2.953125, + "learning_rate": 1.3514343568692133e-09, + "logits/chosen": -2.155691385269165, + "logits/rejected": -2.17978572845459, + "logps/chosen": -0.27056634426116943, + "logps/rejected": -0.2793372571468353, + "loss": 0.3566157817840576, + "loss/core": 0.33336490392684937, + "loss/preference_sft": 0.2702133059501648, + "loss/reference_anchor": 0.43799638748168945, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.4961299896240234, + "rewards/margins": 1.9453226327896118, + "rewards/rejected": 1.5508073568344116, + "ronpo/logit": 0.19453224539756775, + "ronpo/residual": 0.2001989632844925, + "ronpo/residual_abs": 0.2824886739253998, + "ronpo/target": -0.00566669087857008, + "ronpo/target_abs": 0.08824430406093597, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 780 + }, + { + "drift/chosen_abs": 0.47809037566185, + "drift/rejected_abs": 0.10961903631687164, + "epoch": 0.24950337703615416, + "grad_norm": 4.03125, + "learning_rate": 1.2439061440704723e-09, + "logits/chosen": -2.104382038116455, + "logits/rejected": -2.215803623199463, + "logps/chosen": -0.2707865536212921, + "logps/rejected": -0.26247674226760864, + "loss": 1.8099985122680664, + "loss/core": 1.7617250680923462, + "loss/preference_sft": 0.27354034781455994, + "loss/reference_anchor": 0.9381135106086731, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.774395942687988, + "rewards/margins": 3.7094340324401855, + "rewards/rejected": 1.0649611949920654, + "ronpo/logit": 0.37094348669052124, + "ronpo/residual": 0.37685176730155945, + "ronpo/residual_abs": 0.4302590489387512, + "ronpo/target": -0.005908310413360596, + "ronpo/target_abs": 0.08773339539766312, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 785 + }, + { + "drift/chosen_abs": 0.3598199188709259, + "drift/rejected_abs": 0.12054183334112167, + "epoch": 0.25109257052046086, + "grad_norm": 208.0, + "learning_rate": 1.1406108878304468e-09, + "logits/chosen": -2.0071957111358643, + "logits/rejected": -2.026601552963257, + "logps/chosen": -0.31191474199295044, + "logps/rejected": -0.29044240713119507, + "loss": 0.5209998488426208, + "loss/core": 0.4963608384132385, + "loss/preference_sft": 0.3023087680339813, + "loss/reference_anchor": 0.4625491201877594, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.5980522632598877, + "rewards/margins": 2.4207026958465576, + "rewards/rejected": 1.1773498058319092, + "ronpo/logit": 0.2420702427625656, + "ronpo/residual": 0.24213683605194092, + "ronpo/residual_abs": 0.30981531739234924, + "ronpo/target": -6.654262688243762e-05, + "ronpo/target_abs": 0.08093582093715668, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 790 + }, + { + "drift/chosen_abs": 0.2403639853000641, + "drift/rejected_abs": 0.11303351819515228, + "epoch": 0.2526817640047676, + "grad_norm": 1.0234375, + "learning_rate": 1.0415874332705382e-09, + "logits/chosen": -2.1333625316619873, + "logits/rejected": -2.168160915374756, + "logps/chosen": -0.25950658321380615, + "logps/rejected": -0.25287556648254395, + "loss": 0.1263783574104309, + "loss/core": 0.11855258047580719, + "loss/preference_sft": 0.24921035766601562, + "loss/reference_anchor": 0.13159438967704773, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.4028306007385254, + "rewards/margins": 1.274418592453003, + "rewards/rejected": 1.1284120082855225, + "ronpo/logit": 0.12744185328483582, + "ronpo/residual": 0.12694984674453735, + "ronpo/residual_abs": 0.19664797186851501, + "ronpo/target": 0.0004920096835121512, + "ronpo/target_abs": 0.08606652170419693, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 795 + }, + { + "drift/chosen_abs": 0.2836977541446686, + "drift/rejected_abs": 0.15890619158744812, + "epoch": 0.2542709574890743, + "grad_norm": 17.75, + "learning_rate": 9.468730190622484e-10, + "logits/chosen": -2.169442653656006, + "logits/rejected": -2.2410874366760254, + "logps/chosen": -0.26969194412231445, + "logps/rejected": -0.27040061354637146, + "loss": 0.22600746154785156, + "loss/core": 0.18485912680625916, + "loss/preference_sft": 0.26238492131233215, + "loss/reference_anchor": 0.7967280149459839, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.8366193771362305, + "rewards/margins": 1.2492666244506836, + "rewards/rejected": 1.587352991104126, + "ronpo/logit": 0.12492667138576508, + "ronpo/residual": 0.11071588844060898, + "ronpo/residual_abs": 0.17527928948402405, + "ronpo/target": 0.014210781082510948, + "ronpo/target_abs": 0.07547663152217865, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 800 + }, + { + "drift/chosen_abs": 0.3189254403114319, + "drift/rejected_abs": 0.1377745419740677, + "epoch": 0.255860150973381, + "grad_norm": 196.0, + "learning_rate": 8.565032634232195e-10, + "logits/chosen": -2.275631904602051, + "logits/rejected": -2.237766742706299, + "logps/chosen": -0.3036332428455353, + "logps/rejected": -0.30700454115867615, + "loss": 0.9040266275405884, + "loss/core": 0.8683737516403198, + "loss/preference_sft": 0.2961069643497467, + "loss/reference_anchor": 0.6834448575973511, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.1892545223236084, + "rewards/margins": 1.8560397624969482, + "rewards/rejected": 1.3332146406173706, + "ronpo/logit": 0.1856040060520172, + "ronpo/residual": 0.17766329646110535, + "ronpo/residual_abs": 0.3051329255104065, + "ronpo/target": 0.007940702140331268, + "ronpo/target_abs": 0.08157651126384735, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 805 + }, + { + "drift/chosen_abs": 0.21167591214179993, + "drift/rejected_abs": 0.11085629463195801, + "epoch": 0.25744934445768775, + "grad_norm": 7.65625, + "learning_rate": 7.705121507227019e-10, + "logits/chosen": -2.0630433559417725, + "logits/rejected": -2.1151208877563477, + "logps/chosen": -0.2830791175365448, + "logps/rejected": -0.28679680824279785, + "loss": 0.05285482853651047, + "loss/core": 0.046348024159669876, + "loss/preference_sft": 0.2829565405845642, + "loss/reference_anchor": 0.10184051841497421, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.114663600921631, + "rewards/margins": 1.0063090324401855, + "rewards/rejected": 1.1083545684814453, + "ronpo/logit": 0.10063091665506363, + "ronpo/residual": 0.09154792875051498, + "ronpo/residual_abs": 0.1593920886516571, + "ronpo/target": 0.009082979522645473, + "ronpo/target_abs": 0.08527089655399323, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 810 + }, + { + "drift/chosen_abs": 0.25946545600891113, + "drift/rejected_abs": 0.16305600106716156, + "epoch": 0.25903853794199444, + "grad_norm": 13.125, + "learning_rate": 6.889320187013991e-10, + "logits/chosen": -2.107985019683838, + "logits/rejected": -2.0296835899353027, + "logps/chosen": -0.26717063784599304, + "logps/rejected": -0.2779156267642975, + "loss": 0.23226383328437805, + "loss/core": 0.22093184292316437, + "loss/preference_sft": 0.26824456453323364, + "loss/reference_anchor": 0.19981497526168823, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.594414710998535, + "rewards/margins": 0.9783709645271301, + "rewards/rejected": 1.616044044494629, + "ronpo/logit": 0.09783709049224854, + "ronpo/residual": 0.07867446541786194, + "ronpo/residual_abs": 0.22327768802642822, + "ronpo/target": 0.019162625074386597, + "ronpo/target_abs": 0.07972420752048492, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 815 + }, + { + "drift/chosen_abs": 0.1991885006427765, + "drift/rejected_abs": 0.11371134221553802, + "epoch": 0.26062773142630113, + "grad_norm": 2.765625, + "learning_rate": 6.117935463105808e-10, + "logits/chosen": -2.0220797061920166, + "logits/rejected": -1.9778063297271729, + "logps/chosen": -0.2764715254306793, + "logps/rejected": -0.2888193130493164, + "loss": 0.07094263285398483, + "loss/core": 0.06167911738157272, + "loss/preference_sft": 0.2758772373199463, + "loss/reference_anchor": 0.1576823890209198, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.9880034923553467, + "rewards/margins": 0.851704478263855, + "rewards/rejected": 1.1362990140914917, + "ronpo/logit": 0.0851704329252243, + "ronpo/residual": 0.09892288595438004, + "ronpo/residual_abs": 0.15847130119800568, + "ronpo/target": -0.013752436265349388, + "ronpo/target_abs": 0.08235911279916763, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 820 + }, + { + "drift/chosen_abs": 0.288280189037323, + "drift/rejected_abs": 0.1897338479757309, + "epoch": 0.2622169249106079, + "grad_norm": 78.0, + "learning_rate": 5.391257421749826e-10, + "logits/chosen": -2.2123422622680664, + "logits/rejected": -2.1602697372436523, + "logps/chosen": -0.2984291613101959, + "logps/rejected": -0.3011440932750702, + "loss": 0.3465138375759125, + "loss/core": 0.3028988242149353, + "loss/preference_sft": 0.2975984513759613, + "loss/reference_anchor": 0.8425408601760864, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.8811376094818115, + "rewards/margins": 0.9950310587882996, + "rewards/rejected": 1.8861068487167358, + "ronpo/logit": 0.09950310736894608, + "ronpo/residual": 0.09332647174596786, + "ronpo/residual_abs": 0.21904487907886505, + "ronpo/target": 0.006176639813929796, + "ronpo/target_abs": 0.0774788185954094, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 825 + }, + { + "drift/chosen_abs": 0.1840457022190094, + "drift/rejected_abs": 0.11357797682285309, + "epoch": 0.2638061183949146, + "grad_norm": 34.75, + "learning_rate": 4.709559336838461e-10, + "logits/chosen": -2.2351510524749756, + "logits/rejected": -2.236767292022705, + "logps/chosen": -0.27836111187934875, + "logps/rejected": -0.2729012370109558, + "loss": 0.13388831913471222, + "loss/core": 0.12568429112434387, + "loss/preference_sft": 0.2821260094642639, + "loss/reference_anchor": 0.1358683705329895, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8400017023086548, + "rewards/margins": 0.7065569758415222, + "rewards/rejected": 1.1334445476531982, + "ronpo/logit": 0.0706556960940361, + "ronpo/residual": 0.05800905078649521, + "ronpo/residual_abs": 0.18142782151699066, + "ronpo/target": 0.012646635994315147, + "ronpo/target_abs": 0.09177650511264801, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 830 + }, + { + "drift/chosen_abs": 0.4068368971347809, + "drift/rejected_abs": 0.1611480414867401, + "epoch": 0.2653953118792213, + "grad_norm": 24.625, + "learning_rate": 4.0730975671420253e-10, + "logits/chosen": -2.1904501914978027, + "logits/rejected": -2.1559247970581055, + "logps/chosen": -0.242096945643425, + "logps/rejected": -0.2459816187620163, + "loss": 1.0775315761566162, + "loss/core": 1.0418219566345215, + "loss/preference_sft": 0.24760499596595764, + "loss/reference_anchor": 0.6894329786300659, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 4.068270683288574, + "rewards/margins": 2.4569103717803955, + "rewards/rejected": 1.6113605499267578, + "ronpo/logit": 0.24569101631641388, + "ronpo/residual": 0.24678480625152588, + "ronpo/residual_abs": 0.35724449157714844, + "ronpo/target": -0.001093805069103837, + "ronpo/target_abs": 0.08144358545541763, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 835 + }, + { + "drift/chosen_abs": 0.2863476276397705, + "drift/rejected_abs": 0.13716426491737366, + "epoch": 0.266984505363528, + "grad_norm": 0.734375, + "learning_rate": 3.482111459902695e-10, + "logits/chosen": -2.364701986312866, + "logits/rejected": -2.425690174102783, + "logps/chosen": -0.2671613395214081, + "logps/rejected": -0.264343798160553, + "loss": 0.348602831363678, + "loss/core": 0.32751885056495667, + "loss/preference_sft": 0.27554088830947876, + "loss/reference_anchor": 0.3941258490085602, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.8634347915649414, + "rewards/margins": 1.4927136898040771, + "rewards/rejected": 1.3707211017608643, + "ronpo/logit": 0.1492713987827301, + "ronpo/residual": 0.15882544219493866, + "ronpo/residual_abs": 0.20727181434631348, + "ronpo/target": -0.009554052725434303, + "ronpo/target_abs": 0.09001000225543976, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 840 + }, + { + "drift/chosen_abs": 0.2981222867965698, + "drift/rejected_abs": 0.18478380143642426, + "epoch": 0.2685736988478347, + "grad_norm": 6.46875, + "learning_rate": 2.9368232608258796e-10, + "logits/chosen": -2.143329381942749, + "logits/rejected": -2.1359808444976807, + "logps/chosen": -0.24788551032543182, + "logps/rejected": -0.23464611172676086, + "loss": 0.09194006025791168, + "loss/core": 0.07542302459478378, + "loss/preference_sft": 0.24129502475261688, + "loss/reference_anchor": 0.30621108412742615, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.979228973388672, + "rewards/margins": 1.13218092918396, + "rewards/rejected": 1.8470484018325806, + "ronpo/logit": 0.11321810632944107, + "ronpo/residual": 0.09495650231838226, + "ronpo/residual_abs": 0.16129747033119202, + "ronpo/target": 0.018261589109897614, + "ronpo/target_abs": 0.08132471889257431, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 845 + }, + { + "drift/chosen_abs": 0.41143712401390076, + "drift/rejected_abs": 0.14703157544136047, + "epoch": 0.2701628923321414, + "grad_norm": 80.0, + "learning_rate": 2.4374380305025865e-10, + "logits/chosen": -1.953258752822876, + "logits/rejected": -1.88783860206604, + "logps/chosen": -0.27198326587677, + "logps/rejected": -0.28331276774406433, + "loss": 0.8986851572990417, + "loss/core": 0.8608425259590149, + "loss/preference_sft": 0.2750104069709778, + "loss/reference_anchor": 0.7293528318405151, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 4.112533092498779, + "rewards/margins": 2.645296096801758, + "rewards/rejected": 1.4672367572784424, + "ronpo/logit": 0.26452964544296265, + "ronpo/residual": 0.25969162583351135, + "ronpo/residual_abs": 0.3339794874191284, + "ronpo/target": 0.004837975837290287, + "ronpo/target_abs": 0.08931146562099457, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 850 + }, + { + "drift/chosen_abs": 0.20746512711048126, + "drift/rejected_abs": 0.07350940257310867, + "epoch": 0.27175208581644816, + "grad_norm": 2.96875, + "learning_rate": 1.9841435672945938e-10, + "logits/chosen": -2.3771004676818848, + "logits/rejected": -2.3643927574157715, + "logps/chosen": -0.27676576375961304, + "logps/rejected": -0.2748029828071594, + "loss": 0.4189775586128235, + "loss/core": 0.40414971113204956, + "loss/preference_sft": 0.2725796401500702, + "loss/reference_anchor": 0.2692995071411133, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.0746512413024902, + "rewards/margins": 1.339883804321289, + "rewards/rejected": 0.7347675561904907, + "ronpo/logit": 0.13398835062980652, + "ronpo/residual": 0.13738013803958893, + "ronpo/residual_abs": 0.1904793679714203, + "ronpo/target": -0.003391760867089033, + "ronpo/target_abs": 0.0801364928483963, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 855 + }, + { + "drift/chosen_abs": 0.18808570504188538, + "drift/rejected_abs": 0.10459624230861664, + "epoch": 0.27334127930075486, + "grad_norm": 42.25, + "learning_rate": 1.577110336711096e-10, + "logits/chosen": -2.201777935028076, + "logits/rejected": -2.127864360809326, + "logps/chosen": -0.28073182702064514, + "logps/rejected": -0.2788235545158386, + "loss": 0.12705744802951813, + "loss/core": 0.11867976188659668, + "loss/preference_sft": 0.27348923683166504, + "loss/reference_anchor": 0.14020445942878723, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.8808481693267822, + "rewards/margins": 0.8413216471672058, + "rewards/rejected": 1.0395262241363525, + "ronpo/logit": 0.08413217961788177, + "ronpo/residual": 0.07758622616529465, + "ronpo/residual_abs": 0.14888854324817657, + "ronpo/target": 0.006545948330312967, + "ronpo/target_abs": 0.07704494148492813, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 860 + }, + { + "drift/chosen_abs": 0.24855653941631317, + "drift/rejected_abs": 0.12981536984443665, + "epoch": 0.2749304727850616, + "grad_norm": 0.859375, + "learning_rate": 1.2164914073037048e-10, + "logits/chosen": -2.090097427368164, + "logits/rejected": -2.0350818634033203, + "logps/chosen": -0.2797498106956482, + "logps/rejected": -0.2887721359729767, + "loss": 0.11714392900466919, + "loss/core": 0.10699626058340073, + "loss/preference_sft": 0.28749600052833557, + "loss/reference_anchor": 0.1742037832736969, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.485067367553711, + "rewards/margins": 1.1879299879074097, + "rewards/rejected": 1.2971374988555908, + "ronpo/logit": 0.11879298835992813, + "ronpo/residual": 0.11563484370708466, + "ronpo/residual_abs": 0.19152960181236267, + "ronpo/target": 0.003158140927553177, + "ronpo/target_abs": 0.08096446096897125, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 865 + }, + { + "drift/chosen_abs": 0.4628352224826813, + "drift/rejected_abs": 0.22026996314525604, + "epoch": 0.2765196662693683, + "grad_norm": 520.0, + "learning_rate": 9.024223931035357e-11, + "logits/chosen": -2.098940849304199, + "logits/rejected": -2.087498188018799, + "logps/chosen": -0.26976698637008667, + "logps/rejected": -0.2759985327720642, + "loss": 0.723773717880249, + "loss/core": 0.6722049117088318, + "loss/preference_sft": 0.2798956036567688, + "loss/reference_anchor": 1.0033875703811646, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 4.626898765563965, + "rewards/margins": 2.4265968799591064, + "rewards/rejected": 2.2003021240234375, + "ronpo/logit": 0.24265968799591064, + "ronpo/residual": 0.24575605988502502, + "ronpo/residual_abs": 0.30909624695777893, + "ronpo/target": -0.0030963898170739412, + "ronpo/target_abs": 0.08388674259185791, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 870 + }, + { + "drift/chosen_abs": 0.3066100478172302, + "drift/rejected_abs": 0.15433122217655182, + "epoch": 0.278108859753675, + "grad_norm": 5.90625, + "learning_rate": 6.350214026223516e-11, + "logits/chosen": -1.945692777633667, + "logits/rejected": -1.9127715826034546, + "logps/chosen": -0.2703152298927307, + "logps/rejected": -0.27156907320022583, + "loss": 0.22089354693889618, + "loss/core": 0.20463630557060242, + "loss/preference_sft": 0.2696455121040344, + "loss/reference_anchor": 0.29818084836006165, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.064970016479492, + "rewards/margins": 1.5222326517105103, + "rewards/rejected": 1.5427372455596924, + "ronpo/logit": 0.15222327411174774, + "ronpo/residual": 0.1579734981060028, + "ronpo/residual_abs": 0.20636561512947083, + "ronpo/target": -0.005750220734626055, + "ronpo/target_abs": 0.07495192438364029, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 875 + }, + { + "drift/chosen_abs": 0.24454915523529053, + "drift/rejected_abs": 0.09118019789457321, + "epoch": 0.27969805323798175, + "grad_norm": 6.96875, + "learning_rate": 4.143889944367429e-11, + "logits/chosen": -2.112548828125, + "logits/rejected": -2.1549744606018066, + "logps/chosen": -0.2780202031135559, + "logps/rejected": -0.29079365730285645, + "loss": 0.41406092047691345, + "loss/core": 0.40033167600631714, + "loss/preference_sft": 0.28780096769332886, + "loss/reference_anchor": 0.24580499529838562, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.4454915523529053, + "rewards/margins": 1.5506150722503662, + "rewards/rejected": 0.8948766589164734, + "ronpo/logit": 0.1550614982843399, + "ronpo/residual": 0.16445958614349365, + "ronpo/residual_abs": 0.21083009243011475, + "ronpo/target": -0.009398087859153748, + "ronpo/target_abs": 0.0880463570356369, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 880 + }, + { + "drift/chosen_abs": 0.2654206454753876, + "drift/rejected_abs": 0.10342685878276825, + "epoch": 0.28128724672228844, + "grad_norm": 6.1875, + "learning_rate": 2.4060813937225312e-11, + "logits/chosen": -2.159634828567505, + "logits/rejected": -2.221071243286133, + "logps/chosen": -0.2973093092441559, + "logps/rejected": -0.29612240195274353, + "loss": 0.4108871519565582, + "loss/core": 0.38317131996154785, + "loss/preference_sft": 0.2979774475097656, + "loss/reference_anchor": 0.5245189666748047, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.6539218425750732, + "rewards/margins": 1.6225885152816772, + "rewards/rejected": 1.0313332080841064, + "ronpo/logit": 0.16225887835025787, + "ronpo/residual": 0.16746975481510162, + "ronpo/residual_abs": 0.21311867237091064, + "ronpo/target": -0.005210875067859888, + "ronpo/target_abs": 0.08115576207637787, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 885 + }, + { + "drift/chosen_abs": 0.29945269227027893, + "drift/rejected_abs": 0.12191145122051239, + "epoch": 0.28287644020659514, + "grad_norm": 3.625, + "learning_rate": 1.1374418930135132e-11, + "logits/chosen": -2.3053576946258545, + "logits/rejected": -2.2626731395721436, + "logps/chosen": -0.2816936671733856, + "logps/rejected": -0.2805249094963074, + "loss": 0.4204486906528473, + "loss/core": 0.39468875527381897, + "loss/preference_sft": 0.28686416149139404, + "loss/reference_anchor": 0.4865127503871918, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.9918599128723145, + "rewards/margins": 1.7764670848846436, + "rewards/rejected": 1.21539306640625, + "ronpo/logit": 0.1776467114686966, + "ronpo/residual": 0.1912662535905838, + "ronpo/residual_abs": 0.23552899062633514, + "ronpo/target": -0.013619527220726013, + "ronpo/target_abs": 0.08702584356069565, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 890 + }, + { + "drift/chosen_abs": 0.30740076303482056, + "drift/rejected_abs": 0.12458650022745132, + "epoch": 0.2844656336909019, + "grad_norm": 11.375, + "learning_rate": 3.3844852567285754e-12, + "logits/chosen": -2.081117630004883, + "logits/rejected": -2.069899559020996, + "logps/chosen": -0.2847122251987457, + "logps/rejected": -0.2946607172489166, + "loss": 0.373470276594162, + "loss/core": 0.3527851700782776, + "loss/preference_sft": 0.28093528747558594, + "loss/reference_anchor": 0.3856084942817688, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.071903705596924, + "rewards/margins": 1.9304262399673462, + "rewards/rejected": 1.1414774656295776, + "ronpo/logit": 0.19304262101650238, + "ronpo/residual": 0.18837428092956543, + "ronpo/residual_abs": 0.23437511920928955, + "ronpo/target": 0.004668330308049917, + "ronpo/target_abs": 0.07740633934736252, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 895 + }, + { + "drift/chosen_abs": 0.15997305512428284, + "drift/rejected_abs": 0.08303854614496231, + "epoch": 0.2860548271752086, + "grad_norm": 15.25, + "learning_rate": 9.401760429905703e-14, + "logits/chosen": -2.103471279144287, + "logits/rejected": -2.086383819580078, + "logps/chosen": -0.3180239498615265, + "logps/rejected": -0.3187870681285858, + "loss": 0.14068026840686798, + "loss/core": 0.13307660818099976, + "loss/preference_sft": 0.3193908929824829, + "loss/reference_anchor": 0.12013395130634308, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.5992281436920166, + "rewards/margins": 0.8927779197692871, + "rewards/rejected": 0.7064501643180847, + "ronpo/logit": 0.08927779644727707, + "ronpo/residual": 0.08004127442836761, + "ronpo/residual_abs": 0.140147402882576, + "ronpo/target": 0.009236516430974007, + "ronpo/target_abs": 0.06991586834192276, + "ronpo/weight": 1.0, + "ronpo/weight_max": 1.0, + "step": 900 + }, + { + "epoch": 0.2860548271752086, + "step": 900, + "total_flos": 0.0, + "train_loss": 0.42856775469250147, + "train_runtime": 7333.3683, + "train_samples_per_second": 1.964, + "train_steps_per_second": 0.123 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..b6925c1 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:422d96819beecedd3c6d702e6a229dcb27a19a5efa6ee3a5da929382db6990fd +size 7057