From 0738f2ca0a204007eda117fcfcaf045321f5f7c6 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Thu, 23 Jul 2026 13:58:09 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: promotion/qwen3-8b-aaai27-flagship-ht-mnpo-helpfulness-s42 Source: Original Platform --- .gitattributes | 36 + README.md | 24 + all_results.json | 9 + chat_template.jinja | 89 + config.json | 71 + config.yaml | 58 + generation_config.json | 12 + job_status.json | 23 + model.safetensors | 3 + objective_protocol.json | 69 + provenance.json | 14 + resource_profile.json | 21 + run_status.json | 15 + split_manifest.json | 23 + stability_gate.json | 38 + tokenizer.json | 3 + tokenizer_config.json | 30 + train_results.json | 9 + trainer_state.json | 4543 +++++++++++++++++++++++++++++++++++++++ training_args.bin | 3 + 20 files changed, 5093 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 config.yaml create mode 100644 generation_config.json create mode 100644 job_status.json create mode 100644 model.safetensors create mode 100644 objective_protocol.json create mode 100644 provenance.json create mode 100644 resource_profile.json create mode 100644 run_status.json create mode 100644 split_manifest.json create mode 100644 stability_gate.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..703a381 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +--- +library_name: transformers +base_model: Qwen/Qwen3-8B +tags: +- ronpo +- mnpo +- preference-optimization +--- + +# qwen3-8b-aaai27-flagship-ht-mnpo-helpfulness-s42 + +Research checkpoint for the RONPO AAAI revision experiments. + +- Method: ht_mnpo_helpfulness +- Base model: `Qwen/Qwen3-8B` +- Seed: 42 +- Local source at upload time: `/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/full/ht_mnpo_helpfulness/seed42/attempt2` +- Uploaded at UTC: 2026-07-13T13:46:47Z +- Run status metadata: `{"attempt": 2, "effective_batch_size": 16, "method": "ht_mnpo_helpfulness", "objective_protocol": "objective_protocol.json", "optimizer_steps": 900, "resource_profile": "resource_profile.json", "scope_amendment": "scope_amendment_20260713.json", "seed": 42, "split_manifest": "split_manifest.json", "stability_gate": "stability_gate.json", "status": "completed", "wandb_run_id": "50c5aa55b281", "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/50c5aa55b281"}` + +AAAI-27 P1 matched budget: 900 optimizer steps, effective batch 16; passed non-thinking and collapse stability gates; sealed-test metrics are not used for checkpoint selection. + +Intended use: reproducibility and evaluation for the RONPO paper. This +checkpoint is not intended as a production assistant. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..91bda8f --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8600609209819029, + "total_flos": 0.0, + "train_loss": 0.3323093075222439, + "train_runtime": 7030.1668, + "train_samples": 16743, + "train_samples_per_second": 2.048, + "train_steps_per_second": 0.128 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..88b9caa --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +model_name_or_path: Qwen/Qwen3-8B +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/ht_mnpo_helpfulness: 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: ht_mnpo +eta: 0.0075 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.02 +preference_sft_weight: 0.002 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 2.5e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 42 +gradient_accumulation_steps: 16 +gradient_checkpointing: true +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 1 +per_device_eval_batch_size: 1 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/full/ht_mnpo_helpfulness/seed42/attempt2 +run_name: aaai27-flagship-full-ht_mnpo_helpfulness-s42-a2 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/job_status.json b/job_status.json new file mode 100644 index 0000000..c799268 --- /dev/null +++ b/job_status.json @@ -0,0 +1,23 @@ +{ + "status": "completed", + "stage": "full", + "method": "ht_mnpo_helpfulness", + "seed": 42, + "attempt": 2, + "gpu": 2, + "gpus": [ + 2 + ], + "returncode": 0, + "finite_metrics": true, + "oom": false, + "traceback": false, + "reason": "passed", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "50c5aa55b281", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/50c5aa55b281", + "output_dir": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/full/ht_mnpo_helpfulness/seed42/attempt2", + "log": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/logs/full_ht_mnpo_helpfulness_s42_a2.log", + "completed_at": "2026-07-13T13:44:08Z" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..db26665 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79427aa8391b73fd51103cd4dc66d235ba943c299713cd5a674633a80f24130c +size 16381517208 diff --git a/objective_protocol.json b/objective_protocol.json new file mode 100644 index 0000000..35b3b44 --- /dev/null +++ b/objective_protocol.json @@ -0,0 +1,69 @@ +{ + "schema_version": 1, + "frozen_at_kst": "2026-07-12T21:40:00+09:00", + "base_model": "Qwen/Qwen3-8B", + "primary_triple": { + "helpfulness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "ultrafeedback-helpfulness", + "transform": "identity" + }, + "safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "conciseness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "helpsteer-verbosity", + "transform": "negate" + } + }, + "gate_thresholds": { + "raw_score_std_strictly_greater_than": 0.01, + "aggregate_prompt_pair_spearman_median_strictly_less_than": 0.0, + "aggregate_pairwise_top1_mismatch_at_least": 0.5 + }, + "gate_pool": "source-train Qwen3-8B response pool split into deterministic train/validation; sealed source-test excluded", + "training_seeds": [42, 43, 44], + "optimizer_steps": 900, + "effective_batch_size": 16, + "decode": { + "enable_thinking": false, + "temperature": 0.7, + "top_p": 0.9, + "max_new_tokens": 2048, + "decode_seed": 42 + }, + "methods": [ + "ronpo_full_expect", + "ronpo_k_only", + "dpo", + "ipo", + "simpo", + "kto", + "sppo_avg", + "inpo_avg", + "ht_mnpo_helpfulness", + "ht_mnpo_safety", + "ht_mnpo_conciseness" + ], + "selection_policy": "No sealed-test selection. One preregistered configuration per method; failures use only the declared stability retry recipe, at most three attempts.", + "stability_gate": { + "validation_prompts": 128, + "min_candidate_base_mean_word_ratio": 0.33, + "max_candidate_base_mean_word_ratio": 2.0, + "max_consecutive_identical_word_run": 20, + "think_tag_count": 0, + "empty_response_count": 0, + "retry_recipe": "attempt 2 halves learning rate, enables gradient checkpointing, and adds reference=0.02/SFT=0.002 anchoring to unanchored unified losses; attempt 3 quarters learning rate and uses reference=0.05/SFT=0.005; RONPO uses reference=0.05/SFT=0.005 from attempt 1" + }, + "primary_metric": "mean prompt-level minimum of per-prompt min-max-normalized helpfulness, safety, conciseness", + "bootstrap": { + "unit": "prompt", + "paired": true, + "resamples": 2000, + "interval": "percentile_95", + "seed": 42 + } +} diff --git a/provenance.json b/provenance.json new file mode 100644 index 0000000..e549516 --- /dev/null +++ b/provenance.json @@ -0,0 +1,14 @@ +{ + "method": "ht_mnpo_helpfulness", + "seed": 42, + "attempt": 2, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "50c5aa55b281", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/50c5aa55b281", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json", + "scope_amendment": "scope_amendment_20260713.json" +} diff --git a/resource_profile.json b/resource_profile.json new file mode 100644 index 0000000..18eec47 --- /dev/null +++ b/resource_profile.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "measured_at_kst": "2026-07-12T22:38:00+09:00", + "hardware": "NVIDIA B200 183359 MiB reported total; 178.35 GiB PyTorch-visible capacity", + "smoke_evidence": { + "attempted_profile": {"per_device_batch_size": 2, "gradient_accumulation_steps": 8, "gradient_checkpointing": false, "effective_batch_size": 16}, + "outcome": "OOM before completing the 20-step smoke across every launched unified baseline; representative process used 176.87 GiB and failed a 1.62 GiB allocation. KTO also OOMed at the optimizer step.", + "paper_metric": false + }, + "selected_full_training_profile": { + "per_device_batch_size": 2, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": false, + "attention_implementation": "PyTorch SDPA with cuDNN SDPA disabled", + "effective_batch_size": 16, + "applies_symmetrically_to_all_unified_pairwise_methods": true, + "kto_implementation_constraint": "TRL KTO requires actual batch size > 1. KTO uses batch 2 / accumulation 8 and identical AdamW with foreach=False. PyTorch SDPA is used with cuDNN SDPA disabled. The fixed base reference model is placed on a second B200 and its exact logits are returned to the policy device; the policy/optimizer remain on the first B200. This avoids a measured ~6-hour reference-precompute path while leaving the KTO loss, actual/effective batch, optimizer, and 900-step budget unchanged." + }, + "retry_semantics": "Resource selection is separated from method-stability attempts. Full attempt 1 uses the smoke-validated SDPA batch-2 profile and retains the preregistered method LR/anchor; attempts 2 and 3 fall back to batch 1 / accumulation 16 / gradient checkpointing and apply the declared LR/anchor stabilization recipe.", + "provenance_note": "No reward or sealed-evaluation result was available when this hardware profile was selected." +} diff --git a/run_status.json b/run_status.json new file mode 100644 index 0000000..b72d383 --- /dev/null +++ b/run_status.json @@ -0,0 +1,15 @@ +{ + "method": "ht_mnpo_helpfulness", + "seed": 42, + "attempt": 2, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "50c5aa55b281", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/50c5aa55b281", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json", + "scope_amendment": "scope_amendment_20260713.json", + "status": "completed" +} diff --git a/split_manifest.json b/split_manifest.json new file mode 100644 index 0000000..4fb96bd --- /dev/null +++ b/split_manifest.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "split_rule": "sort source-train prompts by sha256('flagship-v1||'+prompt_id); first round(10%) validation", + "validation_fraction": 0.1, + "source_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/avg_oracle", + "base_model": "Qwen/Qwen3-8B", + "counts": { + "train": 16780, + "validation": 1864, + "sealed_test": 604 + }, + "prompt_id_sha256": { + "train": "fcf6f2ea5a0b7bcd41179e6f5288f4d850b2d04bdb729da91907bd18bf733a69", + "validation": "a87ba6f519192164daa487b109a6547dd8cbde46a27a373d6da201ca2d061b8f", + "sealed_test": "b398ced6af6cf73d731e1a5f32a0631d8060323d51eda1e375695566bf5f1867" + }, + "overlaps": { + "train_validation": 0, + "train_sealed_test": 0, + "validation_sealed_test": 0 + }, + "sealed_test_policy": "prompts only until S4a; no objective scores or model generations used in S0-S3" +} diff --git a/stability_gate.json b/stability_gate.json new file mode 100644 index 0000000..374d7cd --- /dev/null +++ b/stability_gate.json @@ -0,0 +1,38 @@ +{ + "status": "passed", + "passed": true, + "base": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 388.1484375, + "max_words": 1379, + "max_repeat_run": 4 + }, + "candidate": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 384.6171875, + "max_words": 1280, + "max_repeat_run": 4 + }, + "candidate_base_mean_word_ratio": 0.9909023207133225, + "thresholds": { + "min_length_ratio": 0.33, + "max_length_ratio": 2.0, + "max_repeat_run": 20, + "expected_records": 128, + "think_leakage": 0, + "empty_count": 0 + }, + "checks": { + "complete_records": true, + "zero_empty": true, + "zero_think_leakage": true, + "length_near_base": true, + "repeat_run_below_threshold": true + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ea3c9e2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..91bda8f --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8600609209819029, + "total_flos": 0.0, + "train_loss": 0.3323093075222439, + "train_runtime": 7030.1668, + "train_samples": 16743, + "train_samples_per_second": 2.048, + "train_steps_per_second": 0.128 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..5dcd276 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,4543 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8600609209819029, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "drift/chosen_abs": 0.2093760073184967, + "drift/rejected_abs": 0.1779964119195938, + "epoch": 0.004778116227677238, + "grad_norm": 608.0, + "ht_mnpo/logit": 0.03132759779691696, + "ht_mnpo/residual": 0.02382758818566799, + "ht_mnpo/residual_abs": 0.24814370274543762, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.111111111111111e-08, + "logits/chosen": -2.080493211746216, + "logits/rejected": -1.8775554895401, + "logps/chosen": -0.31729820370674133, + "logps/rejected": -0.32572638988494873, + "loss": 0.8218833208084106, + "loss/core": 0.8120266199111938, + "loss/preference_sft": 0.31729820370674133, + "loss/reference_anchor": 0.46110573410987854, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.0926239490509033, + "rewards/margins": 0.3132759928703308, + "rewards/rejected": 1.7793481349945068, + "step": 5 + }, + { + "drift/chosen_abs": 0.26713913679122925, + "drift/rejected_abs": 0.12592381238937378, + "epoch": 0.009556232455354476, + "grad_norm": 5.5625, + "ht_mnpo/logit": 0.14135970175266266, + "ht_mnpo/residual": 0.13385972380638123, + "ht_mnpo/residual_abs": 0.14800478518009186, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.5e-08, + "logits/chosen": -2.263683557510376, + "logits/rejected": -2.1684491634368896, + "logps/chosen": -0.30673515796661377, + "logps/rejected": -0.3016442656517029, + "loss": 0.32831090688705444, + "loss/core": 0.31749722361564636, + "loss/preference_sft": 0.30673515796661377, + "loss/reference_anchor": 0.5100095868110657, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.6701548099517822, + "rewards/margins": 1.4135972261428833, + "rewards/rejected": 1.2565577030181885, + "step": 10 + }, + { + "drift/chosen_abs": 0.22741301357746124, + "drift/rejected_abs": 0.07075328379869461, + "epoch": 0.014334348683031715, + "grad_norm": 1616.0, + "ht_mnpo/logit": 0.1659075915813446, + "ht_mnpo/residual": 0.15840759873390198, + "ht_mnpo/residual_abs": 0.16668769717216492, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.888888888888889e-08, + "logits/chosen": -2.327167272567749, + "logits/rejected": -2.2666492462158203, + "logps/chosen": -0.30080440640449524, + "logps/rejected": -0.28799667954444885, + "loss": 0.6700844764709473, + "loss/core": 0.6620814204216003, + "loss/preference_sft": 0.30080440640449524, + "loss/reference_anchor": 0.3700721263885498, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.2735326290130615, + "rewards/margins": 1.6590759754180908, + "rewards/rejected": 0.6144567131996155, + "step": 15 + }, + { + "drift/chosen_abs": 0.236248180270195, + "drift/rejected_abs": 0.2190016508102417, + "epoch": 0.019112464910708952, + "grad_norm": 176.0, + "ht_mnpo/logit": 0.01763411983847618, + "ht_mnpo/residual": 0.01013412605971098, + "ht_mnpo/residual_abs": 0.1537032574415207, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.2777777777777776e-08, + "logits/chosen": -2.3429882526397705, + "logits/rejected": -2.1697194576263428, + "logps/chosen": -0.2879870533943176, + "logps/rejected": -0.30968016386032104, + "loss": 0.3623378872871399, + "loss/core": 0.3473840057849884, + "loss/preference_sft": 0.2879870533943176, + "loss/reference_anchor": 0.7188963890075684, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.3624815940856934, + "rewards/margins": 0.17634114623069763, + "rewards/rejected": 2.186140537261963, + "step": 20 + }, + { + "drift/chosen_abs": 0.29826819896698, + "drift/rejected_abs": 0.179667666554451, + "epoch": 0.023890581138386192, + "grad_norm": 6.3125, + "ht_mnpo/logit": 0.11871922016143799, + "ht_mnpo/residual": 0.11121921241283417, + "ht_mnpo/residual_abs": 0.1725720465183258, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.666666666666667e-08, + "logits/chosen": -2.0951151847839355, + "logits/rejected": -1.8950538635253906, + "logps/chosen": -0.29473376274108887, + "logps/rejected": -0.30570217967033386, + "loss": 0.32861581444740295, + "loss/core": 0.3204094469547272, + "loss/preference_sft": 0.29473376274108887, + "loss/reference_anchor": 0.3808436989784241, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.981147050857544, + "rewards/margins": 1.1871918439865112, + "rewards/rejected": 1.7939550876617432, + "step": 25 + }, + { + "drift/chosen_abs": 0.15562030673027039, + "drift/rejected_abs": 0.09274661540985107, + "epoch": 0.02866869736606343, + "grad_norm": 18.0, + "ht_mnpo/logit": 0.06269876658916473, + "ht_mnpo/residual": 0.05519877001643181, + "ht_mnpo/residual_abs": 0.08922450989484787, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.055555555555555e-08, + "logits/chosen": -2.2576093673706055, + "logits/rejected": -2.082545042037964, + "logps/chosen": -0.29427456855773926, + "logps/rejected": -0.2952222228050232, + "loss": 0.07133828848600388, + "loss/core": 0.06886543333530426, + "loss/preference_sft": 0.29427456855773926, + "loss/reference_anchor": 0.09421543031930923, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.5543291568756104, + "rewards/margins": 0.6269876956939697, + "rewards/rejected": 0.9273414611816406, + "step": 30 + }, + { + "drift/chosen_abs": 0.27138715982437134, + "drift/rejected_abs": 0.3082945942878723, + "epoch": 0.03344681359374067, + "grad_norm": 58.0, + "ht_mnpo/logit": -0.03644378110766411, + "ht_mnpo/residual": -0.043943800032138824, + "ht_mnpo/residual_abs": 0.25199875235557556, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.444444444444444e-08, + "logits/chosen": -2.179772138595581, + "logits/rejected": -1.8775641918182373, + "logps/chosen": -0.2725084125995636, + "logps/rejected": -0.30440372228622437, + "loss": 0.5291619300842285, + "loss/core": 0.5189464092254639, + "loss/preference_sft": 0.2725084125995636, + "loss/reference_anchor": 0.4835292398929596, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 2.712721824645996, + "rewards/margins": -0.3644378185272217, + "rewards/rejected": 3.0771596431732178, + "step": 35 + }, + { + "drift/chosen_abs": 0.25635233521461487, + "drift/rejected_abs": 0.1467134654521942, + "epoch": 0.038224929821417904, + "grad_norm": 1.7421875, + "ht_mnpo/logit": 0.12074834108352661, + "ht_mnpo/residual": 0.11324834823608398, + "ht_mnpo/residual_abs": 0.17528986930847168, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0833333333333334e-07, + "logits/chosen": -2.024980068206787, + "logits/rejected": -1.7612985372543335, + "logps/chosen": -0.2727901041507721, + "logps/rejected": -0.27953046560287476, + "loss": 0.46636730432510376, + "loss/core": 0.4604361057281494, + "loss/preference_sft": 0.2727901041507721, + "loss/reference_anchor": 0.2692839503288269, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.563462495803833, + "rewards/margins": 1.2074834108352661, + "rewards/rejected": 1.3559788465499878, + "step": 40 + }, + { + "drift/chosen_abs": 0.3089280426502228, + "drift/rejected_abs": 0.1763952225446701, + "epoch": 0.04300304604909514, + "grad_norm": 0.353515625, + "ht_mnpo/logit": 0.13275332748889923, + "ht_mnpo/residual": 0.1252533495426178, + "ht_mnpo/residual_abs": 0.17680028080940247, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.2222222222222222e-07, + "logits/chosen": -2.1491565704345703, + "logits/rejected": -1.917433738708496, + "logps/chosen": -0.28032320737838745, + "logps/rejected": -0.28377169370651245, + "loss": 0.4325074553489685, + "loss/core": 0.41803789138793945, + "loss/preference_sft": 0.28032320737838745, + "loss/reference_anchor": 0.6954492330551147, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.088975429534912, + "rewards/margins": 1.3275333642959595, + "rewards/rejected": 1.7614424228668213, + "step": 45 + }, + { + "drift/chosen_abs": 0.20034293830394745, + "drift/rejected_abs": 0.16037920117378235, + "epoch": 0.047781162276772385, + "grad_norm": 228.0, + "ht_mnpo/logit": 0.03978969529271126, + "ht_mnpo/residual": 0.03228970244526863, + "ht_mnpo/residual_abs": 0.13686040043830872, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.3611111111111108e-07, + "logits/chosen": -2.39717960357666, + "logits/rejected": -2.20611572265625, + "logps/chosen": -0.27169185876846313, + "logps/rejected": -0.2966422736644745, + "loss": 0.21604546904563904, + "loss/core": 0.2099648416042328, + "loss/preference_sft": 0.27169185876846313, + "loss/reference_anchor": 0.27686262130737305, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.9987766742706299, + "rewards/margins": 0.39789706468582153, + "rewards/rejected": 1.6008796691894531, + "step": 50 + }, + { + "drift/chosen_abs": 0.2341303527355194, + "drift/rejected_abs": 0.16907049715518951, + "epoch": 0.05255927850444962, + "grad_norm": 136.0, + "ht_mnpo/logit": 0.06506579369306564, + "ht_mnpo/residual": 0.05756579712033272, + "ht_mnpo/residual_abs": 0.07719553261995316, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.5e-07, + "logits/chosen": -2.284651279449463, + "logits/rejected": -2.0335497856140137, + "logps/chosen": -0.2873196005821228, + "logps/rejected": -0.28336891531944275, + "loss": 0.055111657828092575, + "loss/core": 0.04755812883377075, + "loss/preference_sft": 0.2873196005821228, + "loss/reference_anchor": 0.3489445447921753, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.3410696983337402, + "rewards/margins": 0.6506578922271729, + "rewards/rejected": 1.6904115676879883, + "step": 55 + }, + { + "drift/chosen_abs": 0.17374685406684875, + "drift/rejected_abs": 0.11681034415960312, + "epoch": 0.05733739473212686, + "grad_norm": 16.625, + "ht_mnpo/logit": 0.05747275426983833, + "ht_mnpo/residual": 0.049972742795944214, + "ht_mnpo/residual_abs": 0.12492366135120392, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.6388888888888888e-07, + "logits/chosen": -2.3623435497283936, + "logits/rejected": -2.1003968715667725, + "logps/chosen": -0.27126988768577576, + "logps/rejected": -0.27584511041641235, + "loss": 0.14398963749408722, + "loss/core": 0.1407667100429535, + "loss/preference_sft": 0.27126988768577576, + "loss/reference_anchor": 0.13401837646961212, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 1.7370201349258423, + "rewards/margins": 0.574727475643158, + "rewards/rejected": 1.162292718887329, + "step": 60 + }, + { + "drift/chosen_abs": 0.2348300963640213, + "drift/rejected_abs": 0.1319127380847931, + "epoch": 0.062115510959804096, + "grad_norm": 3.5625, + "ht_mnpo/logit": 0.10298211872577667, + "ht_mnpo/residual": 0.09548211842775345, + "ht_mnpo/residual_abs": 0.12007130682468414, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.7777777777777776e-07, + "logits/chosen": -2.171497344970703, + "logits/rejected": -1.954334020614624, + "logps/chosen": -0.2881249189376831, + "logps/rejected": -0.2843600809574127, + "loss": 0.2871479094028473, + "loss/core": 0.2757525146007538, + "loss/preference_sft": 0.2881249189376831, + "loss/reference_anchor": 0.5409581065177917, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.3482909202575684, + "rewards/margins": 1.0298211574554443, + "rewards/rejected": 1.3184694051742554, + "step": 65 + }, + { + "drift/chosen_abs": 0.1808304786682129, + "drift/rejected_abs": 0.11002807319164276, + "epoch": 0.06689362718748133, + "grad_norm": 71.5, + "ht_mnpo/logit": 0.07084055244922638, + "ht_mnpo/residual": 0.06334055960178375, + "ht_mnpo/residual_abs": 0.10876189172267914, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.9166666666666668e-07, + "logits/chosen": -2.284392833709717, + "logits/rejected": -2.0471653938293457, + "logps/chosen": -0.2547953724861145, + "logps/rejected": -0.2760739028453827, + "loss": 0.09047970175743103, + "loss/core": 0.08808381855487823, + "loss/preference_sft": 0.2547953724861145, + "loss/reference_anchor": 0.09431412816047668, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.8078711032867432, + "rewards/margins": 0.708405613899231, + "rewards/rejected": 1.0994654893875122, + "step": 70 + }, + { + "drift/chosen_abs": 0.257373571395874, + "drift/rejected_abs": 0.20142784714698792, + "epoch": 0.07167174341515857, + "grad_norm": 19.5, + "ht_mnpo/logit": 0.055962108075618744, + "ht_mnpo/residual": 0.04846210405230522, + "ht_mnpo/residual_abs": 0.1752299815416336, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.0555555555555553e-07, + "logits/chosen": -2.2204973697662354, + "logits/rejected": -1.941910743713379, + "logps/chosen": -0.2695019841194153, + "logps/rejected": -0.27252477407455444, + "loss": 0.40702590346336365, + "loss/core": 0.39896032214164734, + "loss/preference_sft": 0.2695019841194153, + "loss/reference_anchor": 0.3763294816017151, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.5731680393218994, + "rewards/margins": 0.5596210360527039, + "rewards/rejected": 2.01354718208313, + "step": 75 + }, + { + "drift/chosen_abs": 0.327124685049057, + "drift/rejected_abs": 0.169361412525177, + "epoch": 0.07644985964283581, + "grad_norm": 87.0, + "ht_mnpo/logit": 0.15786626935005188, + "ht_mnpo/residual": 0.15036630630493164, + "ht_mnpo/residual_abs": 0.21834254264831543, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.1944444444444442e-07, + "logits/chosen": -2.004788398742676, + "logits/rejected": -1.7699687480926514, + "logps/chosen": -0.2821227014064789, + "logps/rejected": -0.31434106826782227, + "loss": 0.5702736377716064, + "loss/core": 0.5585130453109741, + "loss/preference_sft": 0.2821227014064789, + "loss/reference_anchor": 0.5598186254501343, + "rewards/accuracies": 0.8125, + "rewards/chosen": 3.2704272270202637, + "rewards/margins": 1.578662633895874, + "rewards/rejected": 1.6917644739151, + "step": 80 + }, + { + "drift/chosen_abs": 0.17778277397155762, + "drift/rejected_abs": 0.11436925828456879, + "epoch": 0.08122797587051304, + "grad_norm": 0.71875, + "ht_mnpo/logit": 0.06345954537391663, + "ht_mnpo/residual": 0.0559595450758934, + "ht_mnpo/residual_abs": 0.1518561840057373, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.3333333333333333e-07, + "logits/chosen": -2.423524856567383, + "logits/rejected": -2.1058201789855957, + "logps/chosen": -0.30849772691726685, + "logps/rejected": -0.3133195638656616, + "loss": 0.34925132989883423, + "loss/core": 0.34384849667549133, + "loss/preference_sft": 0.30849772691726685, + "loss/reference_anchor": 0.23929384350776672, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 1.7773349285125732, + "rewards/margins": 0.634595513343811, + "rewards/rejected": 1.1427395343780518, + "step": 85 + }, + { + "drift/chosen_abs": 0.2773793935775757, + "drift/rejected_abs": 0.21731296181678772, + "epoch": 0.08600609209819028, + "grad_norm": 6.125, + "ht_mnpo/logit": 0.0599784180521965, + "ht_mnpo/residual": 0.05247841402888298, + "ht_mnpo/residual_abs": 0.250608891248703, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4722222222222224e-07, + "logits/chosen": -2.041780471801758, + "logits/rejected": -1.831589698791504, + "logps/chosen": -0.288813978433609, + "logps/rejected": -0.2829095423221588, + "loss": 0.8480337262153625, + "loss/core": 0.8339030146598816, + "loss/preference_sft": 0.288813978433609, + "loss/reference_anchor": 0.6776567697525024, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.7713189125061035, + "rewards/margins": 0.5997841954231262, + "rewards/rejected": 2.171534299850464, + "step": 90 + }, + { + "drift/chosen_abs": 0.2536742091178894, + "drift/rejected_abs": 0.154718279838562, + "epoch": 0.09078420832586753, + "grad_norm": 1.2578125, + "ht_mnpo/logit": 0.09875840693712234, + "ht_mnpo/residual": 0.09125840663909912, + "ht_mnpo/residual_abs": 0.12308160215616226, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4998495746616845e-07, + "logits/chosen": -2.167048215866089, + "logits/rejected": -2.0313143730163574, + "logps/chosen": -0.2899051606655121, + "logps/rejected": -0.27663424611091614, + "loss": 0.19379271566867828, + "loss/core": 0.18530288338661194, + "loss/preference_sft": 0.2899051606655121, + "loss/reference_anchor": 0.39550065994262695, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.53287935256958, + "rewards/margins": 0.987584114074707, + "rewards/rejected": 1.5452953577041626, + "step": 95 + }, + { + "drift/chosen_abs": 0.23249800503253937, + "drift/rejected_abs": 0.22857382893562317, + "epoch": 0.09556232455354477, + "grad_norm": 24.75, + "ht_mnpo/logit": 0.003926327917724848, + "ht_mnpo/residual": -0.0035736740101128817, + "ht_mnpo/residual_abs": 0.19224998354911804, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4992385337738696e-07, + "logits/chosen": -2.255802631378174, + "logits/rejected": -1.9103189706802368, + "logps/chosen": -0.25300678610801697, + "logps/rejected": -0.2917603552341461, + "loss": 0.23590651154518127, + "loss/core": 0.2300805300474167, + "loss/preference_sft": 0.25300678610801697, + "loss/reference_anchor": 0.26599597930908203, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 2.3240771293640137, + "rewards/margins": 0.039263151586055756, + "rewards/rejected": 2.2848141193389893, + "step": 100 + }, + { + "drift/chosen_abs": 0.18887114524841309, + "drift/rejected_abs": 0.2856203615665436, + "epoch": 0.100340440781222, + "grad_norm": 580.0, + "ht_mnpo/logit": -0.0967492014169693, + "ht_mnpo/residual": -0.10424920171499252, + "ht_mnpo/residual_abs": 0.22890052199363708, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4981577053636144e-07, + "logits/chosen": -2.0870354175567627, + "logits/rejected": -1.7681773900985718, + "logps/chosen": -0.24509501457214355, + "logps/rejected": -0.3073538541793823, + "loss": 0.6780762076377869, + "loss/core": 0.6689341068267822, + "loss/preference_sft": 0.24509501457214355, + "loss/reference_anchor": 0.4325965940952301, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.8887115716934204, + "rewards/margins": -0.9674921035766602, + "rewards/rejected": 2.856203556060791, + "step": 105 + }, + { + "drift/chosen_abs": 0.15812969207763672, + "drift/rejected_abs": 0.15588991343975067, + "epoch": 0.10511855700889924, + "grad_norm": 235.0, + "ht_mnpo/logit": 0.002391496207565069, + "ht_mnpo/residual": -0.0051085082814097404, + "ht_mnpo/residual_abs": 0.14227387309074402, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.496607495886281e-07, + "logits/chosen": -2.202495574951172, + "logits/rejected": -1.9575474262237549, + "logps/chosen": -0.33234265446662903, + "logps/rejected": -0.3109672963619232, + "loss": 0.2439870834350586, + "loss/core": 0.239820197224617, + "loss/preference_sft": 0.33234265446662903, + "loss/reference_anchor": 0.1751091033220291, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 1.5793108940124512, + "rewards/margins": 0.02391493320465088, + "rewards/rejected": 1.5553959608078003, + "step": 110 + }, + { + "drift/chosen_abs": 0.25509947538375854, + "drift/rejected_abs": 0.1245686411857605, + "epoch": 0.10989667323657648, + "grad_norm": 1.7734375, + "ht_mnpo/logit": 0.13049247860908508, + "ht_mnpo/residual": 0.12299247831106186, + "ht_mnpo/residual_abs": 0.15432575345039368, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4945884883122553e-07, + "logits/chosen": -2.340104341506958, + "logits/rejected": -2.0771660804748535, + "logps/chosen": -0.2742314338684082, + "logps/rejected": -0.2798416316509247, + "loss": 0.39075616002082825, + "loss/core": 0.3828144669532776, + "loss/preference_sft": 0.2742314338684082, + "loss/reference_anchor": 0.36966127157211304, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.5494778156280518, + "rewards/margins": 1.304924726486206, + "rewards/rejected": 1.2445532083511353, + "step": 115 + }, + { + "drift/chosen_abs": 0.19862906634807587, + "drift/rejected_abs": 0.09067542850971222, + "epoch": 0.11467478946425372, + "grad_norm": 588.0, + "ht_mnpo/logit": 0.1080908328294754, + "ht_mnpo/residual": 0.10059082508087158, + "ht_mnpo/residual_abs": 0.11183144152164459, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.492101441907714e-07, + "logits/chosen": -2.4141907691955566, + "logits/rejected": -2.2686760425567627, + "logps/chosen": -0.2924870550632477, + "logps/rejected": -0.3066648840904236, + "loss": 0.27137768268585205, + "loss/core": 0.2637272775173187, + "loss/preference_sft": 0.2924870550632477, + "loss/reference_anchor": 0.35327258706092834, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.985629677772522, + "rewards/margins": 1.0809080600738525, + "rewards/rejected": 0.9047213792800903, + "step": 120 + }, + { + "drift/chosen_abs": 0.17624413967132568, + "drift/rejected_abs": 0.1153462678194046, + "epoch": 0.11945290569193096, + "grad_norm": 13.9375, + "ht_mnpo/logit": 0.06103396415710449, + "ht_mnpo/residual": 0.053533971309661865, + "ht_mnpo/residual_abs": 0.07010561972856522, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4891472919490977e-07, + "logits/chosen": -2.2220981121063232, + "logits/rejected": -2.035374879837036, + "logps/chosen": -0.3033459782600403, + "logps/rejected": -0.3098070025444031, + "loss": 0.041915275156497955, + "loss/core": 0.038992445915937424, + "loss/preference_sft": 0.3033459782600403, + "loss/reference_anchor": 0.11580680310726166, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.7615543603897095, + "rewards/margins": 0.6103397011756897, + "rewards/rejected": 1.151214361190796, + "step": 125 + }, + { + "drift/chosen_abs": 0.2372765839099884, + "drift/rejected_abs": 0.15658681094646454, + "epoch": 0.12423102191960819, + "grad_norm": 2.78125, + "ht_mnpo/logit": 0.0809415802359581, + "ht_mnpo/residual": 0.07344158738851547, + "ht_mnpo/residual_abs": 0.09841513633728027, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4857271493713894e-07, + "logits/chosen": -2.1383440494537354, + "logits/rejected": -1.9111261367797852, + "logps/chosen": -0.2732178568840027, + "logps/rejected": -0.29032042622566223, + "loss": 0.0999687910079956, + "loss/core": 0.09330854564905167, + "loss/preference_sft": 0.2732178568840027, + "loss/reference_anchor": 0.30569005012512207, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.3718535900115967, + "rewards/margins": 0.8094158172607422, + "rewards/rejected": 1.5624377727508545, + "step": 130 + }, + { + "drift/chosen_abs": 0.15119370818138123, + "drift/rejected_abs": 0.09558363258838654, + "epoch": 0.12900913814728543, + "grad_norm": 15.125, + "ht_mnpo/logit": 0.05558431148529053, + "ht_mnpo/residual": 0.048084307461977005, + "ht_mnpo/residual_abs": 0.0851782038807869, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.481842300350339e-07, + "logits/chosen": -2.00144624710083, + "logits/rejected": -1.7436069250106812, + "logps/chosen": -0.26819977164268494, + "logps/rejected": -0.2890457510948181, + "loss": 0.04468156024813652, + "loss/core": 0.04284118860960007, + "loss/preference_sft": 0.26819977164268494, + "loss/reference_anchor": 0.06519876420497894, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.51114821434021, + "rewards/margins": 0.5558429956436157, + "rewards/rejected": 0.9553052186965942, + "step": 135 + }, + { + "drift/chosen_abs": 0.14579074084758759, + "drift/rejected_abs": 0.10920121520757675, + "epoch": 0.13378725437496267, + "grad_norm": 0.294921875, + "ht_mnpo/logit": 0.03659818693995476, + "ht_mnpo/residual": 0.029098186641931534, + "ht_mnpo/residual_abs": 0.04317057505249977, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4774942058187854e-07, + "logits/chosen": -2.337130308151245, + "logits/rejected": -2.108213424682617, + "logps/chosen": -0.296450674533844, + "logps/rejected": -0.2900570333003998, + "loss": 0.009442105889320374, + "loss/core": 0.00706028938293457, + "loss/preference_sft": 0.296450674533844, + "loss/reference_anchor": 0.08944575488567352, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.4567911624908447, + "rewards/margins": 0.3659818470478058, + "rewards/rejected": 1.0908094644546509, + "step": 140 + }, + { + "drift/chosen_abs": 0.19297416508197784, + "drift/rejected_abs": 0.15170451998710632, + "epoch": 0.1385653706026399, + "grad_norm": 5.90625, + "ht_mnpo/logit": 0.041436947882175446, + "ht_mnpo/residual": 0.03393695130944252, + "ht_mnpo/residual_abs": 0.0991707444190979, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.472684500917257e-07, + "logits/chosen": -2.286118984222412, + "logits/rejected": -2.0098800659179688, + "logps/chosen": -0.2768682837486267, + "logps/rejected": -0.2945496141910553, + "loss": 0.08796399086713791, + "loss/core": 0.08381696045398712, + "loss/preference_sft": 0.2768682837486267, + "loss/reference_anchor": 0.1796645224094391, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 1.9294698238372803, + "rewards/margins": 0.4143694341182709, + "rewards/rejected": 1.5151002407073975, + "step": 145 + }, + { + "drift/chosen_abs": 0.1415017545223236, + "drift/rejected_abs": 0.08456171303987503, + "epoch": 0.14334348683031714, + "grad_norm": 1.90625, + "ht_mnpo/logit": 0.05727854371070862, + "ht_mnpo/residual": 0.04977855831384659, + "ht_mnpo/residual_abs": 0.05690733715891838, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.467414994379065e-07, + "logits/chosen": -2.3385872840881348, + "logits/rejected": -2.1087775230407715, + "logps/chosen": -0.2883468270301819, + "logps/rejected": -0.29340860247612, + "loss": 0.02342948503792286, + "loss/core": 0.021441509947180748, + "loss/preference_sft": 0.2883468270301819, + "loss/reference_anchor": 0.07056401669979095, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.4143787622451782, + "rewards/margins": 0.572785496711731, + "rewards/rejected": 0.8415932655334473, + "step": 150 + }, + { + "drift/chosen_abs": 0.19519026577472687, + "drift/rejected_abs": 0.1824832707643509, + "epoch": 0.14812160305799438, + "grad_norm": 27.25, + "ht_mnpo/logit": 0.01290883868932724, + "ht_mnpo/residual": 0.005408839322626591, + "ht_mnpo/residual_abs": 0.10687340795993805, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4616876678501114e-07, + "logits/chosen": -2.172908306121826, + "logits/rejected": -1.944345235824585, + "logps/chosen": -0.30824723839759827, + "logps/rejected": -0.32008838653564453, + "loss": 0.06261222064495087, + "loss/core": 0.05846996232867241, + "loss/preference_sft": 0.30824723839759827, + "loss/reference_anchor": 0.17628782987594604, + "rewards/accuracies": 0.75, + "rewards/chosen": 1.950910210609436, + "rewards/margins": 0.12908843159675598, + "rewards/rejected": 1.8218215703964233, + "step": 155 + }, + { + "drift/chosen_abs": 0.177214115858078, + "drift/rejected_abs": 0.12131655216217041, + "epoch": 0.15289971928567161, + "grad_norm": 6.65625, + "ht_mnpo/logit": 0.05621614307165146, + "ht_mnpo/residual": 0.04871614649891853, + "ht_mnpo/residual_abs": 0.07616520673036575, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4555046751436735e-07, + "logits/chosen": -2.0418126583099365, + "logits/rejected": -1.8054478168487549, + "logps/chosen": -0.2701527178287506, + "logps/rejected": -0.2704584002494812, + "loss": 0.036772746592760086, + "loss/core": 0.034319765865802765, + "loss/preference_sft": 0.2701527178287506, + "loss/reference_anchor": 0.09563368558883667, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.7721397876739502, + "rewards/margins": 0.5621614456176758, + "rewards/rejected": 1.2099783420562744, + "step": 160 + }, + { + "drift/chosen_abs": 0.23195543885231018, + "drift/rejected_abs": 0.14278283715248108, + "epoch": 0.15767783551334885, + "grad_norm": 3.5, + "ht_mnpo/logit": 0.089602030813694, + "ht_mnpo/residual": 0.08210203796625137, + "ht_mnpo/residual_abs": 0.15943965315818787, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4488683414304425e-07, + "logits/chosen": -2.2963273525238037, + "logits/rejected": -2.1011674404144287, + "logps/chosen": -0.2527472972869873, + "logps/rejected": -0.2691109776496887, + "loss": 0.32412639260292053, + "loss/core": 0.318270742893219, + "loss/preference_sft": 0.2527472972869873, + "loss/reference_anchor": 0.26750943064689636, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 2.3194217681884766, + "rewards/margins": 0.8960202932357788, + "rewards/rejected": 1.4234017133712769, + "step": 165 + }, + { + "drift/chosen_abs": 0.23300659656524658, + "drift/rejected_abs": 0.22859875857830048, + "epoch": 0.1624559517410261, + "grad_norm": 1.578125, + "ht_mnpo/logit": 0.004623569548130035, + "ht_mnpo/residual": -0.0028764219023287296, + "ht_mnpo/residual_abs": 0.1669018566608429, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4417811623641203e-07, + "logits/chosen": -2.144458055496216, + "logits/rejected": -1.9285306930541992, + "logps/chosen": -0.27059751749038696, + "logps/rejected": -0.2944408059120178, + "loss": 0.4071100354194641, + "loss/core": 0.397773414850235, + "loss/preference_sft": 0.27059751749038696, + "loss/reference_anchor": 0.4397706985473633, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.330066204071045, + "rewards/margins": 0.0462358221411705, + "rewards/rejected": 2.283830165863037, + "step": 170 + }, + { + "drift/chosen_abs": 0.23780396580696106, + "drift/rejected_abs": 0.21040399372577667, + "epoch": 0.16723406796870333, + "grad_norm": 0.765625, + "ht_mnpo/logit": 0.027342364192008972, + "ht_mnpo/residual": 0.019842345267534256, + "ht_mnpo/residual_abs": 0.26941126585006714, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4342458031429113e-07, + "logits/chosen": -2.155928134918213, + "logits/rejected": -1.8418967723846436, + "logps/chosen": -0.28269365429878235, + "logps/rejected": -0.3167445659637451, + "loss": 0.8618923425674438, + "loss/core": 0.8492777943611145, + "loss/preference_sft": 0.28269365429878235, + "loss/reference_anchor": 0.6024563908576965, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.3754687309265137, + "rewards/margins": 0.273423433303833, + "rewards/rejected": 2.1020452976226807, + "step": 175 + }, + { + "drift/chosen_abs": 0.17630836367607117, + "drift/rejected_abs": 0.15115013718605042, + "epoch": 0.17201218419638056, + "grad_norm": 3.4375, + "ht_mnpo/logit": 0.02521784044802189, + "ht_mnpo/residual": 0.017717847600579262, + "ht_mnpo/residual_abs": 0.11181072145700455, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4262650975072444e-07, + "logits/chosen": -2.3172078132629395, + "logits/rejected": -2.0724923610687256, + "logps/chosen": -0.2543174624443054, + "logps/rejected": -0.24170401692390442, + "loss": 0.10301873832941055, + "loss/core": 0.09923148900270462, + "loss/preference_sft": 0.2543174624443054, + "loss/reference_anchor": 0.16393068432807922, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 1.7623436450958252, + "rewards/margins": 0.252178430557251, + "rewards/rejected": 1.5101648569107056, + "step": 180 + }, + { + "drift/chosen_abs": 0.4186461567878723, + "drift/rejected_abs": 0.10205858945846558, + "epoch": 0.17679030042405783, + "grad_norm": 32.0, + "ht_mnpo/logit": 0.32172200083732605, + "ht_mnpo/residual": 0.3142220079898834, + "ht_mnpo/residual_abs": 0.3194965720176697, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.417842046674122e-07, + "logits/chosen": -2.145289421081543, + "logits/rejected": -1.9773670434951782, + "logps/chosen": -0.3100564777851105, + "logps/rejected": -0.31372490525245667, + "loss": 1.5905460119247437, + "loss/core": 1.5711582899093628, + "loss/preference_sft": 0.3100564777851105, + "loss/reference_anchor": 0.9383944272994995, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.18438720703125, + "rewards/margins": 3.217219829559326, + "rewards/rejected": 0.9671671986579895, + "step": 185 + }, + { + "drift/chosen_abs": 0.14292044937610626, + "drift/rejected_abs": 0.08453439176082611, + "epoch": 0.18156841665173507, + "grad_norm": 6.78125, + "ht_mnpo/logit": 0.05871760845184326, + "ht_mnpo/residual": 0.051217615604400635, + "ht_mnpo/residual_abs": 0.06432562321424484, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4089798182084843e-07, + "logits/chosen": -2.21405291557312, + "logits/rejected": -2.0281381607055664, + "logps/chosen": -0.2850034534931183, + "logps/rejected": -0.29590287804603577, + "loss": 0.020889047533273697, + "loss/core": 0.01923513039946556, + "loss/preference_sft": 0.2850034534931183, + "loss/reference_anchor": 0.054195452481508255, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.4286693334579468, + "rewards/margins": 0.5871760249137878, + "rewards/rejected": 0.8414933085441589, + "step": 190 + }, + { + "drift/chosen_abs": 0.2953607738018036, + "drift/rejected_abs": 0.0970875471830368, + "epoch": 0.1863465328794123, + "grad_norm": 0.77734375, + "ht_mnpo/logit": 0.2047409564256668, + "ht_mnpo/residual": 0.19724097847938538, + "ht_mnpo/residual_abs": 0.2064896821975708, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.3996817448320195e-07, + "logits/chosen": -2.3270435333251953, + "logits/rejected": -2.1133506298065186, + "logps/chosen": -0.315519243478775, + "logps/rejected": -0.26455676555633545, + "loss": 0.8637374639511108, + "loss/core": 0.8507298231124878, + "loss/preference_sft": 0.315519243478775, + "loss/reference_anchor": 0.6188250780105591, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.9532313346862793, + "rewards/margins": 2.047409772872925, + "rewards/rejected": 0.905821681022644, + "step": 195 + }, + { + "drift/chosen_abs": 0.1497114598751068, + "drift/rejected_abs": 0.10208232700824738, + "epoch": 0.19112464910708954, + "grad_norm": 2.9375, + "ht_mnpo/logit": 0.04762087017297745, + "ht_mnpo/residual": 0.04012087732553482, + "ht_mnpo/residual_abs": 0.061163824051618576, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.3899513231698607e-07, + "logits/chosen": -2.379368305206299, + "logits/rejected": -2.0901029109954834, + "logps/chosen": -0.2885080873966217, + "logps/rejected": -0.29427370429039, + "loss": 0.01607438549399376, + "loss/core": 0.014033469371497631, + "loss/preference_sft": 0.2885080873966217, + "loss/reference_anchor": 0.07319486886262894, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 1.4958560466766357, + "rewards/margins": 0.4762086868286133, + "rewards/rejected": 1.0196473598480225, + "step": 200 + }, + { + "drift/chosen_abs": 0.22366979718208313, + "drift/rejected_abs": 0.16261938214302063, + "epoch": 0.19590276533476678, + "grad_norm": 7.59375, + "ht_mnpo/logit": 0.06350360065698624, + "ht_mnpo/residual": 0.05600360780954361, + "ht_mnpo/residual_abs": 0.15254007279872894, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.3797922124356506e-07, + "logits/chosen": -2.0374624729156494, + "logits/rejected": -1.8821849822998047, + "logps/chosen": -0.30214640498161316, + "logps/rejected": -0.32250362634658813, + "loss": 0.23545308411121368, + "loss/core": 0.2295982837677002, + "loss/preference_sft": 0.30214640498161316, + "loss/reference_anchor": 0.26252511143684387, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.2357821464538574, + "rewards/margins": 0.6350361108779907, + "rewards/rejected": 1.6007457971572876, + "step": 205 + }, + { + "drift/chosen_abs": 0.16679704189300537, + "drift/rejected_abs": 0.1574680507183075, + "epoch": 0.200680881562444, + "grad_norm": 13.8125, + "ht_mnpo/logit": 0.009180588647723198, + "ht_mnpo/residual": 0.001680587651208043, + "ht_mnpo/residual_abs": 0.08262274414300919, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.3692082330554585e-07, + "logits/chosen": -2.282926321029663, + "logits/rejected": -2.038118362426758, + "logps/chosen": -0.3618142306804657, + "logps/rejected": -0.2868843674659729, + "loss": 0.07468167692422867, + "loss/core": 0.07061126083135605, + "loss/preference_sft": 0.3618142306804657, + "loss/reference_anchor": 0.1673388034105301, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 1.6660865545272827, + "rewards/margins": 0.09180586040019989, + "rewards/rejected": 1.5742807388305664, + "step": 210 + }, + { + "drift/chosen_abs": 0.2035273313522339, + "drift/rejected_abs": 0.13128599524497986, + "epoch": 0.20545899779012125, + "grad_norm": 8.875, + "ht_mnpo/logit": 0.07242141664028168, + "ht_mnpo/residual": 0.06492140889167786, + "ht_mnpo/residual_abs": 0.11575053632259369, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.3582033652310765e-07, + "logits/chosen": -2.10392689704895, + "logits/rejected": -1.9740612506866455, + "logps/chosen": -0.27785545587539673, + "logps/rejected": -0.2743791937828064, + "loss": 0.07868362963199615, + "loss/core": 0.07500644028186798, + "loss/preference_sft": 0.27785545587539673, + "loss/reference_anchor": 0.15607400238513947, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.0350780487060547, + "rewards/margins": 0.7242141962051392, + "rewards/rejected": 1.3108640909194946, + "step": 215 + }, + { + "drift/chosen_abs": 0.27838942408561707, + "drift/rejected_abs": 0.20967824757099152, + "epoch": 0.2102371140177985, + "grad_norm": 3.1875, + "ht_mnpo/logit": 0.06874275207519531, + "ht_mnpo/residual": 0.06124274805188179, + "ht_mnpo/residual_abs": 0.17890474200248718, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.346781747443227e-07, + "logits/chosen": -2.0583040714263916, + "logits/rejected": -1.754565954208374, + "logps/chosen": -0.29740554094314575, + "logps/rejected": -0.28887122869491577, + "loss": 0.35021087527275085, + "loss/core": 0.339802622795105, + "loss/preference_sft": 0.29740554094314575, + "loss/reference_anchor": 0.490673303604126, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": 2.7822656631469727, + "rewards/margins": 0.6874275803565979, + "rewards/rejected": 2.0948376655578613, + "step": 220 + }, + { + "drift/chosen_abs": 0.20137110352516174, + "drift/rejected_abs": 0.14107075333595276, + "epoch": 0.21501523024547572, + "grad_norm": 2.234375, + "ht_mnpo/logit": 0.0653047114610672, + "ht_mnpo/residual": 0.057804714888334274, + "ht_mnpo/residual_abs": 0.1389622986316681, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.3349476748952508e-07, + "logits/chosen": -2.24853515625, + "logits/rejected": -2.0200893878936768, + "logps/chosen": -0.27711716294288635, + "logps/rejected": -0.29098328948020935, + "loss": 0.17267122864723206, + "loss/core": 0.1683366894721985, + "loss/preference_sft": 0.27711716294288635, + "loss/reference_anchor": 0.18901614844799042, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.0136349201202393, + "rewards/margins": 0.6530472040176392, + "rewards/rejected": 1.3605875968933105, + "step": 225 + }, + { + "drift/chosen_abs": 0.15755267441272736, + "drift/rejected_abs": 0.16204038262367249, + "epoch": 0.21979334647315296, + "grad_norm": 8.5625, + "ht_mnpo/logit": -0.004402917809784412, + "ht_mnpo/residual": -0.011902913451194763, + "ht_mnpo/residual_abs": 0.13799583911895752, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.3227055978978545e-07, + "logits/chosen": -2.2092113494873047, + "logits/rejected": -1.9548333883285522, + "logps/chosen": -0.27587762475013733, + "logps/rejected": -0.2882309854030609, + "loss": 0.20742078125476837, + "loss/core": 0.20317880809307098, + "loss/preference_sft": 0.27587762475013733, + "loss/reference_anchor": 0.18451109528541565, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 1.575390338897705, + "rewards/margins": -0.04402913898229599, + "rewards/rejected": 1.6194193363189697, + "step": 230 + }, + { + "drift/chosen_abs": 0.22635149955749512, + "drift/rejected_abs": 0.1525922268629074, + "epoch": 0.2245714627008302, + "grad_norm": 478.0, + "ht_mnpo/logit": 0.07377196848392487, + "ht_mnpo/residual": 0.06627197563648224, + "ht_mnpo/residual_abs": 0.17167456448078156, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.3100601201955321e-07, + "logits/chosen": -2.1003198623657227, + "logits/rejected": -1.934924840927124, + "logps/chosen": -0.2860856354236603, + "logps/rejected": -0.2972342371940613, + "loss": 0.60406893491745, + "loss/core": 0.5958213806152344, + "loss/preference_sft": 0.2860856354236603, + "loss/reference_anchor": 0.38376861810684204, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.2631850242614746, + "rewards/margins": 0.7377198338508606, + "rewards/rejected": 1.5254653692245483, + "step": 235 + }, + { + "drift/chosen_abs": 0.13833805918693542, + "drift/rejected_abs": 0.08581127971410751, + "epoch": 0.22934957892850744, + "grad_norm": 6.8125, + "ht_mnpo/logit": 0.05265297740697861, + "ht_mnpo/residual": 0.04515298455953598, + "ht_mnpo/residual_abs": 0.08135689049959183, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.2970159972352864e-07, + "logits/chosen": -2.3458266258239746, + "logits/rejected": -2.1107969284057617, + "logps/chosen": -0.2840951085090637, + "logps/rejected": -0.27419576048851013, + "loss": 0.03435717895627022, + "loss/core": 0.03234320878982544, + "loss/preference_sft": 0.2840951085090637, + "loss/reference_anchor": 0.0722891315817833, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.3823360204696655, + "rewards/margins": 0.5265297889709473, + "rewards/rejected": 0.8558060526847839, + "step": 240 + }, + { + "drift/chosen_abs": 0.17416642606258392, + "drift/rejected_abs": 0.14134536683559418, + "epoch": 0.23412769515618467, + "grad_norm": 40.0, + "ht_mnpo/logit": 0.03294435888528824, + "ht_mnpo/residual": 0.025444358587265015, + "ht_mnpo/residual_abs": 0.11847629398107529, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.2835781343782966e-07, + "logits/chosen": -2.307325839996338, + "logits/rejected": -2.0234155654907227, + "logps/chosen": -0.3149130046367645, + "logps/rejected": -0.30677077174186707, + "loss": 0.1159428134560585, + "loss/core": 0.11232848465442657, + "loss/preference_sft": 0.3149130046367645, + "loss/reference_anchor": 0.1492251455783844, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 1.7409662008285522, + "rewards/margins": 0.3294435739517212, + "rewards/rejected": 1.4115229845046997, + "step": 245 + }, + { + "drift/chosen_abs": 0.2660543918609619, + "drift/rejected_abs": 0.11109892278909683, + "epoch": 0.2389058113838619, + "grad_norm": 868.0, + "ht_mnpo/logit": 0.1549011766910553, + "ht_mnpo/residual": 0.14740118384361267, + "ht_mnpo/residual_abs": 0.15847142040729523, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.2697515850552152e-07, + "logits/chosen": -2.3213107585906982, + "logits/rejected": -2.1739933490753174, + "logps/chosen": -0.25932732224464417, + "logps/rejected": -0.2790648937225342, + "loss": 0.598106324672699, + "loss/core": 0.5861698389053345, + "loss/preference_sft": 0.25932732224464417, + "loss/reference_anchor": 0.5708891153335571, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.6590445041656494, + "rewards/margins": 1.5490118265151978, + "rewards/rejected": 1.1100324392318726, + "step": 250 + }, + { + "drift/chosen_abs": 0.1992466151714325, + "drift/rejected_abs": 0.11664634943008423, + "epoch": 0.24368392761153915, + "grad_norm": 4.4375, + "ht_mnpo/logit": 0.08275730162858963, + "ht_mnpo/residual": 0.0752573013305664, + "ht_mnpo/residual_abs": 0.08343817293643951, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.2555415488657775e-07, + "logits/chosen": -2.1033692359924316, + "logits/rejected": -1.9849478006362915, + "logps/chosen": -0.24639995396137238, + "logps/rejected": -0.2411382645368576, + "loss": 0.04728394001722336, + "loss/core": 0.0447533018887043, + "loss/preference_sft": 0.24639995396137238, + "loss/reference_anchor": 0.10189200937747955, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.9907464981079102, + "rewards/margins": 0.8275730013847351, + "rewards/rejected": 1.1631735563278198, + "step": 255 + }, + { + "drift/chosen_abs": 0.29991379380226135, + "drift/rejected_abs": 0.18915988504886627, + "epoch": 0.24846204383921638, + "grad_norm": 25.25, + "ht_mnpo/logit": 0.11084876209497452, + "ht_mnpo/residual": 0.1033487543463707, + "ht_mnpo/residual_abs": 0.22364541888237, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.240953369623447e-07, + "logits/chosen": -2.131293535232544, + "logits/rejected": -1.8576065301895142, + "logps/chosen": -0.2812115550041199, + "logps/rejected": -0.30115509033203125, + "loss": 0.7878208160400391, + "loss/core": 0.7746709585189819, + "loss/preference_sft": 0.2812115550041199, + "loss/reference_anchor": 0.6293684840202332, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.9983861446380615, + "rewards/margins": 1.108487606048584, + "rewards/rejected": 1.8898985385894775, + "step": 260 + }, + { + "drift/chosen_abs": 0.13449910283088684, + "drift/rejected_abs": 0.12530843913555145, + "epoch": 0.25324016006689365, + "grad_norm": 0.462890625, + "ht_mnpo/logit": 0.009142006747424603, + "ht_mnpo/residual": 0.0016420104075223207, + "ht_mnpo/residual_abs": 0.09060578048229218, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.225992533345824e-07, + "logits/chosen": -2.140869379043579, + "logits/rejected": -1.6922798156738281, + "logps/chosen": -0.29281654953956604, + "logps/rejected": -0.2915118336677551, + "loss": 0.042861003428697586, + "loss/core": 0.04084916412830353, + "loss/preference_sft": 0.29281654953956604, + "loss/reference_anchor": 0.07131018489599228, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 1.3426874876022339, + "rewards/margins": 0.09142010658979416, + "rewards/rejected": 1.2512675523757935, + "step": 265 + }, + { + "drift/chosen_abs": 0.20011694729328156, + "drift/rejected_abs": 0.09070627391338348, + "epoch": 0.25801827629457086, + "grad_norm": 11.375, + "ht_mnpo/logit": 0.11147568374872208, + "ht_mnpo/residual": 0.10397566854953766, + "ht_mnpo/residual_abs": 0.1260581612586975, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.210664666191579e-07, + "logits/chosen": -2.4403769969940186, + "logits/rejected": -2.233523368835449, + "logps/chosen": -0.2794303297996521, + "logps/rejected": -0.28702133893966675, + "loss": 0.2465023547410965, + "loss/core": 0.24205832183361053, + "loss/preference_sft": 0.2794303297996521, + "loss/reference_anchor": 0.19425857067108154, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.9994302988052368, + "rewards/margins": 1.1147568225860596, + "rewards/rejected": 0.8846734166145325, + "step": 270 + }, + { + "drift/chosen_abs": 0.4535955488681793, + "drift/rejected_abs": 0.2866347134113312, + "epoch": 0.2627963925222481, + "grad_norm": 7.96875, + "ht_mnpo/logit": 0.16719956696033478, + "ht_mnpo/residual": 0.15969957411289215, + "ht_mnpo/residual_abs": 0.29034724831581116, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.1949755323446848e-07, + "logits/chosen": -2.2299437522888184, + "logits/rejected": -1.9355762004852295, + "logps/chosen": -0.28128141164779663, + "logps/rejected": -0.28012967109680176, + "loss": 0.9513923525810242, + "loss/core": 0.9261605143547058, + "loss/preference_sft": 0.28128141164779663, + "loss/reference_anchor": 1.2334659099578857, + "rewards/accuracies": 0.875, + "rewards/chosen": 4.535092353820801, + "rewards/margins": 1.671995759010315, + "rewards/rejected": 2.8630967140197754, + "step": 275 + }, + { + "drift/chosen_abs": 0.24227941036224365, + "drift/rejected_abs": 0.1327737718820572, + "epoch": 0.26757450874992533, + "grad_norm": 8.125, + "ht_mnpo/logit": 0.10931800305843353, + "ht_mnpo/residual": 0.1018180102109909, + "ht_mnpo/residual_abs": 0.11589553207159042, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.1789310318467426e-07, + "logits/chosen": -2.0422604084014893, + "logits/rejected": -1.9183323383331299, + "logps/chosen": -0.2838364541530609, + "logps/rejected": -0.27868276834487915, + "loss": 0.17103958129882812, + "loss/core": 0.16334925591945648, + "loss/preference_sft": 0.2838364541530609, + "loss/reference_anchor": 0.356132447719574, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.420574903488159, + "rewards/margins": 1.0931799411773682, + "rewards/rejected": 1.327394962310791, + "step": 280 + }, + { + "drift/chosen_abs": 0.2416619062423706, + "drift/rejected_abs": 0.1695837825536728, + "epoch": 0.2723526249776026, + "grad_norm": 5.875, + "ht_mnpo/logit": 0.07214997708797455, + "ht_mnpo/residual": 0.06464996933937073, + "ht_mnpo/residual_abs": 0.09243568778038025, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.1625371983782182e-07, + "logits/chosen": -2.091132164001465, + "logits/rejected": -1.8592939376831055, + "logps/chosen": -0.3188819885253906, + "logps/rejected": -0.3187660574913025, + "loss": 0.0901908352971077, + "loss/core": 0.07827536761760712, + "loss/preference_sft": 0.3188819885253906, + "loss/reference_anchor": 0.5638847947120667, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.415431499481201, + "rewards/margins": 0.7214997410774231, + "rewards/rejected": 1.6939315795898438, + "step": 285 + }, + { + "drift/chosen_abs": 0.20275557041168213, + "drift/rejected_abs": 0.165045827627182, + "epoch": 0.2771307412052798, + "grad_norm": 18.125, + "ht_mnpo/logit": 0.037836022675037384, + "ht_mnpo/residual": 0.030336027964949608, + "ht_mnpo/residual_abs": 0.08644749969244003, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.14580019698942e-07, + "logits/chosen": -2.215244770050049, + "logits/rejected": -1.9540691375732422, + "logps/chosen": -0.28964218497276306, + "logps/rejected": -0.28602391481399536, + "loss": 0.04661395773291588, + "loss/core": 0.04280799999833107, + "loss/preference_sft": 0.28964218497276306, + "loss/reference_anchor": 0.161333829164505, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.027524471282959, + "rewards/margins": 0.3783602714538574, + "rewards/rejected": 1.649163842201233, + "step": 290 + }, + { + "drift/chosen_abs": 0.2054920196533203, + "drift/rejected_abs": 0.2024458944797516, + "epoch": 0.28190885743295707, + "grad_norm": 448.0, + "ht_mnpo/logit": 0.0030287965200841427, + "ht_mnpo/residual": -0.004471203777939081, + "ht_mnpo/residual_abs": 0.1715690791606903, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.1287263217820773e-07, + "logits/chosen": -2.1912224292755127, + "logits/rejected": -2.0103259086608887, + "logps/chosen": -0.27507418394088745, + "logps/rejected": -0.2898535132408142, + "loss": 0.5342413187026978, + "loss/core": 0.5257778763771057, + "loss/preference_sft": 0.27507418394088745, + "loss/reference_anchor": 0.39566391706466675, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.054013252258301, + "rewards/margins": 0.030287981033325195, + "rewards/rejected": 2.0237250328063965, + "step": 295 + }, + { + "drift/chosen_abs": 0.32648366689682007, + "drift/rejected_abs": 0.14543873071670532, + "epoch": 0.2866869736606343, + "grad_norm": 7.25, + "ht_mnpo/logit": 0.19056273996829987, + "ht_mnpo/residual": 0.18306274712085724, + "ht_mnpo/residual_abs": 0.22434112429618835, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.111321993542385e-07, + "logits/chosen": -2.2440693378448486, + "logits/rejected": -1.9663177728652954, + "logps/chosen": -0.29654619097709656, + "logps/rejected": -0.30812662839889526, + "loss": 1.0591758489608765, + "loss/core": 1.0461701154708862, + "loss/preference_sft": 0.29654619097709656, + "loss/reference_anchor": 0.6206279397010803, + "rewards/accuracies": 0.8125, + "rewards/chosen": 3.2646141052246094, + "rewards/margins": 1.9056276082992554, + "rewards/rejected": 1.3589863777160645, + "step": 300 + }, + { + "drift/chosen_abs": 0.3030497133731842, + "drift/rejected_abs": 0.14767694473266602, + "epoch": 0.29146508988831155, + "grad_norm": 1984.0, + "ht_mnpo/logit": 0.15544824302196503, + "ht_mnpo/residual": 0.1479482352733612, + "ht_mnpo/residual_abs": 0.16500267386436462, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.0935937573264096e-07, + "logits/chosen": -2.2528464794158936, + "logits/rejected": -2.066157579421997, + "logps/chosen": -0.2954830527305603, + "logps/rejected": -0.2760084271430969, + "loss": 0.5691508650779724, + "loss/core": 0.5540502071380615, + "loss/preference_sft": 0.2954830527305603, + "loss/reference_anchor": 0.7254848480224609, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.027735471725464, + "rewards/margins": 1.5544825792312622, + "rewards/rejected": 1.4732530117034912, + "step": 305 + }, + { + "drift/chosen_abs": 0.24608822166919708, + "drift/rejected_abs": 0.19582870602607727, + "epoch": 0.29624320611598876, + "grad_norm": 18.0, + "ht_mnpo/logit": 0.050264328718185425, + "ht_mnpo/residual": 0.0427643321454525, + "ht_mnpo/residual_abs": 0.16030198335647583, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.0755482799987596e-07, + "logits/chosen": -2.362098455429077, + "logits/rejected": -2.1859664916992188, + "logps/chosen": -0.27773889899253845, + "logps/rejected": -0.2750205099582672, + "loss": 0.27516818046569824, + "loss/core": 0.26793110370635986, + "loss/preference_sft": 0.27773889899253845, + "loss/reference_anchor": 0.33407989144325256, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.458608388900757, + "rewards/margins": 0.5026432275772095, + "rewards/rejected": 1.9559653997421265, + "step": 310 + }, + { + "drift/chosen_abs": 0.16757844388484955, + "drift/rejected_abs": 0.13082993030548096, + "epoch": 0.301021322343666, + "grad_norm": 9.875, + "ht_mnpo/logit": 0.045271605253219604, + "ht_mnpo/residual": 0.03777160868048668, + "ht_mnpo/residual_abs": 0.13339407742023468, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.0571923477254526e-07, + "logits/chosen": -2.4076950550079346, + "logits/rejected": -2.056607246398926, + "logps/chosen": -0.28617167472839355, + "logps/rejected": -0.2988688349723816, + "loss": 0.23253202438354492, + "loss/core": 0.22873583436012268, + "loss/preference_sft": 0.28617167472839355, + "loss/reference_anchor": 0.16119354963302612, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 1.6741405725479126, + "rewards/margins": 0.4527161717414856, + "rewards/rejected": 1.2214245796203613, + "step": 315 + }, + { + "drift/chosen_abs": 0.20812535285949707, + "drift/rejected_abs": 0.12873616814613342, + "epoch": 0.30579943857134323, + "grad_norm": 99.0, + "ht_mnpo/logit": 0.08008380234241486, + "ht_mnpo/residual": 0.07258381694555283, + "ht_mnpo/residual_abs": 0.16876870393753052, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.038532863421914e-07, + "logits/chosen": -2.2163033485412598, + "logits/rejected": -1.90713369846344, + "logps/chosen": -0.2852802872657776, + "logps/rejected": -0.29477205872535706, + "loss": 0.333568811416626, + "loss/core": 0.3274485170841217, + "loss/preference_sft": 0.2852802872657776, + "loss/reference_anchor": 0.277485728263855, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.0812535285949707, + "rewards/margins": 0.8008379936218262, + "rewards/rejected": 1.2804152965545654, + "step": 320 + }, + { + "drift/chosen_abs": 0.24407429993152618, + "drift/rejected_abs": 0.11723603308200836, + "epoch": 0.3105775547990205, + "grad_norm": 438.0, + "ht_mnpo/logit": 0.13229139149188995, + "ht_mnpo/residual": 0.12479136139154434, + "ht_mnpo/residual_abs": 0.16577567160129547, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.0195768441570726e-07, + "logits/chosen": -2.1131129264831543, + "logits/rejected": -1.9461160898208618, + "logps/chosen": -0.3240113854408264, + "logps/rejected": -0.3457443118095398, + "loss": 0.32235246896743774, + "loss/core": 0.3146476149559021, + "loss/preference_sft": 0.3240113854408264, + "loss/reference_anchor": 0.3528444468975067, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.4372329711914062, + "rewards/margins": 1.3229138851165771, + "rewards/rejected": 1.114319086074829, + "step": 325 + }, + { + "drift/chosen_abs": 0.1336461305618286, + "drift/rejected_abs": 0.14605210721492767, + "epoch": 0.3153556710266977, + "grad_norm": 9.625, + "ht_mnpo/logit": -0.012384334579110146, + "ht_mnpo/residual": -0.01988433301448822, + "ht_mnpo/residual_abs": 0.10770714282989502, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.0003314185145307e-07, + "logits/chosen": -2.1785032749176025, + "logits/rejected": -1.9495980739593506, + "logps/chosen": -0.27921396493911743, + "logps/rejected": -0.3031226396560669, + "loss": 0.2651267647743225, + "loss/core": 0.2610391080379486, + "loss/preference_sft": 0.27921396493911743, + "loss/reference_anchor": 0.1764618158340454, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.3336832523345947, + "rewards/margins": -0.12384340912103653, + "rewards/rejected": 1.457526445388794, + "step": 330 + }, + { + "drift/chosen_abs": 0.23569095134735107, + "drift/rejected_abs": 0.17359793186187744, + "epoch": 0.32013378725437497, + "grad_norm": 292.0, + "ht_mnpo/logit": 0.06208749860525131, + "ht_mnpo/residual": 0.05458749458193779, + "ht_mnpo/residual_abs": 0.13036298751831055, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.9808038239117913e-07, + "logits/chosen": -2.226792097091675, + "logits/rejected": -1.9871718883514404, + "logps/chosen": -0.2731962502002716, + "logps/rejected": -0.26592594385147095, + "loss": 0.16251419484615326, + "loss/core": 0.15205292403697968, + "loss/preference_sft": 0.2731962502002716, + "loss/reference_anchor": 0.49574416875839233, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.35514497756958, + "rewards/margins": 0.6208750009536743, + "rewards/rejected": 1.7342700958251953, + "step": 335 + }, + { + "drift/chosen_abs": 0.22859697043895721, + "drift/rejected_abs": 0.14668402075767517, + "epoch": 0.3249119034820522, + "grad_norm": 31.625, + "ht_mnpo/logit": 0.0824618712067604, + "ht_mnpo/residual": 0.07496186345815659, + "ht_mnpo/residual_abs": 0.17400358617305756, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.9610014038785646e-07, + "logits/chosen": -2.1349992752075195, + "logits/rejected": -2.008659839630127, + "logps/chosen": -0.29214566946029663, + "logps/rejected": -0.2824420630931854, + "loss": 0.28576305508613586, + "loss/core": 0.280608206987381, + "loss/preference_sft": 0.29214566946029663, + "loss/reference_anchor": 0.22852830588817596, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.2851531505584717, + "rewards/margins": 0.8246185183525085, + "rewards/rejected": 1.4605344533920288, + "step": 340 + }, + { + "drift/chosen_abs": 0.25084584951400757, + "drift/rejected_abs": 0.26426196098327637, + "epoch": 0.32969001970972944, + "grad_norm": 1.6875, + "ht_mnpo/logit": -0.013153704814612865, + "ht_mnpo/residual": -0.020653702318668365, + "ht_mnpo/residual_abs": 0.19523727893829346, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.9409316052951657e-07, + "logits/chosen": -2.104379415512085, + "logits/rejected": -1.8748838901519775, + "logps/chosen": -0.29315048456192017, + "logps/rejected": -0.3299732506275177, + "loss": 0.4619339406490326, + "loss/core": 0.4512755274772644, + "loss/preference_sft": 0.29315048456192017, + "loss/reference_anchor": 0.503603458404541, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.508052349090576, + "rewards/margins": -0.1315370798110962, + "rewards/rejected": 2.639589309692383, + "step": 345 + }, + { + "drift/chosen_abs": 0.21669964492321014, + "drift/rejected_abs": 0.08468327671289444, + "epoch": 0.33446813593740665, + "grad_norm": 10.4375, + "ht_mnpo/logit": 0.13242420554161072, + "ht_mnpo/residual": 0.12492422014474869, + "ht_mnpo/residual_abs": 0.1702018678188324, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.920601975592047e-07, + "logits/chosen": -2.063258409500122, + "logits/rejected": -1.8462562561035156, + "logps/chosen": -0.30922695994377136, + "logps/rejected": -0.3094988763332367, + "loss": 0.6405929327011108, + "loss/core": 0.6327069997787476, + "loss/preference_sft": 0.30922695994377136, + "loss/reference_anchor": 0.3633735775947571, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.165689706802368, + "rewards/margins": 1.3242422342300415, + "rewards/rejected": 0.8414475321769714, + "step": 350 + }, + { + "drift/chosen_abs": 0.34972286224365234, + "drift/rejected_abs": 0.2043858766555786, + "epoch": 0.3392462521650839, + "grad_norm": 71.5, + "ht_mnpo/logit": 0.14536131918430328, + "ht_mnpo/residual": 0.13786129653453827, + "ht_mnpo/residual_abs": 0.18553060293197632, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.900020159911519e-07, + "logits/chosen": -2.251025438308716, + "logits/rejected": -1.9978622198104858, + "logps/chosen": -0.27270156145095825, + "logps/rejected": -0.283389151096344, + "loss": 0.36679738759994507, + "loss/core": 0.3456703722476959, + "loss/preference_sft": 0.27270156145095825, + "loss/reference_anchor": 1.0290827751159668, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 3.495182752609253, + "rewards/margins": 1.45361328125, + "rewards/rejected": 2.041569471359253, + "step": 355 + }, + { + "drift/chosen_abs": 0.31272217631340027, + "drift/rejected_abs": 0.15699851512908936, + "epoch": 0.3440243683927611, + "grad_norm": 99.5, + "ht_mnpo/logit": 0.15567512810230255, + "ht_mnpo/residual": 0.14817515015602112, + "ht_mnpo/residual_abs": 0.20475678145885468, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.879193898232725e-07, + "logits/chosen": -2.2675702571868896, + "logits/rejected": -2.1046242713928223, + "logps/chosen": -0.3180842399597168, + "logps/rejected": -0.28562289476394653, + "loss": 0.7316678762435913, + "loss/core": 0.7170267105102539, + "loss/preference_sft": 0.3180842399597168, + "loss/reference_anchor": 0.7002483606338501, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.126345157623291, + "rewards/margins": 1.5567514896392822, + "rewards/rejected": 1.5695935487747192, + "step": 360 + }, + { + "drift/chosen_abs": 0.3319775462150574, + "drift/rejected_abs": 0.18090906739234924, + "epoch": 0.3488024846204384, + "grad_norm": 1432.0, + "ht_mnpo/logit": 0.15248021483421326, + "ht_mnpo/residual": 0.14498022198677063, + "ht_mnpo/residual_abs": 0.21452391147613525, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8581310224609496e-07, + "logits/chosen": -2.4009974002838135, + "logits/rejected": -2.160431146621704, + "logps/chosen": -0.2676542401313782, + "logps/rejected": -0.2706208825111389, + "loss": 0.7889020442962646, + "loss/core": 0.7692867517471313, + "loss/preference_sft": 0.2676542401313782, + "loss/reference_anchor": 0.9540020823478699, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 3.3183226585388184, + "rewards/margins": 1.5248022079467773, + "rewards/rejected": 1.7935206890106201, + "step": 365 + }, + { + "drift/chosen_abs": 0.3430865705013275, + "drift/rejected_abs": 0.22682738304138184, + "epoch": 0.35358060084811566, + "grad_norm": 3.640625, + "ht_mnpo/logit": 0.1166459172964096, + "ht_mnpo/residual": 0.10914590209722519, + "ht_mnpo/residual_abs": 0.2188396006822586, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8368394534823635e-07, + "logits/chosen": -2.192263126373291, + "logits/rejected": -1.9854545593261719, + "logps/chosen": -0.28942209482192993, + "logps/rejected": -0.32335734367370605, + "loss": 0.8017171621322632, + "loss/core": 0.7787948846817017, + "loss/preference_sft": 0.28942209482192993, + "loss/reference_anchor": 1.1171759366989136, + "rewards/accuracies": 0.8125, + "rewards/chosen": 3.429896593093872, + "rewards/margins": 1.1664592027664185, + "rewards/rejected": 2.2634377479553223, + "step": 370 + }, + { + "drift/chosen_abs": 0.15755537152290344, + "drift/rejected_abs": 0.12303932011127472, + "epoch": 0.35835871707579287, + "grad_norm": 11.5, + "ht_mnpo/logit": 0.03453439846634865, + "ht_mnpo/residual": 0.027034390717744827, + "ht_mnpo/residual_abs": 0.11411330848932266, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8153271981852968e-07, + "logits/chosen": -2.3654236793518066, + "logits/rejected": -2.1134047508239746, + "logps/chosen": -0.26897451281547546, + "logps/rejected": -0.2847447991371155, + "loss": 0.10358532518148422, + "loss/core": 0.10106565803289413, + "loss/preference_sft": 0.26897451281547546, + "loss/reference_anchor": 0.09908583015203476, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.5745567083358765, + "rewards/margins": 0.3453438878059387, + "rewards/rejected": 1.2292128801345825, + "step": 375 + }, + { + "drift/chosen_abs": 0.25554612278938293, + "drift/rejected_abs": 0.20725803077220917, + "epoch": 0.36313683330347013, + "grad_norm": 39.0, + "ht_mnpo/logit": 0.048823170363903046, + "ht_mnpo/residual": 0.04132317006587982, + "ht_mnpo/residual_abs": 0.16914990544319153, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.7936023464491812e-07, + "logits/chosen": -2.299844980239868, + "logits/rejected": -2.0326662063598633, + "logps/chosen": -0.3033023476600647, + "logps/rejected": -0.32690924406051636, + "loss": 0.3187548518180847, + "loss/core": 0.3046416640281677, + "loss/preference_sft": 0.3033023476600647, + "loss/reference_anchor": 0.6753290891647339, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.554044008255005, + "rewards/margins": 0.48823174834251404, + "rewards/rejected": 2.065812110900879, + "step": 380 + }, + { + "drift/chosen_abs": 0.19318945705890656, + "drift/rejected_abs": 0.09826628863811493, + "epoch": 0.36791494953114734, + "grad_norm": 23.75, + "ht_mnpo/logit": 0.09533454477787018, + "ht_mnpo/residual": 0.08783452957868576, + "ht_mnpo/residual_abs": 0.09699156880378723, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.7716730681022723e-07, + "logits/chosen": -2.2895002365112305, + "logits/rejected": -2.017604351043701, + "logps/chosen": -0.2920383810997009, + "logps/rejected": -0.2935320734977722, + "loss": 0.07157917320728302, + "loss/core": 0.06837110221385956, + "loss/preference_sft": 0.2920383810997009, + "loss/reference_anchor": 0.1311999261379242, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.9301793575286865, + "rewards/margins": 0.9533454775810242, + "rewards/rejected": 0.9768339991569519, + "step": 385 + }, + { + "drift/chosen_abs": 0.176864892244339, + "drift/rejected_abs": 0.11490440368652344, + "epoch": 0.3726930657588246, + "grad_norm": 62.5, + "ht_mnpo/logit": 0.07121428102254868, + "ht_mnpo/residual": 0.06371428817510605, + "ht_mnpo/residual_abs": 0.12565705180168152, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.7495476098493152e-07, + "logits/chosen": -1.994593858718872, + "logits/rejected": -1.758146047592163, + "logps/chosen": -0.2808839976787567, + "logps/rejected": -0.3004426062107086, + "loss": 0.32486021518707275, + "loss/core": 0.3205881714820862, + "loss/preference_sft": 0.2808839976787567, + "loss/reference_anchor": 0.1855139434337616, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": 1.767156958580017, + "rewards/margins": 0.712142825126648, + "rewards/rejected": 1.0550141334533691, + "step": 390 + }, + { + "drift/chosen_abs": 0.15783588588237762, + "drift/rejected_abs": 0.10911662876605988, + "epoch": 0.3774711819865018, + "grad_norm": 1.2265625, + "ht_mnpo/logit": 0.04880943149328232, + "ht_mnpo/residual": 0.04130943864583969, + "ht_mnpo/residual_abs": 0.06264761835336685, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.7272342921702937e-07, + "logits/chosen": -2.422060966491699, + "logits/rejected": -2.158655881881714, + "logps/chosen": -0.26892244815826416, + "logps/rejected": -0.2726554572582245, + "loss": 0.026511918753385544, + "loss/core": 0.023896247148513794, + "loss/preference_sft": 0.26892244815826416, + "loss/reference_anchor": 0.10389119386672974, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.5771405696868896, + "rewards/margins": 0.48809438943862915, + "rewards/rejected": 1.0890462398529053, + "step": 395 + }, + { + "drift/chosen_abs": 0.4108216166496277, + "drift/rejected_abs": 0.15476536750793457, + "epoch": 0.3822492982141791, + "grad_norm": 3.125, + "ht_mnpo/logit": 0.25611841678619385, + "ht_mnpo/residual": 0.24861836433410645, + "ht_mnpo/residual_abs": 0.2766857445240021, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.7047415061914393e-07, + "logits/chosen": -2.0853142738342285, + "logits/rejected": -1.8839391469955444, + "logps/chosen": -0.28590038418769836, + "logps/rejected": -0.30821120738983154, + "loss": 1.28208589553833, + "loss/core": 1.2619318962097168, + "loss/preference_sft": 0.28590038418769836, + "loss/reference_anchor": 0.9791082143783569, + "rewards/accuracies": 0.875, + "rewards/chosen": 4.108216285705566, + "rewards/margins": 2.5611839294433594, + "rewards/rejected": 1.547032117843628, + "step": 400 + }, + { + "drift/chosen_abs": 0.16348758339881897, + "drift/rejected_abs": 0.11988557875156403, + "epoch": 0.3870274144418563, + "grad_norm": 0.953125, + "ht_mnpo/logit": 0.04386948049068451, + "ht_mnpo/residual": 0.03636947274208069, + "ht_mnpo/residual_abs": 0.06873659044504166, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.6820777105296707e-07, + "logits/chosen": -2.353846788406372, + "logits/rejected": -2.104539394378662, + "logps/chosen": -0.2770633399486542, + "logps/rejected": -0.2738936245441437, + "loss": 0.0407264307141304, + "loss/core": 0.037368860095739365, + "loss/preference_sft": 0.2770633399486542, + "loss/reference_anchor": 0.14017218351364136, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.6344867944717407, + "rewards/margins": 0.43869471549987793, + "rewards/rejected": 1.1957919597625732, + "step": 405 + }, + { + "drift/chosen_abs": 0.20585384964942932, + "drift/rejected_abs": 0.1777520775794983, + "epoch": 0.39180553066953355, + "grad_norm": 126.0, + "ht_mnpo/logit": 0.028088754042983055, + "ht_mnpo/residual": 0.020588750019669533, + "ht_mnpo/residual_abs": 0.1738569438457489, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.6592514281116553e-07, + "logits/chosen": -2.227536678314209, + "logits/rejected": -2.0844969749450684, + "logps/chosen": -0.27204984426498413, + "logps/rejected": -0.2747117877006531, + "loss": 0.3877445161342621, + "loss/core": 0.38092032074928284, + "loss/preference_sft": 0.27204984426498413, + "loss/reference_anchor": 0.3140057921409607, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.0569136142730713, + "rewards/margins": 0.2808874249458313, + "rewards/rejected": 1.7760260105133057, + "step": 410 + }, + { + "drift/chosen_abs": 0.16413165628910065, + "drift/rejected_abs": 0.13401861488819122, + "epoch": 0.39658364689721076, + "grad_norm": 33.5, + "ht_mnpo/logit": 0.030141586437821388, + "ht_mnpo/residual": 0.022641588002443314, + "ht_mnpo/residual_abs": 0.08408423513174057, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.636271242968684e-07, + "logits/chosen": -2.1277430057525635, + "logits/rejected": -1.9705203771591187, + "logps/chosen": -0.29165926575660706, + "logps/rejected": -0.2823432385921478, + "loss": 0.05560864135622978, + "loss/core": 0.05242307111620903, + "loss/preference_sft": 0.29165926575660706, + "loss/reference_anchor": 0.13011261820793152, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.6413164138793945, + "rewards/margins": 0.30141589045524597, + "rewards/rejected": 1.3399007320404053, + "step": 415 + }, + { + "drift/chosen_abs": 0.25538012385368347, + "drift/rejected_abs": 0.09950107336044312, + "epoch": 0.401361763124888, + "grad_norm": 580.0, + "ht_mnpo/logit": 0.16329319775104523, + "ht_mnpo/residual": 0.1557932049036026, + "ht_mnpo/residual_abs": 0.19033615291118622, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.6131457970085684e-07, + "logits/chosen": -2.0850791931152344, + "logits/rejected": -1.8701903820037842, + "logps/chosen": -0.28022292256355286, + "logps/rejected": -0.2937292456626892, + "loss": 0.26428142189979553, + "loss/core": 0.26025694608688354, + "loss/preference_sft": 0.28022292256355286, + "loss/reference_anchor": 0.17319951951503754, + "rewards/accuracies": 0.75, + "rewards/chosen": 2.552206516265869, + "rewards/margins": 1.6329319477081299, + "rewards/rejected": 0.9192743301391602, + "step": 420 + }, + { + "drift/chosen_abs": 0.22731247544288635, + "drift/rejected_abs": 0.19945786893367767, + "epoch": 0.40613987935256524, + "grad_norm": 0.60546875, + "ht_mnpo/logit": 0.02869640663266182, + "ht_mnpo/residual": 0.021196408197283745, + "ht_mnpo/residual_abs": 0.17983601987361908, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.5898837867657727e-07, + "logits/chosen": -2.156146764755249, + "logits/rejected": -1.908090353012085, + "logps/chosen": -0.29179322719573975, + "logps/rejected": -0.3070332705974579, + "loss": 0.43886813521385193, + "loss/core": 0.4307543635368347, + "loss/preference_sft": 0.29179322719573975, + "loss/reference_anchor": 0.3765077292919159, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.2724695205688477, + "rewards/margins": 0.2869641184806824, + "rewards/rejected": 1.9855053424835205, + "step": 425 + }, + { + "drift/chosen_abs": 0.26135116815567017, + "drift/rejected_abs": 0.1429075300693512, + "epoch": 0.4109179955802425, + "grad_norm": 432.0, + "ht_mnpo/logit": 0.1191127672791481, + "ht_mnpo/residual": 0.11161275207996368, + "ht_mnpo/residual_abs": 0.13684937357902527, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.5664939601310023e-07, + "logits/chosen": -2.198462963104248, + "logits/rejected": -1.987441062927246, + "logps/chosen": -0.2752520442008972, + "logps/rejected": -0.28098762035369873, + "loss": 0.267617791891098, + "loss/core": 0.25560644268989563, + "loss/preference_sft": 0.2752520442008972, + "loss/reference_anchor": 0.5730425119400024, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.6121487617492676, + "rewards/margins": 1.1911275386810303, + "rewards/rejected": 1.4210212230682373, + "step": 430 + }, + { + "drift/chosen_abs": 0.1936320811510086, + "drift/rejected_abs": 0.11375801265239716, + "epoch": 0.4156961118079197, + "grad_norm": 11.6875, + "ht_mnpo/logit": 0.08014325797557831, + "ht_mnpo/residual": 0.07264326512813568, + "ht_mnpo/residual_abs": 0.09690365940332413, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.5429851130614807e-07, + "logits/chosen": -2.1249842643737793, + "logits/rejected": -1.8592697381973267, + "logps/chosen": -0.3011928200721741, + "logps/rejected": -0.31247323751449585, + "loss": 0.05613823980093002, + "loss/core": 0.05347650125622749, + "loss/preference_sft": 0.3011928200721741, + "loss/reference_anchor": 0.10296765714883804, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.9342870712280273, + "rewards/margins": 0.8014326095581055, + "rewards/rejected": 1.1328542232513428, + "step": 435 + }, + { + "drift/chosen_abs": 0.2973514199256897, + "drift/rejected_abs": 0.10415633022785187, + "epoch": 0.420474228035597, + "grad_norm": 476.0, + "ht_mnpo/logit": 0.22436299920082092, + "ht_mnpo/residual": 0.21686296164989471, + "ht_mnpo/residual_abs": 0.2422914057970047, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.51936608627315e-07, + "logits/chosen": -2.127042293548584, + "logits/rejected": -1.8621797561645508, + "logps/chosen": -0.30403512716293335, + "logps/rejected": -0.3232555389404297, + "loss": 1.0134453773498535, + "loss/core": 1.0037100315093994, + "loss/preference_sft": 0.30403512716293335, + "loss/reference_anchor": 0.4563591480255127, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.9734644889831543, + "rewards/margins": 2.2436299324035645, + "rewards/rejected": 0.7298344373703003, + "step": 440 + }, + { + "drift/chosen_abs": 0.19231143593788147, + "drift/rejected_abs": 0.09996972978115082, + "epoch": 0.4252523442632742, + "grad_norm": 31.875, + "ht_mnpo/logit": 0.09261570125818253, + "ht_mnpo/residual": 0.0851157084107399, + "ht_mnpo/residual_abs": 0.09635229408740997, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.4956457619160375e-07, + "logits/chosen": -2.433109760284424, + "logits/rejected": -2.0341031551361084, + "logps/chosen": -0.29921525716781616, + "logps/rejected": -0.3007660508155823, + "loss": 0.08675873279571533, + "loss/core": 0.0813307985663414, + "loss/preference_sft": 0.29921525716781616, + "loss/reference_anchor": 0.2414754182100296, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.9225269556045532, + "rewards/margins": 0.9261569976806641, + "rewards/rejected": 0.9963698387145996, + "step": 445 + }, + { + "drift/chosen_abs": 0.15865620970726013, + "drift/rejected_abs": 0.11887378990650177, + "epoch": 0.43003046049095145, + "grad_norm": 152.0, + "ht_mnpo/logit": 0.0395728163421154, + "ht_mnpo/residual": 0.03207281976938248, + "ht_mnpo/residual_abs": 0.12159354984760284, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.471833060234044e-07, + "logits/chosen": -2.210747241973877, + "logits/rejected": -1.945389986038208, + "logps/chosen": -0.2810204029083252, + "logps/rejected": -0.30275437235832214, + "loss": 0.1328326165676117, + "loss/core": 0.129548579454422, + "loss/preference_sft": 0.2810204029083252, + "loss/reference_anchor": 0.1361009180545807, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 1.5836235284805298, + "rewards/margins": 0.3957282602787018, + "rewards/rejected": 1.1878951787948608, + "step": 450 + }, + { + "drift/chosen_abs": 0.25045090913772583, + "drift/rejected_abs": 0.13791918754577637, + "epoch": 0.43480857671862866, + "grad_norm": 100.0, + "ht_mnpo/logit": 0.112596794962883, + "ht_mnpo/residual": 0.10509680211544037, + "ht_mnpo/residual_abs": 0.15906308591365814, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.4479369362104037e-07, + "logits/chosen": -2.2764220237731934, + "logits/rejected": -2.0353825092315674, + "logps/chosen": -0.29567039012908936, + "logps/rejected": -0.30568239092826843, + "loss": 0.3157535195350647, + "loss/core": 0.30716970562934875, + "loss/preference_sft": 0.29567039012908936, + "loss/reference_anchor": 0.3996273875236511, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.50372576713562, + "rewards/margins": 1.1259678602218628, + "rewards/rejected": 1.3777577877044678, + "step": 455 + }, + { + "drift/chosen_abs": 0.1878124624490738, + "drift/rejected_abs": 0.14636285603046417, + "epoch": 0.4395866929463059, + "grad_norm": 3.53125, + "ht_mnpo/logit": 0.04185149818658829, + "ht_mnpo/residual": 0.03435150161385536, + "ht_mnpo/residual_abs": 0.07132266461849213, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.4239663762000817e-07, + "logits/chosen": -2.317431688308716, + "logits/rejected": -2.0158653259277344, + "logps/chosen": -0.2637852132320404, + "logps/rejected": -0.256014883518219, + "loss": 0.026135962456464767, + "loss/core": 0.023039747029542923, + "loss/preference_sft": 0.2637852132320404, + "loss/reference_anchor": 0.1284322738647461, + "rewards/accuracies": 0.75, + "rewards/chosen": 1.878124475479126, + "rewards/margins": 0.41851502656936646, + "rewards/rejected": 1.4596095085144043, + "step": 460 + }, + { + "drift/chosen_abs": 0.2502024471759796, + "drift/rejected_abs": 0.16358964145183563, + "epoch": 0.44436480917398313, + "grad_norm": 17.75, + "ht_mnpo/logit": 0.0867425948381424, + "ht_mnpo/residual": 0.07924258708953857, + "ht_mnpo/residual_abs": 0.15044549107551575, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.3999303945503747e-07, + "logits/chosen": -2.1240248680114746, + "logits/rejected": -1.9239466190338135, + "logps/chosen": -0.31266671419143677, + "logps/rejected": -0.3123919367790222, + "loss": 0.19001127779483795, + "loss/core": 0.183534175157547, + "loss/preference_sft": 0.31266671419143677, + "loss/reference_anchor": 0.2925878167152405, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.501178741455078, + "rewards/margins": 0.8674259185791016, + "rewards/rejected": 1.6337528228759766, + "step": 465 + }, + { + "drift/chosen_abs": 0.10216370970010757, + "drift/rejected_abs": 0.12402309477329254, + "epoch": 0.4491429254016604, + "grad_norm": 117.0, + "ht_mnpo/logit": -0.02179349772632122, + "ht_mnpo/residual": -0.029293496161699295, + "ht_mnpo/residual_abs": 0.09518697112798691, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.3758380302109808e-07, + "logits/chosen": -2.449554681777954, + "logits/rejected": -2.152750015258789, + "logps/chosen": -0.27131548523902893, + "logps/rejected": -0.2756040394306183, + "loss": 0.09835384786128998, + "loss/core": 0.09614761918783188, + "loss/preference_sft": 0.27131548523902893, + "loss/reference_anchor": 0.0831795483827591, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.021196961402893, + "rewards/margins": -0.21793493628501892, + "rewards/rejected": 1.2391319274902344, + "step": 470 + }, + { + "drift/chosen_abs": 0.15605005621910095, + "drift/rejected_abs": 0.09966011345386505, + "epoch": 0.4539210416293376, + "grad_norm": 86.0, + "ht_mnpo/logit": 0.05629577487707138, + "ht_mnpo/residual": 0.048795782029628754, + "ht_mnpo/residual_abs": 0.1054389700293541, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.351698343334823e-07, + "logits/chosen": -2.2466909885406494, + "logits/rejected": -1.9857765436172485, + "logps/chosen": -0.2887324392795563, + "logps/rejected": -0.28986722230911255, + "loss": 0.1072268858551979, + "loss/core": 0.10453776270151138, + "loss/preference_sft": 0.2887324392795563, + "loss/reference_anchor": 0.10558246076107025, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.5577256679534912, + "rewards/margins": 0.5629578828811646, + "rewards/rejected": 0.9947676658630371, + "step": 475 + }, + { + "drift/chosen_abs": 0.1687658131122589, + "drift/rejected_abs": 0.13127224147319794, + "epoch": 0.45869915785701487, + "grad_norm": 2.265625, + "ht_mnpo/logit": 0.03775458782911301, + "ht_mnpo/residual": 0.030254587531089783, + "ht_mnpo/residual_abs": 0.09340941905975342, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.3275204118708942e-07, + "logits/chosen": -2.268664598464966, + "logits/rejected": -2.0399532318115234, + "logps/chosen": -0.27916237711906433, + "logps/rejected": -0.2935280203819275, + "loss": 0.08096377551555634, + "loss/core": 0.0781911164522171, + "loss/preference_sft": 0.27916237711906433, + "loss/reference_anchor": 0.11071737110614777, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 1.6864219903945923, + "rewards/margins": 0.37754589319229126, + "rewards/rejected": 1.3088760375976562, + "step": 480 + }, + { + "drift/chosen_abs": 0.22259202599525452, + "drift/rejected_abs": 0.16338473558425903, + "epoch": 0.46347727408469214, + "grad_norm": 21.875, + "ht_mnpo/logit": 0.05938667058944702, + "ht_mnpo/residual": 0.0518866702914238, + "ht_mnpo/residual_abs": 0.1358523964881897, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.3033133281504132e-07, + "logits/chosen": -2.1293797492980957, + "logits/rejected": -1.8755149841308594, + "logps/chosen": -0.2896140217781067, + "logps/rejected": -0.2774605453014374, + "loss": 0.21404489874839783, + "loss/core": 0.20932647585868835, + "loss/preference_sft": 0.2896140217781067, + "loss/reference_anchor": 0.2069588154554367, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.225459098815918, + "rewards/margins": 0.5938667058944702, + "rewards/rejected": 1.6315925121307373, + "step": 485 + }, + { + "drift/chosen_abs": 0.24862901866436005, + "drift/rejected_abs": 0.13741818070411682, + "epoch": 0.46825539031236935, + "grad_norm": 14.125, + "ht_mnpo/logit": 0.11346457898616791, + "ht_mnpo/residual": 0.10596457868814468, + "ht_mnpo/residual_abs": 0.14489370584487915, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.2790861954675702e-07, + "logits/chosen": -2.007351875305176, + "logits/rejected": -1.7956119775772095, + "logps/chosen": -0.2839955687522888, + "logps/rejected": -0.28634992241859436, + "loss": 0.21525898575782776, + "loss/core": 0.21072621643543243, + "loss/preference_sft": 0.2839955687522888, + "loss/reference_anchor": 0.19823768734931946, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.4841885566711426, + "rewards/margins": 1.1346458196640015, + "rewards/rejected": 1.3495427370071411, + "step": 490 + }, + { + "drift/chosen_abs": 0.14347688853740692, + "drift/rejected_abs": 0.09557531028985977, + "epoch": 0.4730335065400466, + "grad_norm": 16.375, + "ht_mnpo/logit": 0.048013053834438324, + "ht_mnpo/residual": 0.0405130572617054, + "ht_mnpo/residual_abs": 0.09559812396764755, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.2548481246561504e-07, + "logits/chosen": -2.1509878635406494, + "logits/rejected": -1.9557958841323853, + "logps/chosen": -0.29566025733947754, + "logps/rejected": -0.31424885988235474, + "loss": 0.09365914762020111, + "loss/core": 0.09119492024183273, + "loss/preference_sft": 0.29566025733947754, + "loss/reference_anchor": 0.09364519268274307, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.4331740140914917, + "rewards/margins": 0.48013052344322205, + "rewards/rejected": 0.9530435800552368, + "step": 495 + }, + { + "drift/chosen_abs": 0.2629546821117401, + "drift/rejected_abs": 0.2393544614315033, + "epoch": 0.4778116227677238, + "grad_norm": 68.5, + "ht_mnpo/logit": 0.024394646286964417, + "ht_mnpo/residual": 0.016894647851586342, + "ht_mnpo/residual_abs": 0.20083613693714142, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.230608230663321e-07, + "logits/chosen": -2.3212738037109375, + "logits/rejected": -2.0327229499816895, + "logps/chosen": -0.27783674001693726, + "logps/rejected": -0.2940555810928345, + "loss": 0.32842302322387695, + "loss/core": 0.31986716389656067, + "loss/preference_sft": 0.27783674001693726, + "loss/reference_anchor": 0.40000849962234497, + "rewards/accuracies": 0.75, + "rewards/chosen": 2.6291868686676025, + "rewards/margins": 0.24394643306732178, + "rewards/rejected": 2.385240316390991, + "step": 500 + }, + { + "drift/chosen_abs": 0.2415897399187088, + "drift/rejected_abs": 0.13792410492897034, + "epoch": 0.4825897389954011, + "grad_norm": 36.75, + "ht_mnpo/logit": 0.10351978242397308, + "ht_mnpo/residual": 0.09601978212594986, + "ht_mnpo/residual_abs": 0.17322488129138947, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.206375629121874e-07, + "logits/chosen": -2.1131768226623535, + "logits/rejected": -1.8549270629882812, + "logps/chosen": -0.2813659906387329, + "logps/rejected": -0.28420156240463257, + "loss": 0.38091525435447693, + "loss/core": 0.37360432744026184, + "loss/preference_sft": 0.2813659906387329, + "loss/reference_anchor": 0.33741098642349243, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.4129114151000977, + "rewards/margins": 1.0351978540420532, + "rewards/rejected": 1.3777135610580444, + "step": 505 + }, + { + "drift/chosen_abs": 0.18775442242622375, + "drift/rejected_abs": 0.1655738800764084, + "epoch": 0.4873678552230783, + "grad_norm": 2.984375, + "ht_mnpo/logit": 0.022700464352965355, + "ht_mnpo/residual": 0.015200458467006683, + "ht_mnpo/residual_abs": 0.09576776623725891, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.1821594329222079e-07, + "logits/chosen": -2.3727898597717285, + "logits/rejected": -2.067782402038574, + "logps/chosen": -0.27853554487228394, + "logps/rejected": -0.296770304441452, + "loss": 0.06544214487075806, + "loss/core": 0.06130026653409004, + "loss/preference_sft": 0.27853554487228394, + "loss/reference_anchor": 0.17924094200134277, + "rewards/accuracies": 0.75, + "rewards/chosen": 1.8766510486602783, + "rewards/margins": 0.22700461745262146, + "rewards/rejected": 1.649646520614624, + "step": 510 + }, + { + "drift/chosen_abs": 0.2205163985490799, + "drift/rejected_abs": 0.15725180506706238, + "epoch": 0.49214597145075556, + "grad_norm": 0.453125, + "ht_mnpo/logit": 0.06345703452825546, + "ht_mnpo/residual": 0.05595703050494194, + "ht_mnpo/residual_abs": 0.1105417013168335, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.157968748785344e-07, + "logits/chosen": -2.234668016433716, + "logits/rejected": -1.8898760080337524, + "logps/chosen": -0.28748154640197754, + "logps/rejected": -0.2847779393196106, + "loss": 0.09203025698661804, + "loss/core": 0.08730658143758774, + "loss/preference_sft": 0.28748154640197754, + "loss/reference_anchor": 0.20743636786937714, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 2.2042317390441895, + "rewards/margins": 0.634570300579071, + "rewards/rejected": 1.5696613788604736, + "step": 515 + }, + { + "drift/chosen_abs": 0.27758151292800903, + "drift/rejected_abs": 0.15809740126132965, + "epoch": 0.49692408767843277, + "grad_norm": 36.5, + "ht_mnpo/logit": 0.11959369480609894, + "ht_mnpo/residual": 0.11209367215633392, + "ht_mnpo/residual_abs": 0.2109643965959549, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.1338126738382597e-07, + "logits/chosen": -2.270226001739502, + "logits/rejected": -2.025266170501709, + "logps/chosen": -0.2758481800556183, + "logps/rejected": -0.26102808117866516, + "loss": 0.30472052097320557, + "loss/core": 0.2978774905204773, + "loss/preference_sft": 0.2758481800556183, + "loss/reference_anchor": 0.31456848978996277, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.7751450538635254, + "rewards/margins": 1.195936918258667, + "rewards/rejected": 1.5792083740234375, + "step": 520 + }, + { + "drift/chosen_abs": 0.2691987454891205, + "drift/rejected_abs": 0.15224085748195648, + "epoch": 0.50170220390611, + "grad_norm": 207.0, + "ht_mnpo/logit": 0.12795712053775787, + "ht_mnpo/residual": 0.12045712769031525, + "ht_mnpo/residual_abs": 0.2664859890937805, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.1097002921928316e-07, + "logits/chosen": -2.1398863792419434, + "logits/rejected": -1.8041080236434937, + "logps/chosen": -0.30692705512046814, + "logps/rejected": -0.33052974939346313, + "loss": 0.7266106009483337, + "loss/core": 0.718201756477356, + "loss/preference_sft": 0.30692705512046814, + "loss/reference_anchor": 0.3897519111633301, + "rewards/accuracies": 0.75, + "rewards/chosen": 2.6892130374908447, + "rewards/margins": 1.279571294784546, + "rewards/rejected": 1.409641981124878, + "step": 525 + }, + { + "drift/chosen_abs": 0.25865083932876587, + "drift/rejected_abs": 0.0813761055469513, + "epoch": 0.5064803201337873, + "grad_norm": 19.75, + "ht_mnpo/logit": 0.1827838122844696, + "ht_mnpo/residual": 0.17528381943702698, + "ht_mnpo/residual_abs": 0.19191543757915497, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0856406715296717e-07, + "logits/chosen": -2.2501230239868164, + "logits/rejected": -1.9791781902313232, + "logps/chosen": -0.2921066880226135, + "logps/rejected": -0.30198463797569275, + "loss": 0.7703770399093628, + "loss/core": 0.7617815136909485, + "loss/preference_sft": 0.2921066880226135, + "loss/reference_anchor": 0.4005637764930725, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.5850157737731934, + "rewards/margins": 1.8278381824493408, + "rewards/rejected": 0.7571774125099182, + "step": 530 + }, + { + "drift/chosen_abs": 0.21662993729114532, + "drift/rejected_abs": 0.2024078369140625, + "epoch": 0.5112584363614645, + "grad_norm": 3.640625, + "ht_mnpo/logit": 0.014330679550766945, + "ht_mnpo/residual": 0.006830704398453236, + "ht_mnpo/residual_abs": 0.24278083443641663, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.061642859688146e-07, + "logits/chosen": -2.3559963703155518, + "logits/rejected": -2.213176727294922, + "logps/chosen": -0.31593745946884155, + "logps/rejected": -0.317546546459198, + "loss": 1.2009260654449463, + "loss/core": 1.1859214305877686, + "loss/preference_sft": 0.31593745946884155, + "loss/reference_anchor": 0.7186298370361328, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.1661620140075684, + "rewards/margins": 0.1433069258928299, + "rewards/rejected": 2.022855281829834, + "step": 535 + }, + { + "drift/chosen_abs": 0.19005665183067322, + "drift/rejected_abs": 0.07541195303201675, + "epoch": 0.5160365525891417, + "grad_norm": 6.3125, + "ht_mnpo/logit": 0.11505275964736938, + "ht_mnpo/residual": 0.10755278170108795, + "ht_mnpo/residual_abs": 0.13360702991485596, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0377158812638491e-07, + "logits/chosen": -2.269131898880005, + "logits/rejected": -2.05694317817688, + "logps/chosen": -0.3238433599472046, + "logps/rejected": -0.3159184157848358, + "loss": 0.26619070768356323, + "loss/core": 0.2617434561252594, + "loss/preference_sft": 0.3238433599472046, + "loss/reference_anchor": 0.18997910618782043, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.9001648426055908, + "rewards/margins": 1.1505277156829834, + "rewards/rejected": 0.7496371865272522, + "step": 540 + }, + { + "drift/chosen_abs": 0.2369406670331955, + "drift/rejected_abs": 0.0789075642824173, + "epoch": 0.5208146688168189, + "grad_norm": 13.875, + "ht_mnpo/logit": 0.15915516018867493, + "ht_mnpo/residual": 0.1516551524400711, + "ht_mnpo/residual_abs": 0.1720009595155716, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0138687342148249e-07, + "logits/chosen": -2.434389352798462, + "logits/rejected": -2.192600727081299, + "logps/chosen": -0.28083503246307373, + "logps/rejected": -0.2732694745063782, + "loss": 0.35217469930648804, + "loss/core": 0.34702593088150024, + "loss/preference_sft": 0.28083503246307373, + "loss/reference_anchor": 0.229354590177536, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.3685014247894287, + "rewards/margins": 1.5915515422821045, + "rewards/rejected": 0.7769501209259033, + "step": 545 + }, + { + "drift/chosen_abs": 0.2724258005619049, + "drift/rejected_abs": 0.12468817085027695, + "epoch": 0.5255927850444962, + "grad_norm": 2.078125, + "ht_mnpo/logit": 0.14971444010734558, + "ht_mnpo/residual": 0.14221441745758057, + "ht_mnpo/residual_abs": 0.1590646654367447, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.90110386477801e-08, + "logits/chosen": -2.0379040241241455, + "logits/rejected": -1.7684564590454102, + "logps/chosen": -0.3047271966934204, + "logps/rejected": -0.3073706328868866, + "loss": 0.3628101348876953, + "loss/core": 0.3507049083709717, + "loss/preference_sft": 0.3047271966934204, + "loss/reference_anchor": 0.5747870206832886, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.7242579460144043, + "rewards/margins": 1.497144341468811, + "rewards/rejected": 1.227113962173462, + "step": 550 + }, + { + "drift/chosen_abs": 0.31042712926864624, + "drift/rejected_abs": 0.18171672523021698, + "epoch": 0.5303709012721735, + "grad_norm": 1568.0, + "ht_mnpo/logit": 0.12890593707561493, + "ht_mnpo/residual": 0.1214059591293335, + "ht_mnpo/residual_abs": 0.24321047961711884, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.664497725957164e-08, + "logits/chosen": -2.0988707542419434, + "logits/rejected": -1.9185583591461182, + "logps/chosen": -0.2986465096473694, + "logps/rejected": -0.29767924547195435, + "loss": 1.2037521600723267, + "loss/core": 1.1875665187835693, + "loss/preference_sft": 0.2986465096473694, + "loss/reference_anchor": 0.7794210314750671, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.103039026260376, + "rewards/margins": 1.2890592813491821, + "rewards/rejected": 1.8139795064926147, + "step": 555 + }, + { + "drift/chosen_abs": 0.19171099364757538, + "drift/rejected_abs": 0.10674883425235748, + "epoch": 0.5351490174998507, + "grad_norm": 0.10498046875, + "ht_mnpo/logit": 0.08548573404550552, + "ht_mnpo/residual": 0.0779857337474823, + "ht_mnpo/residual_abs": 0.10989482700824738, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.428957903578045e-08, + "logits/chosen": -2.2540018558502197, + "logits/rejected": -2.0119833946228027, + "logps/chosen": -0.26887863874435425, + "logps/rejected": -0.26674965023994446, + "loss": 0.09468104690313339, + "loss/core": 0.09218241274356842, + "loss/preference_sft": 0.26887863874435425, + "loss/reference_anchor": 0.09804411232471466, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.915593147277832, + "rewards/margins": 0.8548572659492493, + "rewards/rejected": 1.060735821723938, + "step": 560 + }, + { + "drift/chosen_abs": 0.2377406656742096, + "drift/rejected_abs": 0.20291483402252197, + "epoch": 0.5399271337275279, + "grad_norm": 51.25, + "ht_mnpo/logit": 0.0349448099732399, + "ht_mnpo/residual": 0.027444809675216675, + "ht_mnpo/residual_abs": 0.16346418857574463, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.194572974534976e-08, + "logits/chosen": -1.9928922653198242, + "logits/rejected": -1.8542598485946655, + "logps/chosen": -0.2798479497432709, + "logps/rejected": -0.2841023802757263, + "loss": 0.23373666405677795, + "loss/core": 0.22751207649707794, + "loss/preference_sft": 0.2798479497432709, + "loss/reference_anchor": 0.2832440733909607, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.37677001953125, + "rewards/margins": 0.3494481146335602, + "rewards/rejected": 2.0273215770721436, + "step": 565 + }, + { + "drift/chosen_abs": 0.15430274605751038, + "drift/rejected_abs": 0.1468575894832611, + "epoch": 0.5447052499552052, + "grad_norm": 40.75, + "ht_mnpo/logit": 0.0075254677794873714, + "ht_mnpo/residual": 2.5472789275227115e-05, + "ht_mnpo/residual_abs": 0.1423179656267166, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.96143108141412e-08, + "logits/chosen": -2.259716749191284, + "logits/rejected": -2.0335276126861572, + "logps/chosen": -0.3092796802520752, + "logps/rejected": -0.30099526047706604, + "loss": 0.22612264752388, + "loss/core": 0.22155435383319855, + "loss/preference_sft": 0.3092796802520752, + "loss/reference_anchor": 0.19748570024967194, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 1.5426218509674072, + "rewards/margins": 0.0752546414732933, + "rewards/rejected": 1.46736741065979, + "step": 570 + }, + { + "drift/chosen_abs": 0.16749927401542664, + "drift/rejected_abs": 0.10397598892450333, + "epoch": 0.5494833661828824, + "grad_norm": 16.375, + "ht_mnpo/logit": 0.06339401006698608, + "ht_mnpo/residual": 0.05589400604367256, + "ht_mnpo/residual_abs": 0.06961973011493683, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.72961989934668e-08, + "logits/chosen": -2.3487277030944824, + "logits/rejected": -2.1463935375213623, + "logps/chosen": -0.26586031913757324, + "logps/rejected": -0.27124351263046265, + "loss": 0.035394370555877686, + "loss/core": 0.033535875380039215, + "loss/preference_sft": 0.26586031913757324, + "loss/reference_anchor": 0.06633836030960083, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.6726531982421875, + "rewards/margins": 0.6339401006698608, + "rewards/rejected": 1.0387132167816162, + "step": 575 + }, + { + "drift/chosen_abs": 0.17038491368293762, + "drift/rejected_abs": 0.12820149958133698, + "epoch": 0.5542614824105596, + "grad_norm": 239.0, + "ht_mnpo/logit": 0.04234672710299492, + "ht_mnpo/residual": 0.03484673053026199, + "ht_mnpo/residual_abs": 0.13217854499816895, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.499226603037854e-08, + "logits/chosen": -2.1979475021362305, + "logits/rejected": -1.981332540512085, + "logps/chosen": -0.3051069676876068, + "logps/rejected": -0.30111053586006165, + "loss": 0.24051013588905334, + "loss/core": 0.23614850640296936, + "loss/preference_sft": 0.3051069676876068, + "loss/reference_anchor": 0.18757155537605286, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 1.7025823593139648, + "rewards/margins": 0.4234672486782074, + "rewards/rejected": 1.2791149616241455, + "step": 580 + }, + { + "drift/chosen_abs": 0.13287939131259918, + "drift/rejected_abs": 0.12429249286651611, + "epoch": 0.5590395986382368, + "grad_norm": 9.375, + "ht_mnpo/logit": 0.008631383068859577, + "ht_mnpo/residual": 0.0011313877766951919, + "ht_mnpo/residual_abs": 0.11924134194850922, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.270337833983987e-08, + "logits/chosen": -2.3649332523345947, + "logits/rejected": -2.1235404014587402, + "logps/chosen": -0.2820286154747009, + "logps/rejected": -0.30536872148513794, + "loss": 0.13413414359092712, + "loss/core": 0.13124492764472961, + "loss/preference_sft": 0.2820286154747009, + "loss/reference_anchor": 0.11625870317220688, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.328194260597229, + "rewards/margins": 0.08631384372711182, + "rewards/rejected": 1.2418802976608276, + "step": 585 + }, + { + "drift/chosen_abs": 0.25080275535583496, + "drift/rejected_abs": 0.15849502384662628, + "epoch": 0.5638177148659141, + "grad_norm": 134.0, + "ht_mnpo/logit": 0.0924316793680191, + "ht_mnpo/residual": 0.08493169397115707, + "ht_mnpo/residual_abs": 0.20355942845344543, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.04303966789025e-08, + "logits/chosen": -2.183878183364868, + "logits/rejected": -2.0252556800842285, + "logps/chosen": -0.2765917181968689, + "logps/rejected": -0.27511608600616455, + "loss": 0.5015133619308472, + "loss/core": 0.49359574913978577, + "loss/preference_sft": 0.2765917181968689, + "loss/reference_anchor": 0.36821773648262024, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.5080275535583496, + "rewards/margins": 0.9243170022964478, + "rewards/rejected": 1.5837104320526123, + "step": 590 + }, + { + "drift/chosen_abs": 0.23994091153144836, + "drift/rejected_abs": 0.13920912146568298, + "epoch": 0.5685958310935914, + "grad_norm": 76.0, + "ht_mnpo/logit": 0.100993812084198, + "ht_mnpo/residual": 0.09349381923675537, + "ht_mnpo/residual_abs": 0.15858511626720428, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 7.817417582301098e-08, + "logits/chosen": -2.1511635780334473, + "logits/rejected": -2.000541925430298, + "logps/chosen": -0.28564023971557617, + "logps/rejected": -0.2844979763031006, + "loss": 0.19406889379024506, + "loss/core": 0.19000914692878723, + "loss/preference_sft": 0.28564023971557617, + "loss/reference_anchor": 0.174424409866333, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.398890972137451, + "rewards/margins": 1.0099382400512695, + "rewards/rejected": 1.3889529705047607, + "step": 595 + }, + { + "drift/chosen_abs": 0.1661892980337143, + "drift/rejected_abs": 0.14502601325511932, + "epoch": 0.5733739473212686, + "grad_norm": 1.375, + "ht_mnpo/logit": 0.022466612979769707, + "ht_mnpo/residual": 0.014966619201004505, + "ht_mnpo/residual_abs": 0.10731730610132217, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 7.59355642445566e-08, + "logits/chosen": -2.216158628463745, + "logits/rejected": -2.082313060760498, + "logps/chosen": -0.2715299725532532, + "logps/rejected": -0.27377790212631226, + "loss": 0.13416652381420135, + "loss/core": 0.130991131067276, + "loss/preference_sft": 0.2715299725532532, + "loss/reference_anchor": 0.13161684572696686, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.6614080667495728, + "rewards/margins": 0.22466608881950378, + "rewards/rejected": 1.4367419481277466, + "step": 600 + }, + { + "drift/chosen_abs": 0.1707453578710556, + "drift/rejected_abs": 0.1772371083498001, + "epoch": 0.5781520635489459, + "grad_norm": 524.0, + "ht_mnpo/logit": -0.006207427475601435, + "ht_mnpo/residual": -0.013707419857382774, + "ht_mnpo/residual_abs": 0.13802742958068848, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 7.37154037938015e-08, + "logits/chosen": -2.2585723400115967, + "logits/rejected": -1.9065723419189453, + "logps/chosen": -0.2895175814628601, + "logps/rejected": -0.3101463317871094, + "loss": 0.2447996586561203, + "loss/core": 0.24052676558494568, + "loss/preference_sft": 0.2895175814628601, + "loss/reference_anchor": 0.18469442427158356, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.7074534893035889, + "rewards/margins": -0.06207428127527237, + "rewards/rejected": 1.769527792930603, + "step": 605 + }, + { + "drift/chosen_abs": 0.36278414726257324, + "drift/rejected_abs": 0.23082184791564941, + "epoch": 0.5829301797766231, + "grad_norm": 872.0, + "ht_mnpo/logit": 0.13208219408988953, + "ht_mnpo/residual": 0.1245821937918663, + "ht_mnpo/residual_abs": 0.31827038526535034, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 7.151452938229325e-08, + "logits/chosen": -2.1874964237213135, + "logits/rejected": -1.9628692865371704, + "logps/chosen": -0.2622188925743103, + "logps/rejected": -0.27765434980392456, + "loss": 1.3239902257919312, + "loss/core": 1.3052964210510254, + "loss/preference_sft": 0.2622188925743103, + "loss/reference_anchor": 0.9084656834602356, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.6278419494628906, + "rewards/margins": 1.3208218812942505, + "rewards/rejected": 2.3070197105407715, + "step": 610 + }, + { + "drift/chosen_abs": 0.18900258839130402, + "drift/rejected_abs": 0.08098079264163971, + "epoch": 0.5877082960043003, + "grad_norm": 21.5, + "ht_mnpo/logit": 0.10846320539712906, + "ht_mnpo/residual": 0.10096319764852524, + "ht_mnpo/residual_abs": 0.11534751951694489, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.933376866888883e-08, + "logits/chosen": -2.2008748054504395, + "logits/rejected": -2.009446859359741, + "logps/chosen": -0.2769085466861725, + "logps/rejected": -0.29790812730789185, + "loss": 0.10706748068332672, + "loss/core": 0.1036563292145729, + "loss/preference_sft": 0.2769085466861725, + "loss/reference_anchor": 0.142866849899292, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.8900257349014282, + "rewards/margins": 1.0846320390701294, + "rewards/rejected": 0.8053938746452332, + "step": 615 + }, + { + "drift/chosen_abs": 0.238611102104187, + "drift/rejected_abs": 0.1267508566379547, + "epoch": 0.5924864122319775, + "grad_norm": 16.75, + "ht_mnpo/logit": 0.11258834600448608, + "ht_mnpo/residual": 0.10508836805820465, + "ht_mnpo/residual_abs": 0.15256913006305695, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.717394174850605e-08, + "logits/chosen": -2.3503708839416504, + "logits/rejected": -1.996098518371582, + "logps/chosen": -0.2766886353492737, + "logps/rejected": -0.2806140184402466, + "loss": 0.23508873581886292, + "loss/core": 0.23049023747444153, + "loss/preference_sft": 0.2766886353492737, + "loss/reference_anchor": 0.20225539803504944, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.3849875926971436, + "rewards/margins": 1.1258835792541504, + "rewards/rejected": 1.2591038942337036, + "step": 620 + }, + { + "drift/chosen_abs": 0.2566293179988861, + "drift/rejected_abs": 0.17416846752166748, + "epoch": 0.5972645284596548, + "grad_norm": 1.4609375, + "ht_mnpo/logit": 0.08759935945272446, + "ht_mnpo/residual": 0.08009935915470123, + "ht_mnpo/residual_abs": 0.19110621511936188, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.503586084371926e-08, + "logits/chosen": -2.2542307376861572, + "logits/rejected": -1.9469505548477173, + "logps/chosen": -0.2917229235172272, + "logps/rejected": -0.30690133571624756, + "loss": 0.4792788028717041, + "loss/core": 0.46942344307899475, + "loss/preference_sft": 0.2917229235172272, + "loss/reference_anchor": 0.46359556913375854, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 2.564573287963867, + "rewards/margins": 0.8759937286376953, + "rewards/rejected": 1.688579797744751, + "step": 625 + }, + { + "drift/chosen_abs": 0.19692203402519226, + "drift/rejected_abs": 0.11281520128250122, + "epoch": 0.602042644687332, + "grad_norm": 12.9375, + "ht_mnpo/logit": 0.08999810367822647, + "ht_mnpo/residual": 0.08249811083078384, + "ht_mnpo/residual_abs": 0.10153649747371674, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.29203299993155e-08, + "logits/chosen": -2.2926273345947266, + "logits/rejected": -2.0748157501220703, + "logps/chosen": -0.2932100296020508, + "logps/rejected": -0.29019203782081604, + "loss": 0.0861930325627327, + "loss/core": 0.08310763537883759, + "loss/preference_sft": 0.2932100296020508, + "loss/reference_anchor": 0.1249486431479454, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 1.968793511390686, + "rewards/margins": 0.8999810218811035, + "rewards/rejected": 1.068812608718872, + "step": 630 + }, + { + "drift/chosen_abs": 0.18911968171596527, + "drift/rejected_abs": 0.11328919231891632, + "epoch": 0.6068207609150092, + "grad_norm": 80.0, + "ht_mnpo/logit": 0.07597161084413528, + "ht_mnpo/residual": 0.06847161799669266, + "ht_mnpo/residual_abs": 0.10563920438289642, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.082814477992656e-08, + "logits/chosen": -2.048794984817505, + "logits/rejected": -1.8921749591827393, + "logps/chosen": -0.2832937240600586, + "logps/rejected": -0.3034849762916565, + "loss": 0.1296667456626892, + "loss/core": 0.1261918693780899, + "loss/preference_sft": 0.2832937240600586, + "loss/reference_anchor": 0.14541494846343994, + "rewards/accuracies": 0.75, + "rewards/chosen": 1.8911077976226807, + "rewards/margins": 0.7597162127494812, + "rewards/rejected": 1.1313915252685547, + "step": 635 + }, + { + "drift/chosen_abs": 0.21820612251758575, + "drift/rejected_abs": 0.14316417276859283, + "epoch": 0.6115988771426865, + "grad_norm": 1.2578125, + "ht_mnpo/logit": 0.07545202970504761, + "ht_mnpo/residual": 0.06795202940702438, + "ht_mnpo/residual_abs": 0.11145929247140884, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.87600919708494e-08, + "logits/chosen": -2.21930193901062, + "logits/rejected": -1.9665937423706055, + "logps/chosen": -0.26268836855888367, + "logps/rejected": -0.27242428064346313, + "loss": 0.09963403642177582, + "loss/core": 0.09529232233762741, + "loss/preference_sft": 0.26268836855888367, + "loss/reference_anchor": 0.19081640243530273, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.181434154510498, + "rewards/margins": 0.7545203566551208, + "rewards/rejected": 1.4269136190414429, + "step": 640 + }, + { + "drift/chosen_abs": 0.22275419533252716, + "drift/rejected_abs": 0.12799355387687683, + "epoch": 0.6163769933703638, + "grad_norm": 16.125, + "ht_mnpo/logit": 0.09602762758731842, + "ht_mnpo/residual": 0.08852764219045639, + "ht_mnpo/residual_abs": 0.19490960240364075, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.671694928216829e-08, + "logits/chosen": -2.2347233295440674, + "logits/rejected": -2.0241684913635254, + "logps/chosen": -0.27447259426116943, + "logps/rejected": -0.27523598074913025, + "loss": 0.44739675521850586, + "loss/core": 0.44131603837013245, + "loss/preference_sft": 0.27447259426116943, + "loss/reference_anchor": 0.2765862047672272, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.227006435394287, + "rewards/margins": 0.9602764248847961, + "rewards/rejected": 1.2667300701141357, + "step": 645 + }, + { + "drift/chosen_abs": 0.20366021990776062, + "drift/rejected_abs": 0.20054574310779572, + "epoch": 0.621155109598041, + "grad_norm": 278.0, + "ht_mnpo/logit": 0.003406074596568942, + "ht_mnpo/residual": -0.004093932919204235, + "ht_mnpo/residual_abs": 0.15771286189556122, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.469948505629e-08, + "logits/chosen": -2.2959370613098145, + "logits/rejected": -1.9112621545791626, + "logps/chosen": -0.286499559879303, + "logps/rejected": -0.30210891366004944, + "loss": 0.28895777463912964, + "loss/core": 0.2796488106250763, + "loss/preference_sft": 0.286499559879303, + "loss/reference_anchor": 0.4367947578430176, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.0365281105041504, + "rewards/margins": 0.0340607650578022, + "rewards/rejected": 2.002467632293701, + "step": 650 + }, + { + "drift/chosen_abs": 0.2379056215286255, + "drift/rejected_abs": 0.18813912570476532, + "epoch": 0.6259332258257182, + "grad_norm": 5.21875, + "ht_mnpo/logit": 0.05104045942425728, + "ht_mnpo/residual": 0.043540455400943756, + "ht_mnpo/residual_abs": 0.1262165606021881, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.270845797900168e-08, + "logits/chosen": -2.0275120735168457, + "logits/rejected": -1.7314637899398804, + "logps/chosen": -0.27975013852119446, + "logps/rejected": -0.2768488824367523, + "loss": 0.09684029966592789, + "loss/core": 0.08728532493114471, + "loss/preference_sft": 0.27975013852119446, + "loss/reference_anchor": 0.4497728943824768, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 2.378608226776123, + "rewards/margins": 0.5104045271873474, + "rewards/rejected": 1.8682035207748413, + "step": 655 + }, + { + "drift/chosen_abs": 0.12607762217521667, + "drift/rejected_abs": 0.08264373242855072, + "epoch": 0.6307113420533954, + "grad_norm": 16.5, + "ht_mnpo/logit": 0.043472785502672195, + "ht_mnpo/residual": 0.03597278520464897, + "ht_mnpo/residual_abs": 0.047445155680179596, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.0744616794160104e-08, + "logits/chosen": -2.3355751037597656, + "logits/rejected": -2.13651180267334, + "logps/chosen": -0.297650545835495, + "logps/rejected": -0.2983691096305847, + "loss": 0.014283724129199982, + "loss/core": 0.011677760630846024, + "loss/preference_sft": 0.297650545835495, + "loss/reference_anchor": 0.10053311288356781, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.2598559856414795, + "rewards/margins": 0.43472784757614136, + "rewards/rejected": 0.8251279592514038, + "step": 660 + }, + { + "drift/chosen_abs": 0.12593212723731995, + "drift/rejected_abs": 0.09534545242786407, + "epoch": 0.6354894582810727, + "grad_norm": 0.12890625, + "ht_mnpo/logit": 0.030613893643021584, + "ht_mnpo/residual": 0.02311389520764351, + "ht_mnpo/residual_abs": 0.06221969053149223, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.880870002211963e-08, + "logits/chosen": -2.3577072620391846, + "logits/rejected": -2.1732094287872314, + "logps/chosen": -0.28674453496932983, + "logps/rejected": -0.28307873010635376, + "loss": 0.016248947009444237, + "loss/core": 0.014818708412349224, + "loss/preference_sft": 0.28674453496932983, + "loss/reference_anchor": 0.04283740743994713, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 1.2591238021850586, + "rewards/margins": 0.30613890290260315, + "rewards/rejected": 0.9529846906661987, + "step": 665 + }, + { + "drift/chosen_abs": 0.23004214465618134, + "drift/rejected_abs": 0.10869423300027847, + "epoch": 0.6402675745087499, + "grad_norm": 13.4375, + "ht_mnpo/logit": 0.12265989929437637, + "ht_mnpo/residual": 0.11515990644693375, + "ht_mnpo/residual_abs": 0.13466432690620422, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.6901435682004996e-08, + "logits/chosen": -2.0747597217559814, + "logits/rejected": -1.8630584478378296, + "logps/chosen": -0.26450079679489136, + "logps/rejected": -0.28103771805763245, + "loss": 0.10082646459341049, + "loss/core": 0.09776544570922852, + "loss/preference_sft": 0.26450079679489136, + "loss/reference_anchor": 0.12660036981105804, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.30025315284729, + "rewards/margins": 1.2265989780426025, + "rewards/rejected": 1.0736541748046875, + "step": 670 + }, + { + "drift/chosen_abs": 0.16283902525901794, + "drift/rejected_abs": 0.1080724224448204, + "epoch": 0.6450456907364271, + "grad_norm": 17.375, + "ht_mnpo/logit": 0.0548507496714592, + "ht_mnpo/residual": 0.04735075309872627, + "ht_mnpo/residual_abs": 0.06609907001256943, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.502354101793314e-08, + "logits/chosen": -2.2743802070617676, + "logits/rejected": -2.0450034141540527, + "logps/chosen": -0.28120556473731995, + "logps/rejected": -0.28399839997291565, + "loss": 0.03949340060353279, + "loss/core": 0.036700859665870667, + "loss/preference_sft": 0.28120556473731995, + "loss/reference_anchor": 0.1115063801407814, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.6273581981658936, + "rewards/margins": 0.548507571220398, + "rewards/rejected": 1.0788507461547852, + "step": 675 + }, + { + "drift/chosen_abs": 0.2523915767669678, + "drift/rejected_abs": 0.1427987664937973, + "epoch": 0.6498238069641044, + "grad_norm": 84.5, + "ht_mnpo/logit": 0.11365683376789093, + "ht_mnpo/residual": 0.1061568409204483, + "ht_mnpo/residual_abs": 0.22065427899360657, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.3175722229287034e-08, + "logits/chosen": -2.2273154258728027, + "logits/rejected": -1.9926984310150146, + "logps/chosen": -0.30488717555999756, + "logps/rejected": -0.29885029792785645, + "loss": 0.5688546895980835, + "loss/core": 0.5607632994651794, + "loss/preference_sft": 0.30488717555999756, + "loss/reference_anchor": 0.3740817904472351, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": 2.523131847381592, + "rewards/margins": 1.136568307876587, + "rewards/rejected": 1.386563777923584, + "step": 680 + }, + { + "drift/chosen_abs": 0.14107666909694672, + "drift/rejected_abs": 0.1379661113023758, + "epoch": 0.6546019231917817, + "grad_norm": 20.375, + "ht_mnpo/logit": 0.0031998082995414734, + "ht_mnpo/residual": -0.004300191067159176, + "ht_mnpo/residual_abs": 0.06062345951795578, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.135867420514276e-08, + "logits/chosen": -2.150909423828125, + "logits/rejected": -2.010291814804077, + "logps/chosen": -0.2966502010822296, + "logps/rejected": -0.2863856256008148, + "loss": 0.02938271500170231, + "loss/core": 0.02649505063891411, + "loss/preference_sft": 0.2966502010822296, + "loss/reference_anchor": 0.11471836268901825, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.4100998640060425, + "rewards/margins": 0.03199812024831772, + "rewards/rejected": 1.3781017065048218, + "step": 685 + }, + { + "drift/chosen_abs": 0.2356395274400711, + "drift/rejected_abs": 0.2509574294090271, + "epoch": 0.6593800394194589, + "grad_norm": 128.0, + "ht_mnpo/logit": -0.01525307446718216, + "ht_mnpo/residual": -0.02275308221578598, + "ht_mnpo/residual_abs": 0.23793688416481018, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.957308026295035e-08, + "logits/chosen": -2.145141839981079, + "logits/rejected": -1.8953478336334229, + "logps/chosen": -0.2728975713253021, + "logps/rejected": -0.2741245925426483, + "loss": 0.6547005772590637, + "loss/core": 0.6441726684570312, + "loss/preference_sft": 0.2728975713253021, + "loss/reference_anchor": 0.4991017282009125, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 2.3562231063842773, + "rewards/margins": -0.15253083407878876, + "rewards/rejected": 2.508754014968872, + "step": 690 + }, + { + "drift/chosen_abs": 0.23801188170909882, + "drift/rejected_abs": 0.13991716504096985, + "epoch": 0.6641581556471361, + "grad_norm": 2.9375, + "ht_mnpo/logit": 0.09799585491418839, + "ht_mnpo/residual": 0.09049585461616516, + "ht_mnpo/residual_abs": 0.12964896857738495, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.7819611891566084e-08, + "logits/chosen": -2.3769142627716064, + "logits/rejected": -2.1696107387542725, + "logps/chosen": -0.28293538093566895, + "logps/rejected": -0.29280370473861694, + "loss": 0.3060581088066101, + "loss/core": 0.2948313355445862, + "loss/preference_sft": 0.28293538093566895, + "loss/reference_anchor": 0.5330466032028198, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.378937244415283, + "rewards/margins": 0.9799586534500122, + "rewards/rejected": 1.3989789485931396, + "step": 695 + }, + { + "drift/chosen_abs": 0.11122707277536392, + "drift/rejected_abs": 0.12570586800575256, + "epoch": 0.6689362718748133, + "grad_norm": 139.0, + "ht_mnpo/logit": -0.014081744477152824, + "ht_mnpo/residual": -0.021581750363111496, + "ht_mnpo/residual_abs": 0.08934501558542252, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.609892849873286e-08, + "logits/chosen": -2.561699867248535, + "logits/rejected": -2.2563445568084717, + "logps/chosen": -0.30130958557128906, + "logps/rejected": -0.31445008516311646, + "loss": 0.093419149518013, + "loss/core": 0.09064619243144989, + "loss/preference_sft": 0.30130958557128906, + "loss/reference_anchor": 0.10851649194955826, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.111412763595581, + "rewards/margins": -0.14081746339797974, + "rewards/rejected": 1.252230167388916, + "step": 700 + }, + { + "drift/chosen_abs": 0.130570650100708, + "drift/rejected_abs": 0.1270247995853424, + "epoch": 0.6737143881024906, + "grad_norm": 55.0, + "ht_mnpo/logit": 0.003545860992744565, + "ht_mnpo/residual": -0.003954136278480291, + "ht_mnpo/residual_abs": 0.0973631739616394, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.4411677163103894e-08, + "logits/chosen": -2.2780685424804688, + "logits/rejected": -2.014157772064209, + "logps/chosen": -0.29029467701911926, + "logps/rejected": -0.2877400517463684, + "loss": 0.07396967709064484, + "loss/core": 0.07192926108837128, + "loss/preference_sft": 0.29029467701911926, + "loss/reference_anchor": 0.07299144566059113, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": 1.30570650100708, + "rewards/margins": 0.03545858711004257, + "rewards/rejected": 1.2702479362487793, + "step": 705 + }, + { + "drift/chosen_abs": 0.0952615886926651, + "drift/rejected_abs": 0.047143109142780304, + "epoch": 0.6784925043301678, + "grad_norm": 139.0, + "ht_mnpo/logit": 0.04816994443535805, + "ht_mnpo/residual": 0.04066994786262512, + "ht_mnpo/residual_abs": 0.04842040687799454, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.2758492390902945e-08, + "logits/chosen": -2.2472968101501465, + "logits/rejected": -2.1181864738464355, + "logps/chosen": -0.2959393560886383, + "logps/rejected": -0.3067687153816223, + "loss": 0.015642011538147926, + "loss/core": 0.014619368128478527, + "loss/preference_sft": 0.2959393560886383, + "loss/reference_anchor": 0.021538157016038895, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.9488660097122192, + "rewards/margins": 0.48169946670532227, + "rewards/rejected": 0.46716657280921936, + "step": 710 + }, + { + "drift/chosen_abs": 0.1471184939146042, + "drift/rejected_abs": 0.19262142479419708, + "epoch": 0.683270620557845, + "grad_norm": 44.5, + "ht_mnpo/logit": -0.04537930339574814, + "ht_mnpo/residual": -0.052879296243190765, + "ht_mnpo/residual_abs": 0.1437372863292694, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.11399958773126e-08, + "logits/chosen": -2.2122180461883545, + "logits/rejected": -1.960108757019043, + "logps/chosen": -0.25510653853416443, + "logps/rejected": -0.2806130051612854, + "loss": 0.3858787417411804, + "loss/core": 0.3799152672290802, + "loss/preference_sft": 0.25510653853416443, + "loss/reference_anchor": 0.2726641297340393, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.470590353012085, + "rewards/margins": -0.45379310846328735, + "rewards/rejected": 1.924383521080017, + "step": 715 + }, + { + "drift/chosen_abs": 0.2561036944389343, + "drift/rejected_abs": 0.1267954558134079, + "epoch": 0.6880487367855223, + "grad_norm": 19.125, + "ht_mnpo/logit": 0.1294882744550705, + "ht_mnpo/residual": 0.12198827415704727, + "ht_mnpo/residual_abs": 0.17171990871429443, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.9556796272679972e-08, + "logits/chosen": -2.2394776344299316, + "logits/rejected": -2.0022566318511963, + "logps/chosen": -0.30328303575515747, + "logps/rejected": -0.2942310869693756, + "loss": 0.3686460554599762, + "loss/core": 0.3613186478614807, + "loss/preference_sft": 0.30328303575515747, + "loss/reference_anchor": 0.3360404670238495, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.559324264526367, + "rewards/margins": 1.2948826551437378, + "rewards/rejected": 1.264441728591919, + "step": 720 + }, + { + "drift/chosen_abs": 0.20471441745758057, + "drift/rejected_abs": 0.12798653542995453, + "epoch": 0.6928268530131996, + "grad_norm": 11.3125, + "ht_mnpo/logit": 0.0769859105348587, + "ht_mnpo/residual": 0.06948591768741608, + "ht_mnpo/residual_abs": 0.10851018130779266, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.8009488953628215e-08, + "logits/chosen": -2.1995251178741455, + "logits/rejected": -2.007638931274414, + "logps/chosen": -0.2995498478412628, + "logps/rejected": -0.29953497648239136, + "loss": 0.05993145704269409, + "loss/core": 0.056487929075956345, + "loss/preference_sft": 0.2995498478412628, + "loss/reference_anchor": 0.14222107827663422, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.044583797454834, + "rewards/margins": 0.7698591947555542, + "rewards/rejected": 1.2747247219085693, + "step": 725 + }, + { + "drift/chosen_abs": 0.3021154999732971, + "drift/rejected_abs": 0.28609389066696167, + "epoch": 0.6976049692408768, + "grad_norm": 7.25, + "ht_mnpo/logit": 0.022638099268078804, + "ht_mnpo/residual": 0.015138095244765282, + "ht_mnpo/residual_abs": 0.22438108921051025, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.649865579915976e-08, + "logits/chosen": -2.218268394470215, + "logits/rejected": -1.9819151163101196, + "logps/chosen": -0.2943797707557678, + "logps/rejected": -0.295765221118927, + "loss": 0.8258689641952515, + "loss/core": 0.8050146102905273, + "loss/preference_sft": 0.2943797707557678, + "loss/reference_anchor": 1.0132826566696167, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.0197532176971436, + "rewards/margins": 0.22638094425201416, + "rewards/rejected": 2.793372392654419, + "step": 730 + }, + { + "drift/chosen_abs": 0.22813507914543152, + "drift/rejected_abs": 0.1425250917673111, + "epoch": 0.702383085468554, + "grad_norm": 6.96875, + "ht_mnpo/logit": 0.08600185066461563, + "ht_mnpo/residual": 0.078501857817173, + "ht_mnpo/residual_abs": 0.12325754016637802, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.5024864971835507e-08, + "logits/chosen": -2.2636451721191406, + "logits/rejected": -2.125495433807373, + "logps/chosen": -0.2841891646385193, + "logps/rejected": -0.29996174573898315, + "loss": 0.22820651531219482, + "loss/core": 0.2230566293001175, + "loss/preference_sft": 0.2841891646385193, + "loss/reference_anchor": 0.22907507419586182, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.281350612640381, + "rewards/margins": 0.8600185513496399, + "rewards/rejected": 1.4213320016860962, + "step": 735 + }, + { + "drift/chosen_abs": 0.2448999136686325, + "drift/rejected_abs": 0.28253763914108276, + "epoch": 0.7071612016962313, + "grad_norm": 792.0, + "ht_mnpo/logit": -0.0376126654446125, + "ht_mnpo/residual": -0.045112669467926025, + "ht_mnpo/residual_abs": 0.2788679897785187, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.3588670704111997e-08, + "logits/chosen": -2.1962971687316895, + "logits/rejected": -1.9129769802093506, + "logps/chosen": -0.30221128463745117, + "logps/rejected": -0.31767523288726807, + "loss": 0.8641053438186646, + "loss/core": 0.8473947644233704, + "loss/preference_sft": 0.30221128463745117, + "loss/reference_anchor": 0.8053110837936401, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.4467480182647705, + "rewards/margins": -0.37612658739089966, + "rewards/rejected": 2.8228747844696045, + "step": 740 + }, + { + "drift/chosen_abs": 0.14628183841705322, + "drift/rejected_abs": 0.19587254524230957, + "epoch": 0.7119393179239085, + "grad_norm": 168.0, + "ht_mnpo/logit": -0.04983377456665039, + "ht_mnpo/residual": -0.05733378604054451, + "ht_mnpo/residual_abs": 0.15143811702728271, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.219061308991721e-08, + "logits/chosen": -2.247026205062866, + "logits/rejected": -1.9038165807724, + "logps/chosen": -0.284883588552475, + "logps/rejected": -0.2793378233909607, + "loss": 0.5056548714637756, + "loss/core": 0.49778372049331665, + "loss/preference_sft": 0.284883588552475, + "loss/reference_anchor": 0.365068644285202, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 1.4588829278945923, + "rewards/margins": -0.49833768606185913, + "rewards/rejected": 1.9572206735610962, + "step": 745 + }, + { + "drift/chosen_abs": 0.14641818404197693, + "drift/rejected_abs": 0.16142067313194275, + "epoch": 0.7167174341515857, + "grad_norm": 580.0, + "ht_mnpo/logit": -0.014964533038437366, + "ht_mnpo/residual": -0.022464536130428314, + "ht_mnpo/residual_abs": 0.1398034393787384, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.0831217881543557e-08, + "logits/chosen": -2.2321152687072754, + "logits/rejected": -2.0529208183288574, + "logps/chosen": -0.30759716033935547, + "logps/rejected": -0.36361774802207947, + "loss": 0.17265477776527405, + "loss/core": 0.1684485375881195, + "loss/preference_sft": 0.30759716033935547, + "loss/reference_anchor": 0.17955359816551208, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.4617996215820312, + "rewards/margins": -0.14964532852172852, + "rewards/rejected": 1.6114448308944702, + "step": 750 + }, + { + "drift/chosen_abs": 0.2566590905189514, + "drift/rejected_abs": 0.1291174292564392, + "epoch": 0.7214955503792629, + "grad_norm": 0.240234375, + "ht_mnpo/logit": 0.15318958461284637, + "ht_mnpo/residual": 0.14568959176540375, + "ht_mnpo/residual_abs": 0.17443707585334778, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.951099629193366e-08, + "logits/chosen": -2.130836009979248, + "logits/rejected": -1.8832871913909912, + "logps/chosen": -0.2614069879055023, + "logps/rejected": -0.29673653841018677, + "loss": 0.428077757358551, + "loss/core": 0.42090481519699097, + "loss/preference_sft": 0.2614069879055023, + "loss/reference_anchor": 0.33250653743743896, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.5654702186584473, + "rewards/margins": 1.5318958759307861, + "rewards/rejected": 1.0335744619369507, + "step": 755 + }, + { + "drift/chosen_abs": 0.13500568270683289, + "drift/rejected_abs": 0.1280622035264969, + "epoch": 0.7262736666069403, + "grad_norm": 20.0, + "ht_mnpo/logit": 0.006745192222297192, + "ht_mnpo/residual": -0.0007548049325123429, + "ht_mnpo/residual_abs": 0.08610773086547852, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.823044480243431e-08, + "logits/chosen": -2.1745476722717285, + "logits/rejected": -1.9323323965072632, + "logps/chosen": -0.27481788396835327, + "logps/rejected": -0.28048986196517944, + "loss": 0.07846397906541824, + "loss/core": 0.07617657631635666, + "loss/preference_sft": 0.27481788396835327, + "loss/reference_anchor": 0.08688807487487793, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": 1.347092628479004, + "rewards/margins": 0.06745192408561707, + "rewards/rejected": 1.2796406745910645, + "step": 760 + }, + { + "drift/chosen_abs": 0.2502586841583252, + "drift/rejected_abs": 0.11848635971546173, + "epoch": 0.7310517828346175, + "grad_norm": 394.0, + "ht_mnpo/logit": 0.13317808508872986, + "ht_mnpo/residual": 0.12567809224128723, + "ht_mnpo/residual_abs": 0.16746442019939423, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.699004497608994e-08, + "logits/chosen": -2.1543080806732178, + "logits/rejected": -1.9849952459335327, + "logps/chosen": -0.2812741994857788, + "logps/rejected": -0.2872457504272461, + "loss": 0.33080416917800903, + "loss/core": 0.32540163397789, + "loss/preference_sft": 0.2812741994857788, + "loss/reference_anchor": 0.24199938774108887, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.501790761947632, + "rewards/margins": 1.3317807912826538, + "rewards/rejected": 1.1700100898742676, + "step": 765 + }, + { + "drift/chosen_abs": 0.16793134808540344, + "drift/rejected_abs": 0.12145377695560455, + "epoch": 0.7358298990622947, + "grad_norm": 13.25, + "ht_mnpo/logit": 0.046602535992860794, + "ht_mnpo/residual": 0.03910253569483757, + "ht_mnpo/residual_abs": 0.08376048505306244, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.5790263276546658e-08, + "logits/chosen": -2.3451311588287354, + "logits/rejected": -2.0735201835632324, + "logps/chosen": -0.2896055281162262, + "logps/rejected": -0.2964259386062622, + "loss": 0.03520764783024788, + "loss/core": 0.032552026212215424, + "loss/preference_sft": 0.2896055281162262, + "loss/reference_anchor": 0.10382010042667389, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.6792783737182617, + "rewards/margins": 0.4660254120826721, + "rewards/rejected": 1.2132529020309448, + "step": 770 + }, + { + "drift/chosen_abs": 0.27133306860923767, + "drift/rejected_abs": 0.18516871333122253, + "epoch": 0.7406080152899719, + "grad_norm": 7.71875, + "ht_mnpo/logit": 0.08782286942005157, + "ht_mnpo/residual": 0.08032286167144775, + "ht_mnpo/residual_abs": 0.19639639556407928, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.4631550892634126e-08, + "logits/chosen": -2.123995304107666, + "logits/rejected": -1.9403555393218994, + "logps/chosen": -0.25651782751083374, + "logps/rejected": -0.27262482047080994, + "loss": 0.2998473048210144, + "loss/core": 0.29379308223724365, + "loss/preference_sft": 0.25651782751083374, + "loss/reference_anchor": 0.2770591974258423, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.711564540863037, + "rewards/margins": 0.8782287836074829, + "rewards/rejected": 1.8333358764648438, + "step": 775 + }, + { + "drift/chosen_abs": 0.2392168492078781, + "drift/rejected_abs": 0.08553509414196014, + "epoch": 0.7453861315176492, + "grad_norm": 396.0, + "ht_mnpo/logit": 0.15414533019065857, + "ht_mnpo/residual": 0.14664529263973236, + "ht_mnpo/residual_abs": 0.15226368606090546, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.3514343568692132e-08, + "logits/chosen": -2.282392740249634, + "logits/rejected": -2.234204053878784, + "logps/chosen": -0.2903969883918762, + "logps/rejected": -0.29160791635513306, + "loss": 0.4157602787017822, + "loss/core": 0.4093073308467865, + "loss/preference_sft": 0.2903969883918762, + "loss/reference_anchor": 0.29360833764076233, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.3913302421569824, + "rewards/margins": 1.5414531230926514, + "rewards/rejected": 0.8498773574829102, + "step": 780 + }, + { + "drift/chosen_abs": 0.2589685916900635, + "drift/rejected_abs": 0.1315867006778717, + "epoch": 0.7501642477453264, + "grad_norm": 4.5, + "ht_mnpo/logit": 0.127206951379776, + "ht_mnpo/residual": 0.11970694363117218, + "ht_mnpo/residual_abs": 0.14667531847953796, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.2439061440704724e-08, + "logits/chosen": -1.9296722412109375, + "logits/rejected": -1.7011972665786743, + "logps/chosen": -0.3024870455265045, + "logps/rejected": -0.3112749755382538, + "loss": 0.255341112613678, + "loss/core": 0.24924786388874054, + "loss/preference_sft": 0.3024870455265045, + "loss/reference_anchor": 0.2744150459766388, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.585298538208008, + "rewards/margins": 1.2720695734024048, + "rewards/rejected": 1.3132292032241821, + "step": 785 + }, + { + "drift/chosen_abs": 0.18013639748096466, + "drift/rejected_abs": 0.11114680767059326, + "epoch": 0.7549423639730036, + "grad_norm": 30.75, + "ht_mnpo/logit": 0.06904369592666626, + "ht_mnpo/residual": 0.06154369190335274, + "ht_mnpo/residual_abs": 0.11316666752099991, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.1406108878304468e-08, + "logits/chosen": -2.1777164936065674, + "logits/rejected": -1.9580738544464111, + "logps/chosen": -0.32455024123191833, + "logps/rejected": -0.33517375588417053, + "loss": 0.3174150884151459, + "loss/core": 0.3100249171257019, + "loss/preference_sft": 0.32455024123191833, + "loss/reference_anchor": 0.3370537757873535, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 1.800376534461975, + "rewards/margins": 0.6904369592666626, + "rewards/rejected": 1.1099395751953125, + "step": 790 + }, + { + "drift/chosen_abs": 0.3415032923221588, + "drift/rejected_abs": 0.18802081048488617, + "epoch": 0.7597204802006808, + "grad_norm": 9.125, + "ht_mnpo/logit": 0.1534634530544281, + "ht_mnpo/residual": 0.14596346020698547, + "ht_mnpo/residual_abs": 0.247287318110466, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0415874332705381e-08, + "logits/chosen": -2.1014323234558105, + "logits/rejected": -1.8502187728881836, + "logps/chosen": -0.33924803137779236, + "logps/rejected": -0.32133159041404724, + "loss": 0.8026763796806335, + "loss/core": 0.7806569337844849, + "loss/preference_sft": 0.33924803137779236, + "loss/reference_anchor": 1.067044973373413, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": 3.411249876022339, + "rewards/margins": 1.5346347093582153, + "rewards/rejected": 1.8766151666641235, + "step": 795 + }, + { + "drift/chosen_abs": 0.11637525260448456, + "drift/rejected_abs": 0.07857302576303482, + "epoch": 0.7644985964283582, + "grad_norm": 1.5859375, + "ht_mnpo/logit": 0.03819254785776138, + "ht_mnpo/residual": 0.030692553147673607, + "ht_mnpo/residual_abs": 0.04601796343922615, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.468730190622484e-09, + "logits/chosen": -2.430523633956909, + "logits/rejected": -2.217977523803711, + "logps/chosen": -0.2681643068790436, + "logps/rejected": -0.2834469676017761, + "loss": 0.013186362572014332, + "loss/core": 0.01157904788851738, + "loss/preference_sft": 0.2681643068790436, + "loss/reference_anchor": 0.05354924127459526, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.1637526750564575, + "rewards/margins": 0.3819255232810974, + "rewards/rejected": 0.7818271517753601, + "step": 800 + }, + { + "drift/chosen_abs": 0.18257346749305725, + "drift/rejected_abs": 0.17861232161521912, + "epoch": 0.7692767126560354, + "grad_norm": 36.5, + "ht_mnpo/logit": 0.004233963787555695, + "ht_mnpo/residual": -0.0032660351134836674, + "ht_mnpo/residual_abs": 0.1263841688632965, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.565032634232194e-09, + "logits/chosen": -2.478210210800171, + "logits/rejected": -2.1900885105133057, + "logps/chosen": -0.28286072611808777, + "logps/rejected": -0.30062970519065857, + "loss": 0.15072055160999298, + "loss/core": 0.14563368260860443, + "loss/preference_sft": 0.28286072611808777, + "loss/reference_anchor": 0.22605614364147186, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.8255739212036133, + "rewards/margins": 0.0423397533595562, + "rewards/rejected": 1.7832343578338623, + "step": 805 + }, + { + "drift/chosen_abs": 0.1565002202987671, + "drift/rejected_abs": 0.13715356588363647, + "epoch": 0.7740548288837126, + "grad_norm": 260.0, + "ht_mnpo/logit": 0.02574211359024048, + "ht_mnpo/residual": 0.018242118880152702, + "ht_mnpo/residual_abs": 0.10176096856594086, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 7.70512150722702e-09, + "logits/chosen": -2.2540268898010254, + "logits/rejected": -1.9384829998016357, + "logps/chosen": -0.26865464448928833, + "logps/rejected": -0.279658704996109, + "loss": 0.06719930469989777, + "loss/core": 0.06475383788347244, + "loss/preference_sft": 0.26865464448928833, + "loss/reference_anchor": 0.0954083725810051, + "rewards/accuracies": 0.75, + "rewards/chosen": 1.5635719299316406, + "rewards/margins": 0.2574211657047272, + "rewards/rejected": 1.3061506748199463, + "step": 810 + }, + { + "drift/chosen_abs": 0.2879694104194641, + "drift/rejected_abs": 0.14862984418869019, + "epoch": 0.7788329451113898, + "grad_norm": 516.0, + "ht_mnpo/logit": 0.14298096299171448, + "ht_mnpo/residual": 0.13548094034194946, + "ht_mnpo/residual_abs": 0.20370686054229736, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.8893201870139915e-09, + "logits/chosen": -2.174163341522217, + "logits/rejected": -1.8267030715942383, + "logps/chosen": -0.27633410692214966, + "logps/rejected": -0.29714980721473694, + "loss": 0.497852623462677, + "loss/core": 0.4906894266605377, + "loss/preference_sft": 0.27633410692214966, + "loss/reference_anchor": 0.33052676916122437, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": 2.8788421154022217, + "rewards/margins": 1.4298094511032104, + "rewards/rejected": 1.4490330219268799, + "step": 815 + }, + { + "drift/chosen_abs": 0.3162073791027069, + "drift/rejected_abs": 0.1260301172733307, + "epoch": 0.7836110613390671, + "grad_norm": 844.0, + "ht_mnpo/logit": 0.19014951586723328, + "ht_mnpo/residual": 0.18264950811862946, + "ht_mnpo/residual_abs": 0.18700771033763885, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.117935463105808e-09, + "logits/chosen": -2.2044544219970703, + "logits/rejected": -2.0138018131256104, + "logps/chosen": -0.26344865560531616, + "logps/rejected": -0.2816709578037262, + "loss": 0.5444236993789673, + "loss/core": 0.535254716873169, + "loss/preference_sft": 0.26344865560531616, + "loss/reference_anchor": 0.4321085810661316, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.161675453186035, + "rewards/margins": 1.9014952182769775, + "rewards/rejected": 1.260180115699768, + "step": 820 + }, + { + "drift/chosen_abs": 0.17280279099941254, + "drift/rejected_abs": 0.11558995395898819, + "epoch": 0.7883891775667443, + "grad_norm": 61.75, + "ht_mnpo/logit": 0.05732642859220505, + "ht_mnpo/residual": 0.04982643201947212, + "ht_mnpo/residual_abs": 0.11239101737737656, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.391257421749826e-09, + "logits/chosen": -2.185276508331299, + "logits/rejected": -1.9858713150024414, + "logps/chosen": -0.27167096734046936, + "logps/rejected": -0.2912501096725464, + "loss": 0.10265527665615082, + "loss/core": 0.09941322356462479, + "loss/preference_sft": 0.27167096734046936, + "loss/reference_anchor": 0.13493572175502777, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.72737717628479, + "rewards/margins": 0.5732643008232117, + "rewards/rejected": 1.1541129350662231, + "step": 825 + }, + { + "drift/chosen_abs": 0.3662811517715454, + "drift/rejected_abs": 0.1733158379793167, + "epoch": 0.7931672937944215, + "grad_norm": 240.0, + "ht_mnpo/logit": 0.1925339549779892, + "ht_mnpo/residual": 0.18503394722938538, + "ht_mnpo/residual_abs": 0.2609553933143616, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.709559336838462e-09, + "logits/chosen": -2.038930654525757, + "logits/rejected": -1.8778717517852783, + "logps/chosen": -0.3013201355934143, + "logps/rejected": -0.30770185589790344, + "loss": 0.6489893198013306, + "loss/core": 0.6383043527603149, + "loss/preference_sft": 0.3013201355934143, + "loss/reference_anchor": 0.5041123032569885, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": 3.658099412918091, + "rewards/margins": 1.9253393411636353, + "rewards/rejected": 1.7327598333358765, + "step": 830 + }, + { + "drift/chosen_abs": 0.21075081825256348, + "drift/rejected_abs": 0.11554797738790512, + "epoch": 0.7979454100220987, + "grad_norm": 7.4375, + "ht_mnpo/logit": 0.09528296440839767, + "ht_mnpo/residual": 0.08778297901153564, + "ht_mnpo/residual_abs": 0.1566668152809143, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.073097567142025e-09, + "logits/chosen": -2.253007173538208, + "logits/rejected": -1.9240471124649048, + "logps/chosen": -0.2810296416282654, + "logps/rejected": -0.288202702999115, + "loss": 0.2865773141384125, + "loss/core": 0.2817952632904053, + "loss/preference_sft": 0.2810296416282654, + "loss/reference_anchor": 0.21100135147571564, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.1059257984161377, + "rewards/margins": 0.9528296589851379, + "rewards/rejected": 1.1530958414077759, + "step": 835 + }, + { + "drift/chosen_abs": 0.16927887499332428, + "drift/rejected_abs": 0.13110263645648956, + "epoch": 0.802723526249776, + "grad_norm": 27.875, + "ht_mnpo/logit": 0.038380250334739685, + "ht_mnpo/residual": 0.030880261212587357, + "ht_mnpo/residual_abs": 0.07921403646469116, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.482111459902695e-09, + "logits/chosen": -2.3635165691375732, + "logits/rejected": -2.133021831512451, + "logps/chosen": -0.27768105268478394, + "logps/rejected": -0.2969363331794739, + "loss": 0.031025763601064682, + "loss/core": 0.02740824781358242, + "loss/preference_sft": 0.27768105268478394, + "loss/reference_anchor": 0.15310758352279663, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.6914350986480713, + "rewards/margins": 0.383802592754364, + "rewards/rejected": 1.307632565498352, + "step": 840 + }, + { + "drift/chosen_abs": 0.23839470744132996, + "drift/rejected_abs": 0.10520993173122406, + "epoch": 0.8075016424774533, + "grad_norm": 520.0, + "ht_mnpo/logit": 0.13712438941001892, + "ht_mnpo/residual": 0.1296243816614151, + "ht_mnpo/residual_abs": 0.17075686156749725, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.9368232608258797e-09, + "logits/chosen": -2.2205090522766113, + "logits/rejected": -1.973059058189392, + "logps/chosen": -0.27566519379615784, + "logps/rejected": -0.27959397435188293, + "loss": 0.3910796046257019, + "loss/core": 0.38557177782058716, + "loss/preference_sft": 0.27566519379615784, + "loss/reference_anchor": 0.24782493710517883, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.3833117485046387, + "rewards/margins": 1.3712438344955444, + "rewards/rejected": 1.0120675563812256, + "step": 845 + }, + { + "drift/chosen_abs": 0.2898247539997101, + "drift/rejected_abs": 0.20496828854084015, + "epoch": 0.8122797587051305, + "grad_norm": 30.25, + "ht_mnpo/logit": 0.08481884747743607, + "ht_mnpo/residual": 0.07731884717941284, + "ht_mnpo/residual_abs": 0.16502617299556732, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4374380305025866e-09, + "logits/chosen": -2.1580593585968018, + "logits/rejected": -2.01873779296875, + "logps/chosen": -0.28473132848739624, + "logps/rejected": -0.30126944184303284, + "loss": 0.3230627179145813, + "loss/core": 0.3124871850013733, + "loss/preference_sft": 0.28473132848739624, + "loss/reference_anchor": 0.500304639339447, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.89532732963562, + "rewards/margins": 0.8481884002685547, + "rewards/rejected": 2.0471386909484863, + "step": 850 + }, + { + "drift/chosen_abs": 0.2713679075241089, + "drift/rejected_abs": 0.1950051337480545, + "epoch": 0.8170578749328078, + "grad_norm": 143.0, + "ht_mnpo/logit": 0.08360113203525543, + "ht_mnpo/residual": 0.07610112428665161, + "ht_mnpo/residual_abs": 0.24734120070934296, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.984143567294594e-09, + "logits/chosen": -2.0692951679229736, + "logits/rejected": -1.8935229778289795, + "logps/chosen": -0.2885996401309967, + "logps/rejected": -0.2952654957771301, + "loss": 0.7806990742683411, + "loss/core": 0.7664183974266052, + "loss/preference_sft": 0.2885996401309967, + "loss/reference_anchor": 0.6851747632026672, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.7128584384918213, + "rewards/margins": 0.8360112309455872, + "rewards/rejected": 1.876847267150879, + "step": 855 + }, + { + "drift/chosen_abs": 0.17620348930358887, + "drift/rejected_abs": 0.16157342493534088, + "epoch": 0.821835991160485, + "grad_norm": 278.0, + "ht_mnpo/logit": 0.01473709475249052, + "ht_mnpo/residual": 0.007237098179757595, + "ht_mnpo/residual_abs": 0.1026911735534668, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.577110336711096e-09, + "logits/chosen": -2.147317409515381, + "logits/rejected": -1.9148515462875366, + "logps/chosen": -0.29422527551651, + "logps/rejected": -0.30900320410728455, + "loss": 0.11610148102045059, + "loss/core": 0.11178989708423615, + "loss/preference_sft": 0.29422527551651, + "loss/reference_anchor": 0.18615707755088806, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.7592500448226929, + "rewards/margins": 0.14737097918987274, + "rewards/rejected": 1.6118791103363037, + "step": 860 + }, + { + "drift/chosen_abs": 0.1526646763086319, + "drift/rejected_abs": 0.18857529759407043, + "epoch": 0.8266141073881622, + "grad_norm": 36.25, + "ht_mnpo/logit": -0.035790909081697464, + "ht_mnpo/residual": -0.04329090565443039, + "ht_mnpo/residual_abs": 0.15114633738994598, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.2164914073037048e-09, + "logits/chosen": -2.1270697116851807, + "logits/rejected": -1.9014345407485962, + "logps/chosen": -0.2999208867549896, + "logps/rejected": -0.30866190791130066, + "loss": 0.393560528755188, + "loss/core": 0.3873744606971741, + "loss/preference_sft": 0.2999208867549896, + "loss/reference_anchor": 0.27931293845176697, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": 1.5241281986236572, + "rewards/margins": -0.3579089939594269, + "rewards/rejected": 1.8820374011993408, + "step": 865 + }, + { + "drift/chosen_abs": 0.29999783635139465, + "drift/rejected_abs": 0.1245269924402237, + "epoch": 0.8313922236158394, + "grad_norm": 2.25, + "ht_mnpo/logit": 0.1785384863615036, + "ht_mnpo/residual": 0.17103847861289978, + "ht_mnpo/residual_abs": 0.18839120864868164, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.024223931035357e-10, + "logits/chosen": -2.1790618896484375, + "logits/rejected": -2.021745204925537, + "logps/chosen": -0.2831207811832428, + "logps/rejected": -0.3100909888744354, + "loss": 0.4790888726711273, + "loss/core": 0.4709659218788147, + "loss/preference_sft": 0.2831207811832428, + "loss/reference_anchor": 0.37783652544021606, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.9999780654907227, + "rewards/margins": 1.7853851318359375, + "rewards/rejected": 1.2145931720733643, + "step": 870 + }, + { + "drift/chosen_abs": 0.13325706124305725, + "drift/rejected_abs": 0.06335233151912689, + "epoch": 0.8361703398435167, + "grad_norm": 22.125, + "ht_mnpo/logit": 0.06985118985176086, + "ht_mnpo/residual": 0.06235118955373764, + "ht_mnpo/residual_abs": 0.08558257669210434, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.350214026223516e-10, + "logits/chosen": -2.2370076179504395, + "logits/rejected": -2.0332276821136475, + "logps/chosen": -0.31737005710601807, + "logps/rejected": -0.32498207688331604, + "loss": 0.06058365851640701, + "loss/core": 0.058684103190898895, + "loss/preference_sft": 0.31737005710601807, + "loss/reference_anchor": 0.06324069201946259, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.3312170505523682, + "rewards/margins": 0.6985118985176086, + "rewards/rejected": 0.63270503282547, + "step": 875 + }, + { + "drift/chosen_abs": 0.376343309879303, + "drift/rejected_abs": 0.2587401270866394, + "epoch": 0.840948456071194, + "grad_norm": 65.0, + "ht_mnpo/logit": 0.11799073219299316, + "ht_mnpo/residual": 0.11049073934555054, + "ht_mnpo/residual_abs": 0.21407637000083923, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.143889944367429e-10, + "logits/chosen": -1.9316622018814087, + "logits/rejected": -1.8352222442626953, + "logps/chosen": -0.29632678627967834, + "logps/rejected": -0.26413092017173767, + "loss": 0.603792667388916, + "loss/core": 0.5875565409660339, + "loss/preference_sft": 0.29632678627967834, + "loss/reference_anchor": 0.7821727991104126, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 3.7634329795837402, + "rewards/margins": 1.179907202720642, + "rewards/rejected": 2.5835256576538086, + "step": 880 + }, + { + "drift/chosen_abs": 0.2143622189760208, + "drift/rejected_abs": 0.10691346973180771, + "epoch": 0.8457265722988712, + "grad_norm": 1.1640625, + "ht_mnpo/logit": 0.10745616257190704, + "ht_mnpo/residual": 0.09995613992214203, + "ht_mnpo/residual_abs": 0.1442701518535614, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.406081393722531e-10, + "logits/chosen": -2.3482344150543213, + "logits/rejected": -2.0240073204040527, + "logps/chosen": -0.3080643117427826, + "logps/rejected": -0.3035167157649994, + "loss": 0.7237991094589233, + "loss/core": 0.7135142087936401, + "loss/preference_sft": 0.3080643117427826, + "loss/reference_anchor": 0.4834347665309906, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.1424150466918945, + "rewards/margins": 1.074561595916748, + "rewards/rejected": 1.067853331565857, + "step": 885 + }, + { + "drift/chosen_abs": 0.29515132308006287, + "drift/rejected_abs": 0.269777774810791, + "epoch": 0.8505046885265484, + "grad_norm": 9.25, + "ht_mnpo/logit": 0.025372039526700974, + "ht_mnpo/residual": 0.017872031778097153, + "ht_mnpo/residual_abs": 0.2621416449546814, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.1374418930135133e-10, + "logits/chosen": -2.138016939163208, + "logits/rejected": -1.8397918939590454, + "logps/chosen": -0.2903136909008026, + "logps/rejected": -0.3230307400226593, + "loss": 1.0775001049041748, + "loss/core": 1.0594369173049927, + "loss/preference_sft": 0.2903136909008026, + "loss/reference_anchor": 0.8741267919540405, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.951321840286255, + "rewards/margins": 0.25372037291526794, + "rewards/rejected": 2.697601795196533, + "step": 890 + }, + { + "drift/chosen_abs": 0.1901109516620636, + "drift/rejected_abs": 0.11028845608234406, + "epoch": 0.8552828047542257, + "grad_norm": 7.53125, + "ht_mnpo/logit": 0.07955867052078247, + "ht_mnpo/residual": 0.07205866277217865, + "ht_mnpo/residual_abs": 0.14294487237930298, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.3844852567285756e-11, + "logits/chosen": -2.2701876163482666, + "logits/rejected": -2.0042386054992676, + "logps/chosen": -0.2893333435058594, + "logps/rejected": -0.2989356517791748, + "loss": 0.26186782121658325, + "loss/core": 0.2570946514606476, + "loss/preference_sft": 0.2893333435058594, + "loss/reference_anchor": 0.20972399413585663, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.8983943462371826, + "rewards/margins": 0.7955867052078247, + "rewards/rejected": 1.1028077602386475, + "step": 895 + }, + { + "drift/chosen_abs": 0.14426791667938232, + "drift/rejected_abs": 0.09550980478525162, + "epoch": 0.8600609209819029, + "grad_norm": 1.375, + "ht_mnpo/logit": 0.04878208041191101, + "ht_mnpo/residual": 0.04128208011388779, + "ht_mnpo/residual_abs": 0.09187668561935425, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.401760429905703e-13, + "logits/chosen": -2.105118751525879, + "logits/rejected": -1.8005974292755127, + "logps/chosen": -0.2834312915802002, + "logps/rejected": -0.30577853322029114, + "loss": 0.10151888430118561, + "loss/core": 0.09912057220935822, + "loss/preference_sft": 0.2834312915802002, + "loss/reference_anchor": 0.09157194197177887, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.4408460855484009, + "rewards/margins": 0.4878207743167877, + "rewards/rejected": 0.953025221824646, + "step": 900 + }, + { + "epoch": 0.8600609209819029, + "step": 900, + "total_flos": 0.0, + "train_loss": 0.3323093075222439, + "train_runtime": 7030.1668, + "train_samples_per_second": 2.048, + "train_steps_per_second": 0.128 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..eeb7c42 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:108c90d70fb7ac868f77c694ac8838a8c38a75b137e345e67a5fb024b88a4e4b +size 7057