commit 9ad339e5b2593fa6498d6f5cb0e73621d91f30e0 Author: ModelHub XC Date: Tue Jul 21 13:43:13 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: promotion/qwen3-8b-aaai27-flagship-sppo-avg-s44 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..a8f4425 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +--- +library_name: transformers +base_model: Qwen/Qwen3-8B +tags: +- ronpo +- mnpo +- preference-optimization +--- + +# qwen3-8b-aaai27-flagship-sppo-avg-s44 + +Research checkpoint for the RONPO AAAI revision experiments. + +- Method: sppo_avg +- Base model: `Qwen/Qwen3-8B` +- Seed: 44 +- Local source at upload time: `/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/sppo_avg/seed44/attempt2` +- Uploaded at UTC: 2026-07-13T06:41:09Z +- Run status metadata: `{"attempt": 2, "effective_batch_size": 16, "method": "sppo_avg", "objective_protocol": "objective_protocol.json", "optimizer_steps": 900, "resource_profile": "resource_profile.json", "seed": 44, "split_manifest": "split_manifest.json", "stability_gate": "stability_gate.json", "status": "completed", "wandb_run_id": "0bbaa979875e", "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/0bbaa979875e"}` + +AAAI-27 P1 matched budget: 900 optimizer steps, effective batch 16; passed non-thinking and collapse stability gates; sealed-test metrics are not used for checkpoint selection. + +Intended use: reproducibility and evaluation for the RONPO paper. This +checkpoint is not intended as a production assistant. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..9b78c49 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 0.688900715245141, + "train_runtime": 7064.5608, + "train_samples": 16746, + "train_samples_per_second": 2.038, + "train_steps_per_second": 0.127 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..29ad57e --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +model_name_or_path: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218 +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/avg: 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: sppo +eta: 0.0075 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.02 +preference_sft_weight: 0.002 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 2.5e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 44 +gradient_accumulation_steps: 16 +gradient_checkpointing: true +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 1 +per_device_eval_batch_size: 1 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/sppo_avg/seed44/attempt2 +run_name: aaai27-flagship-full-sppo_avg-s44-a2 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/job_status.json b/job_status.json new file mode 100644 index 0000000..bac3ade --- /dev/null +++ b/job_status.json @@ -0,0 +1,23 @@ +{ + "status": "completed", + "stage": "full", + "method": "sppo_avg", + "seed": 44, + "attempt": 2, + "gpu": 6, + "gpus": [ + 6 + ], + "returncode": 0, + "finite_metrics": true, + "oom": false, + "traceback": false, + "reason": "passed", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "0bbaa979875e", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/0bbaa979875e", + "output_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/sppo_avg/seed44/attempt2", + "log": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/logs/full_sppo_avg_s44_a2.log", + "completed_at": "2026-07-13T06:38:46Z" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..96fce0c --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca34e78a293479e4a1f1f8140eed232f2cc11a5077f6f329f2bc00a0a7cd863c +size 16381517208 diff --git a/objective_protocol.json b/objective_protocol.json new file mode 100644 index 0000000..35b3b44 --- /dev/null +++ b/objective_protocol.json @@ -0,0 +1,69 @@ +{ + "schema_version": 1, + "frozen_at_kst": "2026-07-12T21:40:00+09:00", + "base_model": "Qwen/Qwen3-8B", + "primary_triple": { + "helpfulness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "ultrafeedback-helpfulness", + "transform": "identity" + }, + "safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "conciseness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "helpsteer-verbosity", + "transform": "negate" + } + }, + "gate_thresholds": { + "raw_score_std_strictly_greater_than": 0.01, + "aggregate_prompt_pair_spearman_median_strictly_less_than": 0.0, + "aggregate_pairwise_top1_mismatch_at_least": 0.5 + }, + "gate_pool": "source-train Qwen3-8B response pool split into deterministic train/validation; sealed source-test excluded", + "training_seeds": [42, 43, 44], + "optimizer_steps": 900, + "effective_batch_size": 16, + "decode": { + "enable_thinking": false, + "temperature": 0.7, + "top_p": 0.9, + "max_new_tokens": 2048, + "decode_seed": 42 + }, + "methods": [ + "ronpo_full_expect", + "ronpo_k_only", + "dpo", + "ipo", + "simpo", + "kto", + "sppo_avg", + "inpo_avg", + "ht_mnpo_helpfulness", + "ht_mnpo_safety", + "ht_mnpo_conciseness" + ], + "selection_policy": "No sealed-test selection. One preregistered configuration per method; failures use only the declared stability retry recipe, at most three attempts.", + "stability_gate": { + "validation_prompts": 128, + "min_candidate_base_mean_word_ratio": 0.33, + "max_candidate_base_mean_word_ratio": 2.0, + "max_consecutive_identical_word_run": 20, + "think_tag_count": 0, + "empty_response_count": 0, + "retry_recipe": "attempt 2 halves learning rate, enables gradient checkpointing, and adds reference=0.02/SFT=0.002 anchoring to unanchored unified losses; attempt 3 quarters learning rate and uses reference=0.05/SFT=0.005; RONPO uses reference=0.05/SFT=0.005 from attempt 1" + }, + "primary_metric": "mean prompt-level minimum of per-prompt min-max-normalized helpfulness, safety, conciseness", + "bootstrap": { + "unit": "prompt", + "paired": true, + "resamples": 2000, + "interval": "percentile_95", + "seed": 42 + } +} diff --git a/pair_manifest.json b/pair_manifest.json new file mode 100644 index 0000000..be034b0 --- /dev/null +++ b/pair_manifest.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "objectives": [ + "helpfulness", + "safety", + "conciseness" + ], + "normalization": "per_prompt_minmax", + "adversary": { + "steps": 25, + "alpha": 1.0, + "kappa": 0.05, + "preference_scale": 8.0 + }, + "ronpo_estimator": "Rao-Blackwellized full sigma expectation with three common response pairs per prompt", + "counts": { + "train": { + "avg": 16746, + "ronpo_full_expect": 50340, + "ronpo_k_only": 50340, + "ht_mnpo_helpfulness": 16743, + "ht_mnpo_safety": 16764, + "ht_mnpo_conciseness": 16755 + }, + "validation": { + "avg": 1859, + "ronpo_full_expect": 5592, + "ronpo_k_only": 5592, + "ht_mnpo_helpfulness": 1864, + "ht_mnpo_safety": 1862, + "ht_mnpo_conciseness": 1858 + } + } +} diff --git a/provenance.json b/provenance.json new file mode 100644 index 0000000..0b5b68b --- /dev/null +++ b/provenance.json @@ -0,0 +1,13 @@ +{ + "method": "sppo_avg", + "seed": 44, + "attempt": 2, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "0bbaa979875e", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/0bbaa979875e", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json" +} diff --git a/resource_profile.json b/resource_profile.json new file mode 100644 index 0000000..18eec47 --- /dev/null +++ b/resource_profile.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "measured_at_kst": "2026-07-12T22:38:00+09:00", + "hardware": "NVIDIA B200 183359 MiB reported total; 178.35 GiB PyTorch-visible capacity", + "smoke_evidence": { + "attempted_profile": {"per_device_batch_size": 2, "gradient_accumulation_steps": 8, "gradient_checkpointing": false, "effective_batch_size": 16}, + "outcome": "OOM before completing the 20-step smoke across every launched unified baseline; representative process used 176.87 GiB and failed a 1.62 GiB allocation. KTO also OOMed at the optimizer step.", + "paper_metric": false + }, + "selected_full_training_profile": { + "per_device_batch_size": 2, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": false, + "attention_implementation": "PyTorch SDPA with cuDNN SDPA disabled", + "effective_batch_size": 16, + "applies_symmetrically_to_all_unified_pairwise_methods": true, + "kto_implementation_constraint": "TRL KTO requires actual batch size > 1. KTO uses batch 2 / accumulation 8 and identical AdamW with foreach=False. PyTorch SDPA is used with cuDNN SDPA disabled. The fixed base reference model is placed on a second B200 and its exact logits are returned to the policy device; the policy/optimizer remain on the first B200. This avoids a measured ~6-hour reference-precompute path while leaving the KTO loss, actual/effective batch, optimizer, and 900-step budget unchanged." + }, + "retry_semantics": "Resource selection is separated from method-stability attempts. Full attempt 1 uses the smoke-validated SDPA batch-2 profile and retains the preregistered method LR/anchor; attempts 2 and 3 fall back to batch 1 / accumulation 16 / gradient checkpointing and apply the declared LR/anchor stabilization recipe.", + "provenance_note": "No reward or sealed-evaluation result was available when this hardware profile was selected." +} diff --git a/run_status.json b/run_status.json new file mode 100644 index 0000000..345ad5b --- /dev/null +++ b/run_status.json @@ -0,0 +1,14 @@ +{ + "method": "sppo_avg", + "seed": 44, + "attempt": 2, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "0bbaa979875e", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/0bbaa979875e", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json", + "status": "completed" +} diff --git a/split_manifest.json b/split_manifest.json new file mode 100644 index 0000000..4fb96bd --- /dev/null +++ b/split_manifest.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "split_rule": "sort source-train prompts by sha256('flagship-v1||'+prompt_id); first round(10%) validation", + "validation_fraction": 0.1, + "source_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/avg_oracle", + "base_model": "Qwen/Qwen3-8B", + "counts": { + "train": 16780, + "validation": 1864, + "sealed_test": 604 + }, + "prompt_id_sha256": { + "train": "fcf6f2ea5a0b7bcd41179e6f5288f4d850b2d04bdb729da91907bd18bf733a69", + "validation": "a87ba6f519192164daa487b109a6547dd8cbde46a27a373d6da201ca2d061b8f", + "sealed_test": "b398ced6af6cf73d731e1a5f32a0631d8060323d51eda1e375695566bf5f1867" + }, + "overlaps": { + "train_validation": 0, + "train_sealed_test": 0, + "validation_sealed_test": 0 + }, + "sealed_test_policy": "prompts only until S4a; no objective scores or model generations used in S0-S3" +} diff --git a/stability_gate.json b/stability_gate.json new file mode 100644 index 0000000..13e6e5e --- /dev/null +++ b/stability_gate.json @@ -0,0 +1,38 @@ +{ + "status": "passed", + "passed": true, + "base": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 388.1484375, + "max_words": 1379, + "max_repeat_run": 4 + }, + "candidate": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 393.4375, + "max_words": 1254, + "max_repeat_run": 4 + }, + "candidate_base_mean_word_ratio": 1.0136263913209749, + "thresholds": { + "min_length_ratio": 0.33, + "max_length_ratio": 2.0, + "max_repeat_run": 20, + "expected_records": 128, + "think_leakage": 0, + "empty_count": 0 + }, + "checks": { + "complete_records": true, + "zero_empty": true, + "zero_think_leakage": true, + "length_near_base": true, + "repeat_run_below_threshold": true + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ffd19d2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..9b78c49 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 0.688900715245141, + "train_runtime": 7064.5608, + "train_samples": 16746, + "train_samples_per_second": 2.038, + "train_steps_per_second": 0.127 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..b5ca673 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,3643 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8599068434252956, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "drift/chosen_abs": 0.3420991003513336, + "drift/rejected_abs": 0.1417631208896637, + "epoch": 0.004777260241251642, + "grad_norm": 374.0, + "learning_rate": 1.111111111111111e-08, + "logits/chosen": -2.2763760089874268, + "logits/rejected": -2.2397658824920654, + "logps/chosen": -0.3073771595954895, + "logps/rejected": -0.3057326674461365, + "loss": 2.170994281768799, + "loss/core": 2.1488795280456543, + "loss/preference_sft": 0.3073771595954895, + "loss/reference_anchor": 1.075006365776062, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.418382167816162, + "rewards/margins": 2.0053153038024902, + "rewards/rejected": 1.4130665063858032, + "step": 5 + }, + { + "drift/chosen_abs": 0.29207316040992737, + "drift/rejected_abs": 0.1325351893901825, + "epoch": 0.009554520482503284, + "grad_norm": 11.125, + "learning_rate": 2.5e-08, + "logits/chosen": -2.2057507038116455, + "logits/rejected": -2.197091579437256, + "logps/chosen": -0.2905746102333069, + "logps/rejected": -0.28117018938064575, + "loss": 0.5789883136749268, + "loss/core": 0.5726724863052368, + "loss/preference_sft": 0.2905746102333069, + "loss/reference_anchor": 0.2867390513420105, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.920731782913208, + "rewards/margins": 1.5960335731506348, + "rewards/rejected": 1.3246982097625732, + "step": 10 + }, + { + "drift/chosen_abs": 0.1926683634519577, + "drift/rejected_abs": 0.08847218751907349, + "epoch": 0.014331780723754926, + "grad_norm": 2.265625, + "learning_rate": 3.888888888888889e-08, + "logits/chosen": -1.8458455801010132, + "logits/rejected": -1.9005146026611328, + "logps/chosen": -0.2896016538143158, + "logps/rejected": -0.28710517287254333, + "loss": 0.23165909945964813, + "loss/core": 0.22878754138946533, + "loss/preference_sft": 0.2896016538143158, + "loss/reference_anchor": 0.11461815983057022, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.9264246225357056, + "rewards/margins": 1.057593584060669, + "rewards/rejected": 0.8688309788703918, + "step": 15 + }, + { + "drift/chosen_abs": 0.1641753613948822, + "drift/rejected_abs": 0.05660538747906685, + "epoch": 0.01910904096500657, + "grad_norm": 4.96875, + "learning_rate": 5.2777777777777776e-08, + "logits/chosen": -2.1686434745788574, + "logits/rejected": -2.0960006713867188, + "logps/chosen": -0.2862441837787628, + "logps/rejected": -0.28391966223716736, + "loss": 0.15841466188430786, + "loss/core": 0.15627533197402954, + "loss/preference_sft": 0.2862441837787628, + "loss/reference_anchor": 0.07834186404943466, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.6407321691513062, + "rewards/margins": 1.0884038209915161, + "rewards/rejected": 0.5523284077644348, + "step": 20 + }, + { + "drift/chosen_abs": 0.18150915205478668, + "drift/rejected_abs": 0.11916539818048477, + "epoch": 0.02388630120625821, + "grad_norm": 73.0, + "learning_rate": 6.666666666666667e-08, + "logits/chosen": -2.2032320499420166, + "logits/rejected": -2.2200589179992676, + "logps/chosen": -0.2700212597846985, + "logps/rejected": -0.2805071473121643, + "loss": 0.263420045375824, + "loss/core": 0.2602730393409729, + "loss/preference_sft": 0.2700212597846985, + "loss/reference_anchor": 0.13034658133983612, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.8135448694229126, + "rewards/margins": 0.6393396258354187, + "rewards/rejected": 1.1742051839828491, + "step": 25 + }, + { + "drift/chosen_abs": 0.2730526924133301, + "drift/rejected_abs": 0.1805466115474701, + "epoch": 0.028663561447509853, + "grad_norm": 156.0, + "learning_rate": 8.055555555555555e-08, + "logits/chosen": -2.1016995906829834, + "logits/rejected": -2.0634474754333496, + "logps/chosen": -0.31322628259658813, + "logps/rejected": -0.27209383249282837, + "loss": 1.1556752920150757, + "loss/core": 1.1436045169830322, + "loss/preference_sft": 0.31322628259658813, + "loss/reference_anchor": 0.5722212791442871, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.7279961109161377, + "rewards/margins": 0.923298180103302, + "rewards/rejected": 1.8046979904174805, + "step": 30 + }, + { + "drift/chosen_abs": 0.18750497698783875, + "drift/rejected_abs": 0.07988496869802475, + "epoch": 0.033440821688761495, + "grad_norm": 3.453125, + "learning_rate": 9.444444444444444e-08, + "logits/chosen": -2.1232945919036865, + "logits/rejected": -2.225569248199463, + "logps/chosen": -0.2830365300178528, + "logps/rejected": -0.26490843296051025, + "loss": 0.17748409509658813, + "loss/core": 0.17516210675239563, + "loss/preference_sft": 0.2830365300178528, + "loss/reference_anchor": 0.08779536187648773, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.8736076354980469, + "rewards/margins": 1.09036123752594, + "rewards/rejected": 0.7832463979721069, + "step": 35 + }, + { + "drift/chosen_abs": 0.23411647975444794, + "drift/rejected_abs": 0.09753605723381042, + "epoch": 0.03821808193001314, + "grad_norm": 13.5, + "learning_rate": 1.0833333333333334e-07, + "logits/chosen": -2.1147046089172363, + "logits/rejected": -2.191413164138794, + "logps/chosen": -0.29881948232650757, + "logps/rejected": -0.3003283143043518, + "loss": 0.31825491786003113, + "loss/core": 0.3145065903663635, + "loss/preference_sft": 0.29881948232650757, + "loss/reference_anchor": 0.1575334519147873, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.3403539657592773, + "rewards/margins": 1.3710139989852905, + "rewards/rejected": 0.9693400263786316, + "step": 40 + }, + { + "drift/chosen_abs": 0.28226155042648315, + "drift/rejected_abs": 0.18430650234222412, + "epoch": 0.04299534217126478, + "grad_norm": 69.5, + "learning_rate": 1.2222222222222222e-07, + "logits/chosen": -2.0363850593566895, + "logits/rejected": -2.076094150543213, + "logps/chosen": -0.28957605361938477, + "logps/rejected": -0.2919236719608307, + "loss": 1.3649934530258179, + "loss/core": 1.3508943319320679, + "loss/preference_sft": 0.28957605361938477, + "loss/reference_anchor": 0.6759929060935974, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.8192150592803955, + "rewards/margins": 0.9820273518562317, + "rewards/rejected": 1.8371880054473877, + "step": 45 + }, + { + "drift/chosen_abs": 0.1817009598016739, + "drift/rejected_abs": 0.1377423107624054, + "epoch": 0.04777260241251642, + "grad_norm": 0.984375, + "learning_rate": 1.3611111111111108e-07, + "logits/chosen": -2.1037697792053223, + "logits/rejected": -2.192720890045166, + "logps/chosen": -0.2859702706336975, + "logps/rejected": -0.2898501455783844, + "loss": 0.35004186630249023, + "loss/core": 0.34600991010665894, + "loss/preference_sft": 0.2859702706336975, + "loss/reference_anchor": 0.17300085723400116, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 1.816737413406372, + "rewards/margins": 0.44307082891464233, + "rewards/rejected": 1.3736662864685059, + "step": 50 + }, + { + "drift/chosen_abs": 0.1675940454006195, + "drift/rejected_abs": 0.0922044888138771, + "epoch": 0.05254986265376806, + "grad_norm": 3.890625, + "learning_rate": 1.5e-07, + "logits/chosen": -2.1252381801605225, + "logits/rejected": -2.2449936866760254, + "logps/chosen": -0.2968815565109253, + "logps/rejected": -0.293079674243927, + "loss": 0.11641957610845566, + "loss/core": 0.11467579752206802, + "loss/preference_sft": 0.2968815565109253, + "loss/reference_anchor": 0.05750115588307381, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.6759402751922607, + "rewards/margins": 0.7550500631332397, + "rewards/rejected": 0.9208904504776001, + "step": 55 + }, + { + "drift/chosen_abs": 0.14563629031181335, + "drift/rejected_abs": 0.1081518679857254, + "epoch": 0.057327122895019705, + "grad_norm": 62.25, + "learning_rate": 1.6388888888888888e-07, + "logits/chosen": -2.1089062690734863, + "logits/rejected": -2.1458160877227783, + "logps/chosen": -0.29047664999961853, + "logps/rejected": -0.2865060865879059, + "loss": 0.29185742139816284, + "loss/core": 0.2883908748626709, + "loss/preference_sft": 0.29047664999961853, + "loss/reference_anchor": 0.14428026974201202, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.4559776782989502, + "rewards/margins": 0.3751867413520813, + "rewards/rejected": 1.0807908773422241, + "step": 60 + }, + { + "drift/chosen_abs": 0.2445906698703766, + "drift/rejected_abs": 0.09041162580251694, + "epoch": 0.06210438313627135, + "grad_norm": 154.0, + "learning_rate": 1.7777777777777776e-07, + "logits/chosen": -2.1424062252044678, + "logits/rejected": -2.2173452377319336, + "logps/chosen": -0.3165227174758911, + "logps/rejected": -0.310057133436203, + "loss": 0.7591328620910645, + "loss/core": 0.7509864568710327, + "loss/preference_sft": 0.3165227174758911, + "loss/reference_anchor": 0.37566640973091125, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.4453353881835938, + "rewards/margins": 1.5434644222259521, + "rewards/rejected": 0.901870846748352, + "step": 65 + }, + { + "drift/chosen_abs": 0.23211221396923065, + "drift/rejected_abs": 0.10898170620203018, + "epoch": 0.06688164337752299, + "grad_norm": 21.5, + "learning_rate": 1.9166666666666668e-07, + "logits/chosen": -2.1081697940826416, + "logits/rejected": -2.1647961139678955, + "logps/chosen": -0.3016880452632904, + "logps/rejected": -0.3038561940193176, + "loss": 0.4961710572242737, + "loss/core": 0.4906535744667053, + "loss/preference_sft": 0.3016880452632904, + "loss/reference_anchor": 0.24570658802986145, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.32112193107605, + "rewards/margins": 1.3848812580108643, + "rewards/rejected": 0.9362408518791199, + "step": 70 + }, + { + "drift/chosen_abs": 0.15495255589485168, + "drift/rejected_abs": 0.10253290832042694, + "epoch": 0.07165890361877464, + "grad_norm": 3.71875, + "learning_rate": 2.0555555555555553e-07, + "logits/chosen": -2.0683295726776123, + "logits/rejected": -2.1414763927459717, + "logps/chosen": -0.28989818692207336, + "logps/rejected": -0.29636505246162415, + "loss": 0.21972903609275818, + "loss/core": 0.21697552502155304, + "loss/preference_sft": 0.28989818692207336, + "loss/reference_anchor": 0.10868575423955917, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.5473926067352295, + "rewards/margins": 0.5245345830917358, + "rewards/rejected": 1.0228580236434937, + "step": 75 + }, + { + "drift/chosen_abs": 0.12912170588970184, + "drift/rejected_abs": 0.07073847949504852, + "epoch": 0.07643616386002627, + "grad_norm": 3.546875, + "learning_rate": 2.1944444444444442e-07, + "logits/chosen": -2.271545648574829, + "logits/rejected": -2.3474278450012207, + "logps/chosen": -0.30917978286743164, + "logps/rejected": -0.30712777376174927, + "loss": 0.09423994272947311, + "loss/core": 0.09269268810749054, + "loss/preference_sft": 0.30917978286743164, + "loss/reference_anchor": 0.046444639563560486, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.2905423641204834, + "rewards/margins": 0.5846601724624634, + "rewards/rejected": 0.7058821320533752, + "step": 80 + }, + { + "drift/chosen_abs": 0.2436620444059372, + "drift/rejected_abs": 0.1702345758676529, + "epoch": 0.08121342410127792, + "grad_norm": 1.859375, + "learning_rate": 2.3333333333333333e-07, + "logits/chosen": -2.1165645122528076, + "logits/rejected": -2.1273322105407715, + "logps/chosen": -0.2930624485015869, + "logps/rejected": -0.28557881712913513, + "loss": 0.9216262698173523, + "loss/core": 0.9119135141372681, + "loss/preference_sft": 0.2930624485015869, + "loss/reference_anchor": 0.45633044838905334, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.4357974529266357, + "rewards/margins": 0.7339332103729248, + "rewards/rejected": 1.7018646001815796, + "step": 85 + }, + { + "drift/chosen_abs": 0.2897084355354309, + "drift/rejected_abs": 0.06650175154209137, + "epoch": 0.08599068434252956, + "grad_norm": 21.375, + "learning_rate": 2.4722222222222224e-07, + "logits/chosen": -2.0020716190338135, + "logits/rejected": -2.18646502494812, + "logps/chosen": -0.30623993277549744, + "logps/rejected": -0.3014351725578308, + "loss": 0.6364718675613403, + "loss/core": 0.6295538544654846, + "loss/preference_sft": 0.30623993277549744, + "loss/reference_anchor": 0.3152781128883362, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.8956007957458496, + "rewards/margins": 2.232067584991455, + "rewards/rejected": 0.6635335087776184, + "step": 90 + }, + { + "drift/chosen_abs": 0.21713528037071228, + "drift/rejected_abs": 0.13760428130626678, + "epoch": 0.09076794458378121, + "grad_norm": 113.5, + "learning_rate": 2.4998495746616845e-07, + "logits/chosen": -2.0369200706481934, + "logits/rejected": -1.9513944387435913, + "logps/chosen": -0.2811819612979889, + "logps/rejected": -0.30305975675582886, + "loss": 0.6681066751480103, + "loss/core": 0.6609320640563965, + "loss/preference_sft": 0.2811819612979889, + "loss/reference_anchor": 0.33061572909355164, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.170912981033325, + "rewards/margins": 0.7965704798698425, + "rewards/rejected": 1.3743422031402588, + "step": 95 + }, + { + "drift/chosen_abs": 0.2975863218307495, + "drift/rejected_abs": 0.1862352043390274, + "epoch": 0.09554520482503284, + "grad_norm": 1.671875, + "learning_rate": 2.4992385337738696e-07, + "logits/chosen": -2.008779525756836, + "logits/rejected": -2.0400052070617676, + "logps/chosen": -0.28240451216697693, + "logps/rejected": -0.31364208459854126, + "loss": 0.9967119097709656, + "loss/core": 0.9862802624702454, + "loss/preference_sft": 0.28240451216697693, + "loss/reference_anchor": 0.4933498501777649, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.9755539894104004, + "rewards/margins": 1.446618676185608, + "rewards/rejected": 1.5289356708526611, + "step": 100 + }, + { + "drift/chosen_abs": 0.31854820251464844, + "drift/rejected_abs": 0.13878673315048218, + "epoch": 0.10032246506628449, + "grad_norm": 4.375, + "learning_rate": 2.4981577053636144e-07, + "logits/chosen": -1.934293508529663, + "logits/rejected": -1.9871383905410767, + "logps/chosen": -0.2761833667755127, + "logps/rejected": -0.26434868574142456, + "loss": 0.7714275121688843, + "loss/core": 0.7632346153259277, + "loss/preference_sft": 0.2761833667755127, + "loss/reference_anchor": 0.38203054666519165, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.1841020584106445, + "rewards/margins": 1.7970466613769531, + "rewards/rejected": 1.3870553970336914, + "step": 105 + }, + { + "drift/chosen_abs": 0.21965603530406952, + "drift/rejected_abs": 0.10388318449258804, + "epoch": 0.10509972530753613, + "grad_norm": 5.09375, + "learning_rate": 2.496607495886281e-07, + "logits/chosen": -2.1238505840301514, + "logits/rejected": -2.097701072692871, + "logps/chosen": -0.2954608201980591, + "logps/rejected": -0.3057326674461365, + "loss": 0.650016188621521, + "loss/core": 0.6429935097694397, + "loss/preference_sft": 0.2954608201980591, + "loss/reference_anchor": 0.3215869665145874, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.1926186084747314, + "rewards/margins": 1.1593815088272095, + "rewards/rejected": 1.0332369804382324, + "step": 110 + }, + { + "drift/chosen_abs": 0.3507579267024994, + "drift/rejected_abs": 0.12545308470726013, + "epoch": 0.10987698554878778, + "grad_norm": 536.0, + "learning_rate": 2.4945884883122553e-07, + "logits/chosen": -2.0215022563934326, + "logits/rejected": -2.0700583457946777, + "logps/chosen": -0.28947877883911133, + "logps/rejected": -0.2858577370643616, + "loss": 1.4179147481918335, + "loss/core": 1.4032930135726929, + "loss/preference_sft": 0.28947877883911133, + "loss/reference_anchor": 0.7021245956420898, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.5075790882110596, + "rewards/margins": 2.2566137313842773, + "rewards/rejected": 1.2509657144546509, + "step": 115 + }, + { + "drift/chosen_abs": 0.1810287982225418, + "drift/rejected_abs": 0.14085547626018524, + "epoch": 0.11465424579003941, + "grad_norm": 181.0, + "learning_rate": 2.492101441907714e-07, + "logits/chosen": -2.2403197288513184, + "logits/rejected": -2.1026992797851562, + "logps/chosen": -0.260287880897522, + "logps/rejected": -0.28016167879104614, + "loss": 0.29911449551582336, + "loss/core": 0.2956363260746002, + "loss/preference_sft": 0.260287880897522, + "loss/reference_anchor": 0.14787891507148743, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.8089395761489868, + "rewards/margins": 0.4003846049308777, + "rewards/rejected": 1.4085547924041748, + "step": 120 + }, + { + "drift/chosen_abs": 0.4085167944431305, + "drift/rejected_abs": 0.10499458014965057, + "epoch": 0.11943150603129106, + "grad_norm": 1.140625, + "learning_rate": 2.4891472919490977e-07, + "logits/chosen": -1.9146769046783447, + "logits/rejected": -1.9590771198272705, + "logps/chosen": -0.28118112683296204, + "logps/rejected": -0.27787166833877563, + "loss": 1.8055627346038818, + "loss/core": 1.7871158123016357, + "loss/preference_sft": 0.28118112683296204, + "loss/reference_anchor": 0.8942378759384155, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.083503723144531, + "rewards/margins": 3.059095859527588, + "rewards/rejected": 1.024407982826233, + "step": 125 + }, + { + "drift/chosen_abs": 0.2690292298793793, + "drift/rejected_abs": 0.1741829663515091, + "epoch": 0.1242087662725427, + "grad_norm": 165.0, + "learning_rate": 2.4857271493713894e-07, + "logits/chosen": -2.0733864307403564, + "logits/rejected": -2.071460247039795, + "logps/chosen": -0.29650020599365234, + "logps/rejected": -0.29231899976730347, + "loss": 1.0611131191253662, + "loss/core": 1.050018072128296, + "loss/preference_sft": 0.29650020599365234, + "loss/reference_anchor": 0.5251095294952393, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.689784049987793, + "rewards/margins": 0.9507070779800415, + "rewards/rejected": 1.7390769720077515, + "step": 130 + }, + { + "drift/chosen_abs": 0.1724942922592163, + "drift/rejected_abs": 0.08941344916820526, + "epoch": 0.12898602651379434, + "grad_norm": 7.40625, + "learning_rate": 2.481842300350339e-07, + "logits/chosen": -2.0276589393615723, + "logits/rejected": -2.0920987129211426, + "logps/chosen": -0.27670344710350037, + "logps/rejected": -0.263897567987442, + "loss": 0.19596371054649353, + "loss/core": 0.19347220659255981, + "loss/preference_sft": 0.27670344710350037, + "loss/reference_anchor": 0.09690549224615097, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 1.7226674556732178, + "rewards/margins": 0.83062744140625, + "rewards/rejected": 0.8920401334762573, + "step": 135 + }, + { + "drift/chosen_abs": 0.22231808304786682, + "drift/rejected_abs": 0.09158378839492798, + "epoch": 0.13376328675504598, + "grad_norm": 1.109375, + "learning_rate": 2.4774942058187854e-07, + "logits/chosen": -2.191605806350708, + "logits/rejected": -2.250380039215088, + "logps/chosen": -0.2993098199367523, + "logps/rejected": -0.2922438979148865, + "loss": 0.7870423793792725, + "loss/core": 0.7786499261856079, + "loss/preference_sft": 0.2993098199367523, + "loss/reference_anchor": 0.38969308137893677, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.2213363647460938, + "rewards/margins": 1.306333303451538, + "rewards/rejected": 0.9150028228759766, + "step": 140 + }, + { + "drift/chosen_abs": 0.1859746277332306, + "drift/rejected_abs": 0.1087212786078453, + "epoch": 0.13854054699629761, + "grad_norm": 24.0, + "learning_rate": 2.472684500917257e-07, + "logits/chosen": -1.9669681787490845, + "logits/rejected": -1.9938608407974243, + "logps/chosen": -0.3318449854850769, + "logps/rejected": -0.3153822422027588, + "loss": 0.21972651779651642, + "loss/core": 0.21688850224018097, + "loss/preference_sft": 0.3318449854850769, + "loss/reference_anchor": 0.10871516168117523, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.858870267868042, + "rewards/margins": 0.7766832709312439, + "rewards/rejected": 1.0821870565414429, + "step": 145 + }, + { + "drift/chosen_abs": 0.14986830949783325, + "drift/rejected_abs": 0.05766589194536209, + "epoch": 0.14331780723754928, + "grad_norm": 3.046875, + "learning_rate": 2.467414994379065e-07, + "logits/chosen": -2.068981170654297, + "logits/rejected": -2.1354150772094727, + "logps/chosen": -0.31118273735046387, + "logps/rejected": -0.316683828830719, + "loss": 0.3042221665382385, + "loss/core": 0.30058878660202026, + "loss/preference_sft": 0.31118273735046387, + "loss/reference_anchor": 0.15054963529109955, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.497855305671692, + "rewards/margins": 0.9212591052055359, + "rewards/rejected": 0.5765963792800903, + "step": 150 + }, + { + "drift/chosen_abs": 0.15884414315223694, + "drift/rejected_abs": 0.08928201347589493, + "epoch": 0.1480950674788009, + "grad_norm": 17.25, + "learning_rate": 2.4616876678501114e-07, + "logits/chosen": -2.2533211708068848, + "logits/rejected": -2.299926280975342, + "logps/chosen": -0.3172222971916199, + "logps/rejected": -0.3034704029560089, + "loss": 0.1397572010755539, + "loss/core": 0.13774201273918152, + "loss/preference_sft": 0.3172222971916199, + "loss/reference_anchor": 0.06903725862503052, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.5872414112091064, + "rewards/margins": 0.6962968111038208, + "rewards/rejected": 0.89094477891922, + "step": 155 + }, + { + "drift/chosen_abs": 0.3720155358314514, + "drift/rejected_abs": 0.11601382493972778, + "epoch": 0.15287232772005255, + "grad_norm": 948.0, + "learning_rate": 2.4555046751436735e-07, + "logits/chosen": -2.1271204948425293, + "logits/rejected": -2.1136441230773926, + "logps/chosen": -0.2963225841522217, + "logps/rejected": -0.280845582485199, + "loss": 1.3421189785003662, + "loss/core": 1.3282365798950195, + "loss/preference_sft": 0.2963225841522217, + "loss/reference_anchor": 0.6644912958145142, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.718188762664795, + "rewards/margins": 2.5581791400909424, + "rewards/rejected": 1.160009741783142, + "step": 160 + }, + { + "drift/chosen_abs": 0.1847013384103775, + "drift/rejected_abs": 0.09540971368551254, + "epoch": 0.15764958796130418, + "grad_norm": 14.9375, + "learning_rate": 2.4488683414304425e-07, + "logits/chosen": -2.129667282104492, + "logits/rejected": -2.230297565460205, + "logps/chosen": -0.3174905776977539, + "logps/rejected": -0.31254857778549194, + "loss": 0.2407360076904297, + "loss/core": 0.23772159218788147, + "loss/preference_sft": 0.3174905776977539, + "loss/reference_anchor": 0.11897067725658417, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.846673607826233, + "rewards/margins": 0.8936681747436523, + "rewards/rejected": 0.9530051350593567, + "step": 165 + }, + { + "drift/chosen_abs": 0.15486204624176025, + "drift/rejected_abs": 0.09620408713817596, + "epoch": 0.16242684820255585, + "grad_norm": 7.65625, + "learning_rate": 2.4417811623641203e-07, + "logits/chosen": -2.2738282680511475, + "logits/rejected": -2.332890510559082, + "logps/chosen": -0.27397435903549194, + "logps/rejected": -0.2843550145626068, + "loss": 0.23323550820350647, + "loss/core": 0.230382040143013, + "loss/preference_sft": 0.27397435903549194, + "loss/reference_anchor": 0.11527644097805023, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.5484516620635986, + "rewards/margins": 0.5872721076011658, + "rewards/rejected": 0.9611795544624329, + "step": 170 + }, + { + "drift/chosen_abs": 0.5246831178665161, + "drift/rejected_abs": 0.20105135440826416, + "epoch": 0.16720410844380748, + "grad_norm": 14.875, + "learning_rate": 2.4342458031429113e-07, + "logits/chosen": -1.8324203491210938, + "logits/rejected": -1.8386204242706299, + "logps/chosen": -0.2887090742588043, + "logps/rejected": -0.2930302619934082, + "loss": 2.519411563873291, + "loss/core": 2.493882656097412, + "loss/preference_sft": 0.2887090742588043, + "loss/reference_anchor": 1.2475916147232056, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 5.246148109436035, + "rewards/margins": 3.2455039024353027, + "rewards/rejected": 2.0006444454193115, + "step": 175 + }, + { + "drift/chosen_abs": 0.3295663297176361, + "drift/rejected_abs": 0.15013830363750458, + "epoch": 0.17198136868505912, + "grad_norm": 0.8984375, + "learning_rate": 2.4262650975072444e-07, + "logits/chosen": -2.1856048107147217, + "logits/rejected": -2.3016016483306885, + "logps/chosen": -0.28502634167671204, + "logps/rejected": -0.29640719294548035, + "loss": 1.1087981462478638, + "loss/core": 1.0972480773925781, + "loss/preference_sft": 0.28502634167671204, + "loss/reference_anchor": 0.5489994883537292, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.2941269874572754, + "rewards/margins": 1.8216378688812256, + "rewards/rejected": 1.4724891185760498, + "step": 180 + }, + { + "drift/chosen_abs": 0.19066013395786285, + "drift/rejected_abs": 0.07735215127468109, + "epoch": 0.17675862892631075, + "grad_norm": 2.484375, + "learning_rate": 2.417842046674122e-07, + "logits/chosen": -2.3281235694885254, + "logits/rejected": -2.3521463871002197, + "logps/chosen": -0.26029735803604126, + "logps/rejected": -0.2679922580718994, + "loss": 0.19912585616111755, + "loss/core": 0.1966342329978943, + "loss/preference_sft": 0.26029735803604126, + "loss/reference_anchor": 0.09855085611343384, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9066013097763062, + "rewards/margins": 1.1336138248443604, + "rewards/rejected": 0.7729876041412354, + "step": 185 + }, + { + "drift/chosen_abs": 0.19174052774906158, + "drift/rejected_abs": 0.09250342845916748, + "epoch": 0.18153588916756241, + "grad_norm": 2.0625, + "learning_rate": 2.4089798182084843e-07, + "logits/chosen": -2.0859601497650146, + "logits/rejected": -2.158560276031494, + "logps/chosen": -0.3072795271873474, + "logps/rejected": -0.30823010206222534, + "loss": 0.2847827076911926, + "loss/core": 0.2813512980937958, + "loss/preference_sft": 0.3072795271873474, + "loss/reference_anchor": 0.14084270596504211, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.9145091772079468, + "rewards/margins": 1.001261591911316, + "rewards/rejected": 0.9132474660873413, + "step": 190 + }, + { + "drift/chosen_abs": 0.21405212581157684, + "drift/rejected_abs": 0.13653413951396942, + "epoch": 0.18631314940881405, + "grad_norm": 2.34375, + "learning_rate": 2.3996817448320195e-07, + "logits/chosen": -2.154674530029297, + "logits/rejected": -2.109269380569458, + "logps/chosen": -0.28905612230300903, + "logps/rejected": -0.2959071695804596, + "loss": 0.47725993394851685, + "loss/core": 0.47195786237716675, + "loss/preference_sft": 0.28905612230300903, + "loss/reference_anchor": 0.2361954003572464, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.1385719776153564, + "rewards/margins": 0.8496444821357727, + "rewards/rejected": 1.288927435874939, + "step": 195 + }, + { + "drift/chosen_abs": 0.2820349335670471, + "drift/rejected_abs": 0.15623252093791962, + "epoch": 0.19109040965006568, + "grad_norm": 118.5, + "learning_rate": 2.3899513231698607e-07, + "logits/chosen": -2.1225414276123047, + "logits/rejected": -2.0437445640563965, + "logps/chosen": -0.29230794310569763, + "logps/rejected": -0.2899274230003357, + "loss": 0.5702517628669739, + "loss/core": 0.5640192031860352, + "loss/preference_sft": 0.29230794310569763, + "loss/reference_anchor": 0.28239911794662476, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.8195438385009766, + "rewards/margins": 1.2614609003067017, + "rewards/rejected": 1.5580826997756958, + "step": 200 + }, + { + "drift/chosen_abs": 0.2620348334312439, + "drift/rejected_abs": 0.1418972909450531, + "epoch": 0.19586766989131732, + "grad_norm": 29.875, + "learning_rate": 2.3797922124356506e-07, + "logits/chosen": -2.029536008834839, + "logits/rejected": -2.0266480445861816, + "logps/chosen": -0.3911769688129425, + "logps/rejected": -0.3083851635456085, + "loss": 0.8301785588264465, + "loss/core": 0.8211765289306641, + "loss/preference_sft": 0.3911769688129425, + "loss/reference_anchor": 0.41098326444625854, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.620229959487915, + "rewards/margins": 1.3009237051010132, + "rewards/rejected": 1.3193061351776123, + "step": 205 + }, + { + "drift/chosen_abs": 0.17181067168712616, + "drift/rejected_abs": 0.07057416439056396, + "epoch": 0.20064493013256898, + "grad_norm": 1.625, + "learning_rate": 2.3692082330554585e-07, + "logits/chosen": -2.251944065093994, + "logits/rejected": -2.347381114959717, + "logps/chosen": -0.25848090648651123, + "logps/rejected": -0.25597459077835083, + "loss": 0.11508001387119293, + "loss/core": 0.11342470347881317, + "loss/preference_sft": 0.25848090648651123, + "loss/reference_anchor": 0.056916702538728714, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.7172927856445312, + "rewards/margins": 1.0118815898895264, + "rewards/rejected": 0.7054111957550049, + "step": 210 + }, + { + "drift/chosen_abs": 0.2751947343349457, + "drift/rejected_abs": 0.17245322465896606, + "epoch": 0.20542219037382062, + "grad_norm": 454.0, + "learning_rate": 2.3582033652310765e-07, + "logits/chosen": -1.9821306467056274, + "logits/rejected": -1.956804633140564, + "logps/chosen": -0.2867414355278015, + "logps/rejected": -0.2776336669921875, + "loss": 0.5166824460029602, + "loss/core": 0.5109937787055969, + "loss/preference_sft": 0.2867414355278015, + "loss/reference_anchor": 0.25575685501098633, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.7519474029541016, + "rewards/margins": 1.0281121730804443, + "rewards/rejected": 1.7238352298736572, + "step": 215 + }, + { + "drift/chosen_abs": 0.347667396068573, + "drift/rejected_abs": 0.07685881108045578, + "epoch": 0.21019945061507225, + "grad_norm": 2.8125, + "learning_rate": 2.346781747443227e-07, + "logits/chosen": -2.0388574600219727, + "logits/rejected": -2.1317574977874756, + "logps/chosen": -0.2959466874599457, + "logps/rejected": -0.2806416153907776, + "loss": 1.4209736585617065, + "loss/core": 1.4063067436218262, + "loss/preference_sft": 0.2959466874599457, + "loss/reference_anchor": 0.7037451267242432, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.4755759239196777, + "rewards/margins": 2.707566022872925, + "rewards/rejected": 0.7680096626281738, + "step": 220 + }, + { + "drift/chosen_abs": 0.3234902322292328, + "drift/rejected_abs": 0.10148955881595612, + "epoch": 0.2149767108563239, + "grad_norm": 188.0, + "learning_rate": 2.3349476748952508e-07, + "logits/chosen": -2.1488888263702393, + "logits/rejected": -2.237083911895752, + "logps/chosen": -0.2739250063896179, + "logps/rejected": -0.27243196964263916, + "loss": 0.5956701040267944, + "loss/core": 0.5892229676246643, + "loss/preference_sft": 0.2739250063896179, + "loss/reference_anchor": 0.2949652075767517, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.233933210372925, + "rewards/margins": 2.221308469772339, + "rewards/rejected": 1.0126246213912964, + "step": 225 + }, + { + "drift/chosen_abs": 0.36533501744270325, + "drift/rejected_abs": 0.1348879039287567, + "epoch": 0.21975397109757555, + "grad_norm": 844.0, + "learning_rate": 2.3227055978978545e-07, + "logits/chosen": -2.0082271099090576, + "logits/rejected": -2.0484988689422607, + "logps/chosen": -0.25067609548568726, + "logps/rejected": -0.2521665692329407, + "loss": 1.3585067987442017, + "loss/core": 1.3445472717285156, + "loss/preference_sft": 0.25067609548568726, + "loss/reference_anchor": 0.6729066967964172, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.6500682830810547, + "rewards/margins": 2.3029842376708984, + "rewards/rejected": 1.3470841646194458, + "step": 230 + }, + { + "drift/chosen_abs": 0.2073029726743698, + "drift/rejected_abs": 0.12087440490722656, + "epoch": 0.2245312313388272, + "grad_norm": 1.5703125, + "learning_rate": 2.3100601201955321e-07, + "logits/chosen": -2.326741933822632, + "logits/rejected": -2.3083624839782715, + "logps/chosen": -0.25950273871421814, + "logps/rejected": -0.2679215967655182, + "loss": 0.7174883484840393, + "loss/core": 0.709865391254425, + "loss/preference_sft": 0.25950273871421814, + "loss/reference_anchor": 0.35520219802856445, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.0721540451049805, + "rewards/margins": 0.8642433285713196, + "rewards/rejected": 1.2079106569290161, + "step": 235 + }, + { + "drift/chosen_abs": 0.339974582195282, + "drift/rejected_abs": 0.19081738591194153, + "epoch": 0.22930849158007882, + "grad_norm": 14.9375, + "learning_rate": 2.2970159972352864e-07, + "logits/chosen": -2.121175527572632, + "logits/rejected": -2.1394689083099365, + "logps/chosen": -0.27825844287872314, + "logps/rejected": -0.29647335410118103, + "loss": 1.2494022846221924, + "loss/core": 1.236472249031067, + "loss/preference_sft": 0.27825844287872314, + "loss/reference_anchor": 0.6186779737472534, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.3997459411621094, + "rewards/margins": 1.4949043989181519, + "rewards/rejected": 1.9048411846160889, + "step": 240 + }, + { + "drift/chosen_abs": 0.3295367658138275, + "drift/rejected_abs": 0.16237182915210724, + "epoch": 0.23408575182133046, + "grad_norm": 552.0, + "learning_rate": 2.2835781343782966e-07, + "logits/chosen": -1.8566938638687134, + "logits/rejected": -1.875113844871521, + "logps/chosen": -0.2936326861381531, + "logps/rejected": -0.29784101247787476, + "loss": 1.379624843597412, + "loss/core": 1.3653779029846191, + "loss/preference_sft": 0.2936326861381531, + "loss/reference_anchor": 0.6829947829246521, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.2930209636688232, + "rewards/margins": 1.6766458749771118, + "rewards/rejected": 1.6163749694824219, + "step": 245 + }, + { + "drift/chosen_abs": 0.15949814021587372, + "drift/rejected_abs": 0.10824527591466904, + "epoch": 0.23886301206258212, + "grad_norm": 0.53125, + "learning_rate": 2.2697515850552152e-07, + "logits/chosen": -2.3425869941711426, + "logits/rejected": -2.325498104095459, + "logps/chosen": -0.2886843681335449, + "logps/rejected": -0.2921077609062195, + "loss": 0.29327505826950073, + "loss/core": 0.28979676961898804, + "loss/preference_sft": 0.2886843681335449, + "loss/reference_anchor": 0.14504632353782654, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.5941321849822998, + "rewards/margins": 0.5128880739212036, + "rewards/rejected": 1.0812442302703857, + "step": 250 + }, + { + "drift/chosen_abs": 0.14964722096920013, + "drift/rejected_abs": 0.05588113144040108, + "epoch": 0.24364027230383375, + "grad_norm": 0.63671875, + "learning_rate": 2.2555415488657775e-07, + "logits/chosen": -2.117321729660034, + "logits/rejected": -2.180022716522217, + "logps/chosen": -0.3025610148906708, + "logps/rejected": -0.30650925636291504, + "loss": 0.14917774498462677, + "loss/core": 0.14709773659706116, + "loss/preference_sft": 0.3025610148906708, + "loss/reference_anchor": 0.07374367862939835, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 1.4962538480758667, + "rewards/margins": 0.9412781000137329, + "rewards/rejected": 0.5549757480621338, + "step": 255 + }, + { + "drift/chosen_abs": 0.23837116360664368, + "drift/rejected_abs": 0.1084081158041954, + "epoch": 0.2484175325450854, + "grad_norm": 4.59375, + "learning_rate": 2.240953369623447e-07, + "logits/chosen": -2.271860122680664, + "logits/rejected": -2.326420307159424, + "logps/chosen": -0.2747231125831604, + "logps/rejected": -0.276181697845459, + "loss": 0.3998209536075592, + "loss/core": 0.3953119218349457, + "loss/preference_sft": 0.2747231125831604, + "loss/reference_anchor": 0.19797541201114655, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.3820481300354004, + "rewards/margins": 1.2980557680130005, + "rewards/rejected": 1.0839922428131104, + "step": 260 + }, + { + "drift/chosen_abs": 0.14227482676506042, + "drift/rejected_abs": 0.07095784693956375, + "epoch": 0.25319479278633705, + "grad_norm": 1.3828125, + "learning_rate": 2.225992533345824e-07, + "logits/chosen": -2.364943265914917, + "logits/rejected": -2.4003348350524902, + "logps/chosen": -0.2837630808353424, + "logps/rejected": -0.28678828477859497, + "loss": 0.1420137584209442, + "loss/core": 0.14004194736480713, + "loss/preference_sft": 0.2837630808353424, + "loss/reference_anchor": 0.07021477073431015, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.422141671180725, + "rewards/margins": 0.7143291234970093, + "rewards/rejected": 0.7078123688697815, + "step": 265 + }, + { + "drift/chosen_abs": 0.18406164646148682, + "drift/rejected_abs": 0.0678694099187851, + "epoch": 0.2579720530275887, + "grad_norm": 12.5, + "learning_rate": 2.210664666191579e-07, + "logits/chosen": -2.1580262184143066, + "logits/rejected": -2.2362945079803467, + "logps/chosen": -0.3155515789985657, + "logps/rejected": -0.29309743642807007, + "loss": 0.20326463878154755, + "loss/core": 0.20061977207660675, + "loss/preference_sft": 0.3155515789985657, + "loss/reference_anchor": 0.10068801790475845, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.839700698852539, + "rewards/margins": 1.163073182106018, + "rewards/rejected": 0.6766273379325867, + "step": 270 + }, + { + "drift/chosen_abs": 0.24695925414562225, + "drift/rejected_abs": 0.11996563524007797, + "epoch": 0.2627493132688403, + "grad_norm": 17.25, + "learning_rate": 2.1949755323446848e-07, + "logits/chosen": -2.250136375427246, + "logits/rejected": -2.232349157333374, + "logps/chosen": -0.2562440037727356, + "logps/rejected": -0.2609252333641052, + "loss": 0.5029779672622681, + "loss/core": 0.49748414754867554, + "loss/preference_sft": 0.2562440037727356, + "loss/reference_anchor": 0.2490653693675995, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.4680981636047363, + "rewards/margins": 1.2709671258926392, + "rewards/rejected": 1.1971310377120972, + "step": 275 + }, + { + "drift/chosen_abs": 0.2423405647277832, + "drift/rejected_abs": 0.13960115611553192, + "epoch": 0.26752657351009196, + "grad_norm": 27.375, + "learning_rate": 2.1789310318467426e-07, + "logits/chosen": -1.8027242422103882, + "logits/rejected": -1.8857799768447876, + "logps/chosen": -0.3176387846469879, + "logps/rejected": -0.3207781910896301, + "loss": 0.3058789372444153, + "loss/core": 0.3022173047065735, + "loss/preference_sft": 0.3176387846469879, + "loss/reference_anchor": 0.1513180285692215, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.4204940795898438, + "rewards/margins": 1.0279791355133057, + "rewards/rejected": 1.3925148248672485, + "step": 280 + }, + { + "drift/chosen_abs": 0.2865580916404724, + "drift/rejected_abs": 0.1650657057762146, + "epoch": 0.2723038337513436, + "grad_norm": 84.0, + "learning_rate": 2.1625371983782182e-07, + "logits/chosen": -2.08091402053833, + "logits/rejected": -2.043867826461792, + "logps/chosen": -0.2672872543334961, + "logps/rejected": -0.2768842577934265, + "loss": 0.6620380282402039, + "loss/core": 0.6549504995346069, + "loss/preference_sft": 0.2672872543334961, + "loss/reference_anchor": 0.32764729857444763, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.865417242050171, + "rewards/margins": 1.2915067672729492, + "rewards/rejected": 1.5739107131958008, + "step": 285 + }, + { + "drift/chosen_abs": 0.20399828255176544, + "drift/rejected_abs": 0.08803847432136536, + "epoch": 0.27708109399259523, + "grad_norm": 20.25, + "learning_rate": 2.14580019698942e-07, + "logits/chosen": -2.1715924739837646, + "logits/rejected": -2.2407965660095215, + "logps/chosen": -0.2938459813594818, + "logps/rejected": -0.2942553758621216, + "loss": 0.3727453947067261, + "loss/core": 0.3684656322002411, + "loss/preference_sft": 0.2938459813594818, + "loss/reference_anchor": 0.18460234999656677, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.0391552448272705, + "rewards/margins": 1.1640946865081787, + "rewards/rejected": 0.8750606775283813, + "step": 290 + }, + { + "drift/chosen_abs": 0.22580035030841827, + "drift/rejected_abs": 0.16777771711349487, + "epoch": 0.28185835423384686, + "grad_norm": 652.0, + "learning_rate": 2.1287263217820773e-07, + "logits/chosen": -2.1450181007385254, + "logits/rejected": -2.1244781017303467, + "logps/chosen": -0.2750265300273895, + "logps/rejected": -0.2840802073478699, + "loss": 0.6499078273773193, + "loss/core": 0.6429253816604614, + "loss/preference_sft": 0.2750265300273895, + "loss/reference_anchor": 0.3216193616390228, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.2560317516326904, + "rewards/margins": 0.5812076330184937, + "rewards/rejected": 1.6748244762420654, + "step": 295 + }, + { + "drift/chosen_abs": 0.27815431356430054, + "drift/rejected_abs": 0.13376235961914062, + "epoch": 0.28663561447509855, + "grad_norm": 11.5, + "learning_rate": 2.111321993542385e-07, + "logits/chosen": -1.9923107624053955, + "logits/rejected": -1.9791291952133179, + "logps/chosen": -0.2681702673435211, + "logps/rejected": -0.28962358832359314, + "loss": 0.390531450510025, + "loss/core": 0.38612592220306396, + "loss/preference_sft": 0.2681702673435211, + "loss/reference_anchor": 0.19345811009407043, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.778730869293213, + "rewards/margins": 1.4537692070007324, + "rewards/rejected": 1.3249620199203491, + "step": 300 + }, + { + "drift/chosen_abs": 0.18262216448783875, + "drift/rejected_abs": 0.1317087560892105, + "epoch": 0.2914128747163502, + "grad_norm": 55.0, + "learning_rate": 2.0935937573264096e-07, + "logits/chosen": -2.0926907062530518, + "logits/rejected": -2.105894088745117, + "logps/chosen": -0.28904959559440613, + "logps/rejected": -0.30133944749832153, + "loss": 0.2560952305793762, + "loss/core": 0.2529827654361725, + "loss/preference_sft": 0.28904959559440613, + "loss/reference_anchor": 0.1267165243625641, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.8257789611816406, + "rewards/margins": 0.510686993598938, + "rewards/rejected": 1.315091848373413, + "step": 305 + }, + { + "drift/chosen_abs": 0.20964379608631134, + "drift/rejected_abs": 0.13515421748161316, + "epoch": 0.2961901349576018, + "grad_norm": 7.53125, + "learning_rate": 2.0755482799987596e-07, + "logits/chosen": -2.020278215408325, + "logits/rejected": -1.9860916137695312, + "logps/chosen": -0.29890328645706177, + "logps/rejected": -0.3041743338108063, + "loss": 0.5346760749816895, + "loss/core": 0.5287860035896301, + "loss/preference_sft": 0.29890328645706177, + "loss/reference_anchor": 0.2646165192127228, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.094669818878174, + "rewards/margins": 0.7476687431335449, + "rewards/rejected": 1.347001075744629, + "step": 310 + }, + { + "drift/chosen_abs": 0.15830251574516296, + "drift/rejected_abs": 0.09903346002101898, + "epoch": 0.30096739519885346, + "grad_norm": 3.578125, + "learning_rate": 2.0571923477254526e-07, + "logits/chosen": -2.271230697631836, + "logits/rejected": -2.2453927993774414, + "logps/chosen": -0.29073256254196167, + "logps/rejected": -0.290340393781662, + "loss": 0.21692290902137756, + "loss/core": 0.21419718861579895, + "loss/preference_sft": 0.29073256254196167, + "loss/reference_anchor": 0.10721520334482193, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.582291603088379, + "rewards/margins": 0.6015550494194031, + "rewards/rejected": 0.9807365536689758, + "step": 315 + }, + { + "drift/chosen_abs": 0.28570857644081116, + "drift/rejected_abs": 0.19199606776237488, + "epoch": 0.3057446554401051, + "grad_norm": 2.859375, + "learning_rate": 2.038532863421914e-07, + "logits/chosen": -2.1323790550231934, + "logits/rejected": -2.0531458854675293, + "logps/chosen": -0.29020506143569946, + "logps/rejected": -0.2748698890209198, + "loss": 0.6716278195381165, + "loss/core": 0.6644001007080078, + "loss/preference_sft": 0.29020506143569946, + "loss/reference_anchor": 0.3323640823364258, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.857085704803467, + "rewards/margins": 0.9378870129585266, + "rewards/rejected": 1.919198751449585, + "step": 320 + }, + { + "drift/chosen_abs": 0.15099409222602844, + "drift/rejected_abs": 0.15222987532615662, + "epoch": 0.31052191568135673, + "grad_norm": 6.4375, + "learning_rate": 2.0195768441570726e-07, + "logits/chosen": -1.9938151836395264, + "logits/rejected": -2.049948215484619, + "logps/chosen": -0.2915424704551697, + "logps/rejected": -0.28644031286239624, + "loss": 0.6587117910385132, + "loss/core": 0.6516103148460388, + "loss/preference_sft": 0.2915424704551697, + "loss/reference_anchor": 0.3259239196777344, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.5096241235733032, + "rewards/margins": -0.008662676438689232, + "rewards/rejected": 1.5182867050170898, + "step": 325 + }, + { + "drift/chosen_abs": 0.2591181993484497, + "drift/rejected_abs": 0.11437433958053589, + "epoch": 0.31529917592260837, + "grad_norm": 18.75, + "learning_rate": 2.0003314185145307e-07, + "logits/chosen": -2.145712375640869, + "logits/rejected": -2.0909924507141113, + "logps/chosen": -0.28042882680892944, + "logps/rejected": -0.2842772603034973, + "loss": 0.46702370047569275, + "loss/core": 0.46183985471725464, + "loss/preference_sft": 0.28042882680892944, + "loss/reference_anchor": 0.23115257918834686, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.5909152030944824, + "rewards/margins": 1.4475888013839722, + "rewards/rejected": 1.1433266401290894, + "step": 330 + }, + { + "drift/chosen_abs": 0.2697688341140747, + "drift/rejected_abs": 0.1486051231622696, + "epoch": 0.32007643616386, + "grad_norm": 26.5, + "learning_rate": 1.9808038239117913e-07, + "logits/chosen": -1.9927841424942017, + "logits/rejected": -1.9601898193359375, + "logps/chosen": -0.2884038984775543, + "logps/rejected": -0.2860269844532013, + "loss": 0.4839557111263275, + "loss/core": 0.4785878658294678, + "loss/preference_sft": 0.2884038984775543, + "loss/reference_anchor": 0.23955455422401428, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.6973326206207275, + "rewards/margins": 1.2938122749328613, + "rewards/rejected": 1.4035199880599976, + "step": 335 + }, + { + "drift/chosen_abs": 0.16600827872753143, + "drift/rejected_abs": 0.08406911045312881, + "epoch": 0.3248536964051117, + "grad_norm": 8.5, + "learning_rate": 1.9610014038785646e-07, + "logits/chosen": -2.222121477127075, + "logits/rejected": -2.2075448036193848, + "logps/chosen": -0.2964716851711273, + "logps/rejected": -0.30573147535324097, + "loss": 0.17239664494991302, + "loss/core": 0.17009879648685455, + "loss/preference_sft": 0.2964716851711273, + "loss/reference_anchor": 0.08524533361196518, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6584606170654297, + "rewards/margins": 0.8196709752082825, + "rewards/rejected": 0.8387895822525024, + "step": 340 + }, + { + "drift/chosen_abs": 0.2879965305328369, + "drift/rejected_abs": 0.2307172268629074, + "epoch": 0.3296309566463633, + "grad_norm": 20.75, + "learning_rate": 1.9409316052951657e-07, + "logits/chosen": -1.9839036464691162, + "logits/rejected": -2.0710253715515137, + "logps/chosen": -0.2770405411720276, + "logps/rejected": -0.2928730249404907, + "loss": 1.3981668949127197, + "loss/core": 1.3837751150131226, + "loss/preference_sft": 0.2770405411720276, + "loss/reference_anchor": 0.6918903589248657, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.87996506690979, + "rewards/margins": 0.5738177299499512, + "rewards/rejected": 2.3061470985412598, + "step": 345 + }, + { + "drift/chosen_abs": 0.3735573887825012, + "drift/rejected_abs": 0.1227908581495285, + "epoch": 0.33440821688761496, + "grad_norm": 75.5, + "learning_rate": 1.920601975592047e-07, + "logits/chosen": -2.171405076980591, + "logits/rejected": -2.1451284885406494, + "logps/chosen": -0.2867693305015564, + "logps/rejected": -0.29171350598335266, + "loss": 1.53303062915802, + "loss/core": 1.5172761678695679, + "loss/preference_sft": 0.2867693305015564, + "loss/reference_anchor": 0.7590527534484863, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.7349324226379395, + "rewards/margins": 2.521451473236084, + "rewards/rejected": 1.2134809494018555, + "step": 350 + }, + { + "drift/chosen_abs": 0.1963830143213272, + "drift/rejected_abs": 0.07833238691091537, + "epoch": 0.3391854771288666, + "grad_norm": 1.8203125, + "learning_rate": 1.900020159911519e-07, + "logits/chosen": -2.0924370288848877, + "logits/rejected": -2.159611701965332, + "logps/chosen": -0.3114902377128601, + "logps/rejected": -0.31218671798706055, + "loss": 0.28884080052375793, + "loss/core": 0.2853604257106781, + "loss/preference_sft": 0.3114902377128601, + "loss/reference_anchor": 0.14286813139915466, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 1.9624801874160767, + "rewards/margins": 1.1923706531524658, + "rewards/rejected": 0.7701095342636108, + "step": 355 + }, + { + "drift/chosen_abs": 0.1754159778356552, + "drift/rejected_abs": 0.18023176491260529, + "epoch": 0.34396273737011823, + "grad_norm": 21.5, + "learning_rate": 1.879193898232725e-07, + "logits/chosen": -2.1387431621551514, + "logits/rejected": -2.1139893531799316, + "logps/chosen": -0.3140478730201721, + "logps/rejected": -0.3199792206287384, + "loss": 0.39963623881340027, + "loss/core": 0.39505690336227417, + "loss/preference_sft": 0.3140478730201721, + "loss/reference_anchor": 0.19756412506103516, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.754159927368164, + "rewards/margins": -0.04186423867940903, + "rewards/rejected": 1.796023964881897, + "step": 360 + }, + { + "drift/chosen_abs": 0.37004557251930237, + "drift/rejected_abs": 0.08763270825147629, + "epoch": 0.34873999761136987, + "grad_norm": 59.5, + "learning_rate": 1.8581310224609496e-07, + "logits/chosen": -2.1298484802246094, + "logits/rejected": -2.202394962310791, + "logps/chosen": -0.2909151315689087, + "logps/rejected": -0.2845023572444916, + "loss": 1.6612498760223389, + "loss/core": 1.6442168951034546, + "loss/preference_sft": 0.2909151315689087, + "loss/reference_anchor": 0.8225665092468262, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.698554277420044, + "rewards/margins": 2.8230626583099365, + "rewards/rejected": 0.8754919767379761, + "step": 365 + }, + { + "drift/chosen_abs": 0.29484832286834717, + "drift/rejected_abs": 0.12339384853839874, + "epoch": 0.3535172578526215, + "grad_norm": 30.125, + "learning_rate": 1.8368394534823635e-07, + "logits/chosen": -2.0275933742523193, + "logits/rejected": -2.0372328758239746, + "logps/chosen": -0.2683495581150055, + "logps/rejected": -0.2806262671947479, + "loss": 0.5110516548156738, + "loss/core": 0.5054527521133423, + "loss/preference_sft": 0.2683495581150055, + "loss/reference_anchor": 0.2531072795391083, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.9470877647399902, + "rewards/margins": 1.7142242193222046, + "rewards/rejected": 1.2328637838363647, + "step": 370 + }, + { + "drift/chosen_abs": 0.4893800616264343, + "drift/rejected_abs": 0.23103968799114227, + "epoch": 0.35829451809387314, + "grad_norm": 10.0625, + "learning_rate": 1.8153271981852968e-07, + "logits/chosen": -2.004993200302124, + "logits/rejected": -1.9773603677749634, + "logps/chosen": -0.3077290654182434, + "logps/rejected": -0.2921420633792877, + "loss": 3.063831329345703, + "loss/core": 3.032869338989258, + "loss/preference_sft": 0.3077290654182434, + "loss/reference_anchor": 1.5173249244689941, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 4.893084526062012, + "rewards/margins": 2.5864005088806152, + "rewards/rejected": 2.3066840171813965, + "step": 375 + }, + { + "drift/chosen_abs": 0.22893986105918884, + "drift/rejected_abs": 0.13307783007621765, + "epoch": 0.36307177833512483, + "grad_norm": 156.0, + "learning_rate": 1.7936023464491812e-07, + "logits/chosen": -2.0798957347869873, + "logits/rejected": -2.190671920776367, + "logps/chosen": -0.2794080078601837, + "logps/rejected": -0.27889859676361084, + "loss": 0.7344735860824585, + "loss/core": 0.7266415357589722, + "loss/preference_sft": 0.2794080078601837, + "loss/reference_anchor": 0.3636623024940491, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.2871289253234863, + "rewards/margins": 0.9574325680732727, + "rewards/rejected": 1.3296962976455688, + "step": 380 + }, + { + "drift/chosen_abs": 0.166697159409523, + "drift/rejected_abs": 0.10592110455036163, + "epoch": 0.36784903857637646, + "grad_norm": 21.0, + "learning_rate": 1.7716730681022723e-07, + "logits/chosen": -2.1135611534118652, + "logits/rejected": -2.0744059085845947, + "logps/chosen": -0.31136012077331543, + "logps/rejected": -0.2861802577972412, + "loss": 0.23299483954906464, + "loss/core": 0.23006677627563477, + "loss/preference_sft": 0.31136012077331543, + "loss/reference_anchor": 0.11526670306921005, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.6641048192977905, + "rewards/margins": 0.6075918078422546, + "rewards/rejected": 1.0565129518508911, + "step": 385 + }, + { + "drift/chosen_abs": 0.22516271471977234, + "drift/rejected_abs": 0.1247841864824295, + "epoch": 0.3726262988176281, + "grad_norm": 1.9453125, + "learning_rate": 1.7495476098493152e-07, + "logits/chosen": -2.1953511238098145, + "logits/rejected": -2.2125370502471924, + "logps/chosen": -0.27903926372528076, + "logps/rejected": -0.28518766164779663, + "loss": 0.3761298954486847, + "loss/core": 0.37184834480285645, + "loss/preference_sft": 0.27903926372528076, + "loss/reference_anchor": 0.18617263436317444, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.250232219696045, + "rewards/margins": 1.0177348852157593, + "rewards/rejected": 1.232497215270996, + "step": 390 + }, + { + "drift/chosen_abs": 0.3017219603061676, + "drift/rejected_abs": 0.16852721571922302, + "epoch": 0.37740355905887973, + "grad_norm": 3.484375, + "learning_rate": 1.7272342921702937e-07, + "logits/chosen": -2.087266445159912, + "logits/rejected": -2.1111271381378174, + "logps/chosen": -0.3051111102104187, + "logps/rejected": -0.30405479669570923, + "loss": 1.1819294691085815, + "loss/core": 1.1696186065673828, + "loss/preference_sft": 0.3051111102104187, + "loss/reference_anchor": 0.5850434303283691, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.0164356231689453, + "rewards/margins": 1.3323934078216553, + "rewards/rejected": 1.6840423345565796, + "step": 395 + }, + { + "drift/chosen_abs": 0.27575579285621643, + "drift/rejected_abs": 0.20994627475738525, + "epoch": 0.38218081930013137, + "grad_norm": 2.453125, + "learning_rate": 1.7047415061914393e-07, + "logits/chosen": -2.057664632797241, + "logits/rejected": -2.212043046951294, + "logps/chosen": -0.2742888331413269, + "logps/rejected": -0.2956988215446472, + "loss": 1.1588386297225952, + "loss/core": 1.1468188762664795, + "loss/preference_sft": 0.2742888331413269, + "loss/reference_anchor": 0.5735639333724976, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.7565925121307373, + "rewards/margins": 0.6574581861495972, + "rewards/rejected": 2.0991344451904297, + "step": 400 + }, + { + "drift/chosen_abs": 0.25579676032066345, + "drift/rejected_abs": 0.1272418051958084, + "epoch": 0.386958079541383, + "grad_norm": 202.0, + "learning_rate": 1.6820777105296707e-07, + "logits/chosen": -2.1076152324676514, + "logits/rejected": -2.20607590675354, + "logps/chosen": -0.3273892402648926, + "logps/rejected": -0.28913551568984985, + "loss": 0.8974112272262573, + "loss/core": 0.8878701329231262, + "loss/preference_sft": 0.3273892402648926, + "loss/reference_anchor": 0.4443127512931824, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.552988052368164, + "rewards/margins": 1.2856799364089966, + "rewards/rejected": 1.2673081159591675, + "step": 405 + }, + { + "drift/chosen_abs": 0.2772047519683838, + "drift/rejected_abs": 0.1283237189054489, + "epoch": 0.39173533978263464, + "grad_norm": 1.5703125, + "learning_rate": 1.6592514281116553e-07, + "logits/chosen": -2.0812337398529053, + "logits/rejected": -2.0970664024353027, + "logps/chosen": -0.26817846298217773, + "logps/rejected": -0.2727079689502716, + "loss": 0.5803694725036621, + "loss/core": 0.574087381362915, + "loss/preference_sft": 0.26817846298217773, + "loss/reference_anchor": 0.2872823476791382, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.771658420562744, + "rewards/margins": 1.4890962839126587, + "rewards/rejected": 1.2825623750686646, + "step": 410 + }, + { + "drift/chosen_abs": 0.23107464611530304, + "drift/rejected_abs": 0.0726277232170105, + "epoch": 0.3965126000238863, + "grad_norm": 10.5625, + "learning_rate": 1.636271242968684e-07, + "logits/chosen": -2.0305263996124268, + "logits/rejected": -2.0474953651428223, + "logps/chosen": -0.30018436908721924, + "logps/rejected": -0.2916948199272156, + "loss": 0.43483519554138184, + "loss/core": 0.42992615699768066, + "loss/preference_sft": 0.30018436908721924, + "loss/reference_anchor": 0.21543259918689728, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.310746431350708, + "rewards/margins": 1.701195478439331, + "rewards/rejected": 0.6095508933067322, + "step": 415 + }, + { + "drift/chosen_abs": 0.32632407546043396, + "drift/rejected_abs": 0.18786516785621643, + "epoch": 0.40128986026513797, + "grad_norm": 7.3125, + "learning_rate": 1.6131457970085684e-07, + "logits/chosen": -1.924475908279419, + "logits/rejected": -1.9443466663360596, + "logps/chosen": -0.2839311957359314, + "logps/rejected": -0.2862430214881897, + "loss": 0.9467588663101196, + "loss/core": 0.9368128776550293, + "loss/preference_sft": 0.2839311957359314, + "loss/reference_anchor": 0.4689028859138489, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.2614026069641113, + "rewards/margins": 1.3844013214111328, + "rewards/rejected": 1.8770010471343994, + "step": 420 + }, + { + "drift/chosen_abs": 0.29126811027526855, + "drift/rejected_abs": 0.19718101620674133, + "epoch": 0.4060671205063896, + "grad_norm": 98.0, + "learning_rate": 1.5898837867657727e-07, + "logits/chosen": -1.9464651346206665, + "logits/rejected": -1.9939228296279907, + "logps/chosen": -0.2907729744911194, + "logps/rejected": -0.27748432755470276, + "loss": 0.9655182957649231, + "loss/core": 0.9553742408752441, + "loss/preference_sft": 0.2907729744911194, + "loss/reference_anchor": 0.4781121611595154, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.9124670028686523, + "rewards/margins": 0.9434226751327515, + "rewards/rejected": 1.9690440893173218, + "step": 425 + }, + { + "drift/chosen_abs": 0.2520788609981537, + "drift/rejected_abs": 0.08256536722183228, + "epoch": 0.41084438074764124, + "grad_norm": 9.3125, + "learning_rate": 1.5664939601310023e-07, + "logits/chosen": -2.176443099975586, + "logits/rejected": -2.245333194732666, + "logps/chosen": -0.30696016550064087, + "logps/rejected": -0.3062490224838257, + "loss": 0.846347451210022, + "loss/core": 0.8373504877090454, + "loss/preference_sft": 0.30696016550064087, + "loss/reference_anchor": 0.41915664076805115, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.5197196006774902, + "rewards/margins": 1.6960992813110352, + "rewards/rejected": 0.8236203193664551, + "step": 430 + }, + { + "drift/chosen_abs": 0.24701747298240662, + "drift/rejected_abs": 0.12345242500305176, + "epoch": 0.41562164098889287, + "grad_norm": 3.65625, + "learning_rate": 1.5429851130614807e-07, + "logits/chosen": -2.1048972606658936, + "logits/rejected": -2.033947467803955, + "logps/chosen": -0.2840856909751892, + "logps/rejected": -0.2637600898742676, + "loss": 0.45461979508399963, + "loss/core": 0.4495505392551422, + "loss/preference_sft": 0.2840856909751892, + "loss/reference_anchor": 0.2250523865222931, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.468121290206909, + "rewards/margins": 1.2350847721099854, + "rewards/rejected": 1.233036756515503, + "step": 435 + }, + { + "drift/chosen_abs": 0.21977201104164124, + "drift/rejected_abs": 0.0888163223862648, + "epoch": 0.4203989012301445, + "grad_norm": 88.5, + "learning_rate": 1.51936608627315e-07, + "logits/chosen": -1.996376395225525, + "logits/rejected": -2.0227012634277344, + "logps/chosen": -0.31263792514801025, + "logps/rejected": -0.3157431185245514, + "loss": 0.3650742173194885, + "loss/core": 0.3608359098434448, + "loss/preference_sft": 0.31263792514801025, + "loss/reference_anchor": 0.1806512176990509, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.1961891651153564, + "rewards/margins": 1.3100742101669312, + "rewards/rejected": 0.8861148953437805, + "step": 440 + }, + { + "drift/chosen_abs": 0.3700529634952545, + "drift/rejected_abs": 0.17902246117591858, + "epoch": 0.42517616147139614, + "grad_norm": 180.0, + "learning_rate": 1.4956457619160375e-07, + "logits/chosen": -2.0445446968078613, + "logits/rejected": -2.02923321723938, + "logps/chosen": -0.2807472348213196, + "logps/rejected": -0.26079219579696655, + "loss": 1.3479831218719482, + "loss/core": 1.3340699672698975, + "loss/preference_sft": 0.2807472348213196, + "loss/reference_anchor": 0.6675775647163391, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.700012683868408, + "rewards/margins": 1.9215444326400757, + "rewards/rejected": 1.778468132019043, + "step": 445 + }, + { + "drift/chosen_abs": 0.19167892634868622, + "drift/rejected_abs": 0.08485953509807587, + "epoch": 0.4299534217126478, + "grad_norm": 5.6875, + "learning_rate": 1.471833060234044e-07, + "logits/chosen": -2.1029083728790283, + "logits/rejected": -2.1657981872558594, + "logps/chosen": -0.30128949880599976, + "logps/rejected": -0.3000229001045227, + "loss": 0.2482064664363861, + "loss/core": 0.24514798820018768, + "loss/preference_sft": 0.30128949880599976, + "loss/reference_anchor": 0.12279714643955231, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9167217016220093, + "rewards/margins": 1.0688178539276123, + "rewards/rejected": 0.847903847694397, + "step": 450 + }, + { + "drift/chosen_abs": 0.2169889211654663, + "drift/rejected_abs": 0.13156619668006897, + "epoch": 0.4347306819538994, + "grad_norm": 46.5, + "learning_rate": 1.4479369362104037e-07, + "logits/chosen": -2.13124418258667, + "logits/rejected": -2.2444398403167725, + "logps/chosen": -0.2661250829696655, + "logps/rejected": -0.28118157386779785, + "loss": 0.36354073882102966, + "loss/core": 0.35940924286842346, + "loss/preference_sft": 0.2661250829696655, + "loss/reference_anchor": 0.17996393144130707, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.1693220138549805, + "rewards/margins": 0.8696813583374023, + "rewards/rejected": 1.2996408939361572, + "step": 455 + }, + { + "drift/chosen_abs": 0.20733904838562012, + "drift/rejected_abs": 0.09261975437402725, + "epoch": 0.4395079421951511, + "grad_norm": 3.65625, + "learning_rate": 1.4239663762000817e-07, + "logits/chosen": -2.1311872005462646, + "logits/rejected": -2.0629684925079346, + "logps/chosen": -0.29252859950065613, + "logps/rejected": -0.2915221154689789, + "loss": 0.34754329919815063, + "loss/core": 0.3435174524784088, + "loss/preference_sft": 0.29252859950065613, + "loss/reference_anchor": 0.17203818261623383, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.070737361907959, + "rewards/margins": 1.146690845489502, + "rewards/rejected": 0.924046516418457, + "step": 460 + }, + { + "drift/chosen_abs": 0.25771060585975647, + "drift/rejected_abs": 0.14189331233501434, + "epoch": 0.44428520243640274, + "grad_norm": 96.5, + "learning_rate": 1.3999303945503747e-07, + "logits/chosen": -2.1602511405944824, + "logits/rejected": -2.2427048683166504, + "logps/chosen": -0.3139217793941498, + "logps/rejected": -0.3135758340358734, + "loss": 0.8517921566963196, + "loss/core": 0.842727780342102, + "loss/preference_sft": 0.3139217793941498, + "loss/reference_anchor": 0.4218299984931946, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.5764236450195312, + "rewards/margins": 1.158347487449646, + "rewards/rejected": 1.4180762767791748, + "step": 465 + }, + { + "drift/chosen_abs": 0.15380379557609558, + "drift/rejected_abs": 0.10157140344381332, + "epoch": 0.4490624626776544, + "grad_norm": 4.71875, + "learning_rate": 1.3758380302109808e-07, + "logits/chosen": -2.1404902935028076, + "logits/rejected": -2.1485772132873535, + "logps/chosen": -0.2958316206932068, + "logps/rejected": -0.26554960012435913, + "loss": 0.1802210509777069, + "loss/core": 0.17784848809242249, + "loss/preference_sft": 0.2958316206932068, + "loss/reference_anchor": 0.08904588967561722, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.537377953529358, + "rewards/margins": 0.5247774124145508, + "rewards/rejected": 1.0126006603240967, + "step": 470 + }, + { + "drift/chosen_abs": 0.1924518495798111, + "drift/rejected_abs": 0.0540982261300087, + "epoch": 0.453839722918906, + "grad_norm": 1.3125, + "learning_rate": 1.351698343334823e-07, + "logits/chosen": -2.097944498062134, + "logits/rejected": -2.1556663513183594, + "logps/chosen": -0.3016926348209381, + "logps/rejected": -0.29718315601348877, + "loss": 0.4531112611293793, + "loss/core": 0.44802337884902954, + "loss/preference_sft": 0.3016926348209381, + "loss/reference_anchor": 0.22422640025615692, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.9209903478622437, + "rewards/margins": 1.3928687572479248, + "rewards/rejected": 0.5281215906143188, + "step": 475 + }, + { + "drift/chosen_abs": 0.24358682334423065, + "drift/rejected_abs": 0.13433735072612762, + "epoch": 0.45861698316015764, + "grad_norm": 219.0, + "learning_rate": 1.3275204118708942e-07, + "logits/chosen": -2.0871355533599854, + "logits/rejected": -1.9663331508636475, + "logps/chosen": -0.3032604455947876, + "logps/rejected": -0.3135993778705597, + "loss": 1.2207492589950562, + "loss/core": 1.2080614566802979, + "loss/preference_sft": 0.3032604455947876, + "loss/reference_anchor": 0.6040719747543335, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.4357502460479736, + "rewards/margins": 1.0935547351837158, + "rewards/rejected": 1.3421953916549683, + "step": 480 + }, + { + "drift/chosen_abs": 0.2088780403137207, + "drift/rejected_abs": 0.10938602685928345, + "epoch": 0.4633942434014093, + "grad_norm": 105.5, + "learning_rate": 1.3033133281504132e-07, + "logits/chosen": -1.9471505880355835, + "logits/rejected": -1.9641081094741821, + "logps/chosen": -0.2977781295776367, + "logps/rejected": -0.2958124577999115, + "loss": 0.46540889143943787, + "loss/core": 0.4602069854736328, + "loss/preference_sft": 0.2977781295776367, + "loss/reference_anchor": 0.23032017052173615, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.0887653827667236, + "rewards/margins": 0.9960877299308777, + "rewards/rejected": 1.0926777124404907, + "step": 485 + }, + { + "drift/chosen_abs": 0.23758061230182648, + "drift/rejected_abs": 0.11018860340118408, + "epoch": 0.4681715036426609, + "grad_norm": 10.0625, + "learning_rate": 1.2790861954675702e-07, + "logits/chosen": -2.0518007278442383, + "logits/rejected": -2.0877254009246826, + "logps/chosen": -0.31017646193504333, + "logps/rejected": -0.3069363832473755, + "loss": 0.29436618089675903, + "loss/core": 0.2908337116241455, + "loss/preference_sft": 0.31017646193504333, + "loss/reference_anchor": 0.1456061452627182, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.374509334564209, + "rewards/margins": 1.2785499095916748, + "rewards/rejected": 1.0959594249725342, + "step": 490 + }, + { + "drift/chosen_abs": 0.1931905448436737, + "drift/rejected_abs": 0.09922536462545395, + "epoch": 0.47294876388391255, + "grad_norm": 47.5, + "learning_rate": 1.2548481246561504e-07, + "logits/chosen": -2.055600643157959, + "logits/rejected": -2.0807411670684814, + "logps/chosen": -0.28315284848213196, + "logps/rejected": -0.28561973571777344, + "loss": 0.28335708379745483, + "loss/core": 0.2799864411354065, + "loss/preference_sft": 0.28315284848213196, + "loss/reference_anchor": 0.14021827280521393, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9313852787017822, + "rewards/margins": 0.9417422413825989, + "rewards/rejected": 0.9896429181098938, + "step": 495 + }, + { + "drift/chosen_abs": 0.3655059039592743, + "drift/rejected_abs": 0.16599538922309875, + "epoch": 0.47772602412516424, + "grad_norm": 400.0, + "learning_rate": 1.230608230663321e-07, + "logits/chosen": -2.061601161956787, + "logits/rejected": -2.0343308448791504, + "logps/chosen": -0.28122663497924805, + "logps/rejected": -0.2838439345359802, + "loss": 1.412589430809021, + "loss/core": 1.3980388641357422, + "loss/preference_sft": 0.28122663497924805, + "loss/reference_anchor": 0.6994101405143738, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.652092456817627, + "rewards/margins": 2.0014448165893555, + "rewards/rejected": 1.650647759437561, + "step": 500 + }, + { + "drift/chosen_abs": 0.34218084812164307, + "drift/rejected_abs": 0.13698555529117584, + "epoch": 0.4825032843664159, + "grad_norm": 3.296875, + "learning_rate": 1.206375629121874e-07, + "logits/chosen": -2.0119738578796387, + "logits/rejected": -2.002357006072998, + "logps/chosen": -0.31075453758239746, + "logps/rejected": -0.31518369913101196, + "loss": 0.8807415962219238, + "loss/core": 0.8713995814323425, + "loss/preference_sft": 0.31075453758239746, + "loss/reference_anchor": 0.436026394367218, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.421598434448242, + "rewards/margins": 2.0699105262756348, + "rewards/rejected": 1.3516877889633179, + "step": 505 + }, + { + "drift/chosen_abs": 0.23318608105182648, + "drift/rejected_abs": 0.12134194374084473, + "epoch": 0.4872805446076675, + "grad_norm": 9.3125, + "learning_rate": 1.1821594329222079e-07, + "logits/chosen": -2.206329584121704, + "logits/rejected": -2.1498043537139893, + "logps/chosen": -0.29152804613113403, + "logps/rejected": -0.28346484899520874, + "loss": 1.038659691810608, + "loss/core": 1.027789831161499, + "loss/preference_sft": 0.29152804613113403, + "loss/reference_anchor": 0.5143306851387024, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.331209897994995, + "rewards/margins": 1.118821382522583, + "rewards/rejected": 1.2123886346817017, + "step": 510 + }, + { + "drift/chosen_abs": 0.18352241814136505, + "drift/rejected_abs": 0.1801319718360901, + "epoch": 0.49205780484891914, + "grad_norm": 62.75, + "learning_rate": 1.157968748785344e-07, + "logits/chosen": -1.9077939987182617, + "logits/rejected": -1.9757105112075806, + "logps/chosen": -0.2810531258583069, + "logps/rejected": -0.28047794103622437, + "loss": 0.3822907507419586, + "loss/core": 0.37794721126556396, + "loss/preference_sft": 0.2810531258583069, + "loss/reference_anchor": 0.18907247483730316, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.832468032836914, + "rewards/margins": 0.033732883632183075, + "rewards/rejected": 1.7987350225448608, + "step": 515 + }, + { + "drift/chosen_abs": 0.32048696279525757, + "drift/rejected_abs": 0.1339716911315918, + "epoch": 0.4968350650901708, + "grad_norm": 11.625, + "learning_rate": 1.1338126738382597e-07, + "logits/chosen": -1.9192636013031006, + "logits/rejected": -1.922795295715332, + "logps/chosen": -0.28838467597961426, + "logps/rejected": -0.29027271270751953, + "loss": 1.2226439714431763, + "loss/core": 1.209959864616394, + "loss/preference_sft": 0.28838467597961426, + "loss/reference_anchor": 0.6053675413131714, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.204869508743286, + "rewards/margins": 1.8670326471328735, + "rewards/rejected": 1.3378373384475708, + "step": 520 + }, + { + "drift/chosen_abs": 0.21345123648643494, + "drift/rejected_abs": 0.07307306677103043, + "epoch": 0.5016123253314224, + "grad_norm": 2.796875, + "learning_rate": 1.1097002921928316e-07, + "logits/chosen": -2.0818798542022705, + "logits/rejected": -2.164365291595459, + "logps/chosen": -0.2766956388950348, + "logps/rejected": -0.274677574634552, + "loss": 0.38875702023506165, + "loss/core": 0.3843541443347931, + "loss/preference_sft": 0.2766956388950348, + "loss/reference_anchor": 0.1924724131822586, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.1327221393585205, + "rewards/margins": 1.4029709100723267, + "rewards/rejected": 0.729751467704773, + "step": 525 + }, + { + "drift/chosen_abs": 0.16321828961372375, + "drift/rejected_abs": 0.08229781687259674, + "epoch": 0.5063895855726741, + "grad_norm": 1.96875, + "learning_rate": 1.0856406715296717e-07, + "logits/chosen": -2.1804051399230957, + "logits/rejected": -2.278212070465088, + "logps/chosen": -0.2839670479297638, + "logps/rejected": -0.2922574579715729, + "loss": 0.15642455220222473, + "loss/core": 0.1543111354112625, + "loss/preference_sft": 0.2839670479297638, + "loss/reference_anchor": 0.07727432250976562, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.6320085525512695, + "rewards/margins": 0.8094345331192017, + "rewards/rejected": 0.8225740194320679, + "step": 530 + }, + { + "drift/chosen_abs": 0.334153413772583, + "drift/rejected_abs": 0.12186181545257568, + "epoch": 0.5111668458139257, + "grad_norm": 2.28125, + "learning_rate": 1.061642859688146e-07, + "logits/chosen": -2.1094107627868652, + "logits/rejected": -2.038062334060669, + "logps/chosen": -0.266519695520401, + "logps/rejected": -0.27537840604782104, + "loss": 0.5663573741912842, + "loss/core": 0.560210108757019, + "loss/preference_sft": 0.266519695520401, + "loss/reference_anchor": 0.28070589900016785, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.3413596153259277, + "rewards/margins": 2.1438353061676025, + "rewards/rejected": 1.1975243091583252, + "step": 535 + }, + { + "drift/chosen_abs": 0.2915368974208832, + "drift/rejected_abs": 0.08553571999073029, + "epoch": 0.5159441060551774, + "grad_norm": 402.0, + "learning_rate": 1.0377158812638491e-07, + "logits/chosen": -2.1358208656311035, + "logits/rejected": -2.2264904975891113, + "logps/chosen": -0.2915400564670563, + "logps/rejected": -0.28075969219207764, + "loss": 0.7008722424507141, + "loss/core": 0.6933463215827942, + "loss/preference_sft": 0.2915400564670563, + "loss/reference_anchor": 0.34714406728744507, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.914684534072876, + "rewards/margins": 2.095693588256836, + "rewards/rejected": 0.8189910054206848, + "step": 540 + }, + { + "drift/chosen_abs": 0.2902545928955078, + "drift/rejected_abs": 0.14741960167884827, + "epoch": 0.520721366296429, + "grad_norm": 203.0, + "learning_rate": 1.0138687342148249e-07, + "logits/chosen": -2.057218551635742, + "logits/rejected": -2.1033124923706055, + "logps/chosen": -0.30191048979759216, + "logps/rejected": -0.32187768816947937, + "loss": 1.2491960525512695, + "loss/core": 1.2362278699874878, + "loss/preference_sft": 0.30191048979759216, + "loss/reference_anchor": 0.6182119846343994, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.9013302326202393, + "rewards/margins": 1.42949640750885, + "rewards/rejected": 1.4718337059020996, + "step": 545 + }, + { + "drift/chosen_abs": 0.24866187572479248, + "drift/rejected_abs": 0.20654296875, + "epoch": 0.5254986265376806, + "grad_norm": 4.46875, + "learning_rate": 9.90110386477801e-08, + "logits/chosen": -2.1251871585845947, + "logits/rejected": -2.2004730701446533, + "logps/chosen": -0.2589234709739685, + "logps/rejected": -0.2543538212776184, + "loss": 0.9448872804641724, + "loss/core": 0.935012936592102, + "loss/preference_sft": 0.2589234709739685, + "loss/reference_anchor": 0.46782487630844116, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.4865527153015137, + "rewards/margins": 0.4227861762046814, + "rewards/rejected": 2.0637667179107666, + "step": 550 + }, + { + "drift/chosen_abs": 0.2124369591474533, + "drift/rejected_abs": 0.15426431596279144, + "epoch": 0.5302758867789323, + "grad_norm": 16.625, + "learning_rate": 9.664497725957164e-08, + "logits/chosen": -2.1896917819976807, + "logits/rejected": -2.211423873901367, + "logps/chosen": -0.29176199436187744, + "logps/rejected": -0.27853789925575256, + "loss": 0.46647167205810547, + "loss/core": 0.46127018332481384, + "loss/preference_sft": 0.29176199436187744, + "loss/reference_anchor": 0.2309001386165619, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.1223769187927246, + "rewards/margins": 0.5811587572097778, + "rewards/rejected": 1.5412182807922363, + "step": 555 + }, + { + "drift/chosen_abs": 0.267036497592926, + "drift/rejected_abs": 0.19418852031230927, + "epoch": 0.5350531470201839, + "grad_norm": 241.0, + "learning_rate": 9.428957903578045e-08, + "logits/chosen": -1.961870789527893, + "logits/rejected": -1.9748855829238892, + "logps/chosen": -0.28386375308036804, + "logps/rejected": -0.28560274839401245, + "loss": 1.1362648010253906, + "loss/core": 1.1244474649429321, + "loss/preference_sft": 0.28386375308036804, + "loss/reference_anchor": 0.5624862909317017, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.6667327880859375, + "rewards/margins": 0.7276301980018616, + "rewards/rejected": 1.9391027688980103, + "step": 560 + }, + { + "drift/chosen_abs": 0.348408579826355, + "drift/rejected_abs": 0.19584575295448303, + "epoch": 0.5398304072614356, + "grad_norm": 3.71875, + "learning_rate": 9.194572974534976e-08, + "logits/chosen": -2.02203369140625, + "logits/rejected": -2.0890095233917236, + "logps/chosen": -0.2719302773475647, + "logps/rejected": -0.2636726200580597, + "loss": 1.6560252904891968, + "loss/core": 1.6390788555145264, + "loss/preference_sft": 0.2719302773475647, + "loss/reference_anchor": 0.8201166391372681, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.4829654693603516, + "rewards/margins": 1.5329525470733643, + "rewards/rejected": 1.9500129222869873, + "step": 565 + }, + { + "drift/chosen_abs": 0.22708716988563538, + "drift/rejected_abs": 0.166388139128685, + "epoch": 0.5446076675026872, + "grad_norm": 79.5, + "learning_rate": 8.96143108141412e-08, + "logits/chosen": -1.9567362070083618, + "logits/rejected": -1.9499162435531616, + "logps/chosen": -0.27698129415512085, + "logps/rejected": -0.27503517270088196, + "loss": 0.4375968873500824, + "loss/core": 0.4327124059200287, + "loss/preference_sft": 0.27698129415512085, + "loss/reference_anchor": 0.21652822196483612, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.270296812057495, + "rewards/margins": 0.6087193489074707, + "rewards/rejected": 1.6615772247314453, + "step": 570 + }, + { + "drift/chosen_abs": 0.2241770476102829, + "drift/rejected_abs": 0.08824467658996582, + "epoch": 0.5493849277439389, + "grad_norm": 1.484375, + "learning_rate": 8.72961989934668e-08, + "logits/chosen": -2.2255711555480957, + "logits/rejected": -2.238135814666748, + "logps/chosen": -0.28202304244041443, + "logps/rejected": -0.2808363735675812, + "loss": 0.4496285319328308, + "loss/core": 0.4446125626564026, + "loss/preference_sft": 0.28202304244041443, + "loss/reference_anchor": 0.22259919345378876, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.241278886795044, + "rewards/margins": 1.3592052459716797, + "rewards/rejected": 0.8820737600326538, + "step": 575 + }, + { + "drift/chosen_abs": 0.1400548666715622, + "drift/rejected_abs": 0.10045614093542099, + "epoch": 0.5541621879851905, + "grad_norm": 9.0625, + "learning_rate": 8.499226603037854e-08, + "logits/chosen": -2.2342448234558105, + "logits/rejected": -2.270289659500122, + "logps/chosen": -0.29988688230514526, + "logps/rejected": -0.301006555557251, + "loss": 0.20430853962898254, + "loss/core": 0.20168928802013397, + "loss/preference_sft": 0.29988688230514526, + "loss/reference_anchor": 0.1009727343916893, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.4003478288650513, + "rewards/margins": 0.40323466062545776, + "rewards/rejected": 0.9971132278442383, + "step": 580 + }, + { + "drift/chosen_abs": 0.22656722366809845, + "drift/rejected_abs": 0.10197553783655167, + "epoch": 0.5589394482264421, + "grad_norm": 1.9921875, + "learning_rate": 8.270337833983987e-08, + "logits/chosen": -2.027678966522217, + "logits/rejected": -2.1094918251037598, + "logps/chosen": -0.30382272601127625, + "logps/rejected": -0.29690489172935486, + "loss": 0.5977300405502319, + "loss/core": 0.5912034511566162, + "loss/preference_sft": 0.30382272601127625, + "loss/reference_anchor": 0.2959441840648651, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.2652218341827393, + "rewards/margins": 1.2462141513824463, + "rewards/rejected": 1.019007682800293, + "step": 585 + }, + { + "drift/chosen_abs": 0.21109525859355927, + "drift/rejected_abs": 0.1344117820262909, + "epoch": 0.5637167084676937, + "grad_norm": 42.75, + "learning_rate": 8.04303966789025e-08, + "logits/chosen": -2.124232769012451, + "logits/rejected": -2.143303632736206, + "logps/chosen": -0.2748453617095947, + "logps/rejected": -0.27825242280960083, + "loss": 0.5149407982826233, + "loss/core": 0.5092951655387878, + "loss/preference_sft": 0.2748453617095947, + "loss/reference_anchor": 0.25479501485824585, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.109945297241211, + "rewards/margins": 0.7668979167938232, + "rewards/rejected": 1.3430473804473877, + "step": 590 + }, + { + "drift/chosen_abs": 0.21537485718727112, + "drift/rejected_abs": 0.11827908456325531, + "epoch": 0.5684939687089454, + "grad_norm": 7.4375, + "learning_rate": 7.817417582301098e-08, + "logits/chosen": -1.8684651851654053, + "logits/rejected": -1.888282060623169, + "logps/chosen": -0.28441476821899414, + "logps/rejected": -0.2889414429664612, + "loss": 0.2612078785896301, + "loss/core": 0.2580533027648926, + "loss/preference_sft": 0.28441476821899414, + "loss/reference_anchor": 0.1292877197265625, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.153564929962158, + "rewards/margins": 0.9714028239250183, + "rewards/rejected": 1.1821620464324951, + "step": 595 + }, + { + "drift/chosen_abs": 0.1943017989397049, + "drift/rejected_abs": 0.12395434081554413, + "epoch": 0.5732712289501971, + "grad_norm": 37.25, + "learning_rate": 7.59355642445566e-08, + "logits/chosen": -2.2329204082489014, + "logits/rejected": -2.143657922744751, + "logps/chosen": -0.2619364559650421, + "logps/rejected": -0.271796315908432, + "loss": 0.325387179851532, + "loss/core": 0.3216429650783539, + "loss/preference_sft": 0.2619364559650421, + "loss/reference_anchor": 0.16101622581481934, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9421056509017944, + "rewards/margins": 0.7342923283576965, + "rewards/rejected": 1.2078135013580322, + "step": 600 + }, + { + "drift/chosen_abs": 0.21480529010295868, + "drift/rejected_abs": 0.09283845126628876, + "epoch": 0.5780484891914487, + "grad_norm": 226.0, + "learning_rate": 7.37154037938015e-08, + "logits/chosen": -2.0678582191467285, + "logits/rejected": -2.1681721210479736, + "logps/chosen": -0.32561880350112915, + "logps/rejected": -0.33226823806762695, + "loss": 0.3545982241630554, + "loss/core": 0.3504359722137451, + "loss/preference_sft": 0.32561880350112915, + "loss/reference_anchor": 0.17555202543735504, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.146212339401245, + "rewards/margins": 1.2209621667861938, + "rewards/rejected": 0.925250232219696, + "step": 605 + }, + { + "drift/chosen_abs": 0.21430733799934387, + "drift/rejected_abs": 0.11561570316553116, + "epoch": 0.5828257494327004, + "grad_norm": 6.875, + "learning_rate": 7.151452938229325e-08, + "logits/chosen": -2.048251152038574, + "logits/rejected": -2.0495190620422363, + "logps/chosen": -0.2833295464515686, + "logps/rejected": -0.2934442460536957, + "loss": 0.5116078853607178, + "loss/core": 0.5059792399406433, + "loss/preference_sft": 0.2833295464515686, + "loss/reference_anchor": 0.2530977129936218, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.142573356628418, + "rewards/margins": 0.9865756034851074, + "rewards/rejected": 1.1559979915618896, + "step": 610 + }, + { + "drift/chosen_abs": 0.30491968989372253, + "drift/rejected_abs": 0.16428294777870178, + "epoch": 0.587603009673952, + "grad_norm": 860.0, + "learning_rate": 6.933376866888883e-08, + "logits/chosen": -2.034879446029663, + "logits/rejected": -2.1349658966064453, + "logps/chosen": -0.2771604657173157, + "logps/rejected": -0.3132755160331726, + "loss": 1.1284635066986084, + "loss/core": 1.1167380809783936, + "loss/preference_sft": 0.2771604657173157, + "loss/reference_anchor": 0.558564305305481, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.047480344772339, + "rewards/margins": 1.4195116758346558, + "rewards/rejected": 1.6279685497283936, + "step": 615 + }, + { + "drift/chosen_abs": 0.2592160105705261, + "drift/rejected_abs": 0.14850470423698425, + "epoch": 0.5923802699152036, + "grad_norm": 788.0, + "learning_rate": 6.717394174850605e-08, + "logits/chosen": -2.144191265106201, + "logits/rejected": -2.159964084625244, + "logps/chosen": -0.28378695249557495, + "logps/rejected": -0.2941509485244751, + "loss": 0.699615478515625, + "loss/core": 0.6921217441558838, + "loss/preference_sft": 0.28378695249557495, + "loss/reference_anchor": 0.3463062047958374, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.591275930404663, + "rewards/margins": 1.1079087257385254, + "rewards/rejected": 1.4833673238754272, + "step": 620 + }, + { + "drift/chosen_abs": 0.19440719485282898, + "drift/rejected_abs": 0.1438429057598114, + "epoch": 0.5971575301564552, + "grad_norm": 7.3125, + "learning_rate": 6.503586084371926e-08, + "logits/chosen": -2.1661837100982666, + "logits/rejected": -2.161391019821167, + "logps/chosen": -0.2829047739505768, + "logps/rejected": -0.30115213990211487, + "loss": 0.34582698345184326, + "loss/core": 0.3418407738208771, + "loss/preference_sft": 0.2829047739505768, + "loss/reference_anchor": 0.17102031409740448, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9429868459701538, + "rewards/margins": 0.5073825120925903, + "rewards/rejected": 1.4356043338775635, + "step": 625 + }, + { + "drift/chosen_abs": 0.29011595249176025, + "drift/rejected_abs": 0.13060837984085083, + "epoch": 0.6019347903977069, + "grad_norm": 35.5, + "learning_rate": 6.29203299993155e-08, + "logits/chosen": -2.108480215072632, + "logits/rejected": -2.092172622680664, + "logps/chosen": -0.30079469084739685, + "logps/rejected": -0.3097718358039856, + "loss": 0.8803378939628601, + "loss/core": 0.8710153698921204, + "loss/preference_sft": 0.30079469084739685, + "loss/reference_anchor": 0.4360447824001312, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 2.9011597633361816, + "rewards/margins": 1.9104152917861938, + "rewards/rejected": 0.9907445907592773, + "step": 630 + }, + { + "drift/chosen_abs": 0.17126503586769104, + "drift/rejected_abs": 0.13067471981048584, + "epoch": 0.6067120506389586, + "grad_norm": 50.75, + "learning_rate": 6.082814477992656e-08, + "logits/chosen": -2.184509754180908, + "logits/rejected": -2.1129708290100098, + "logps/chosen": -0.304170161485672, + "logps/rejected": -0.3077109456062317, + "loss": 0.464498370885849, + "loss/core": 0.45929422974586487, + "loss/preference_sft": 0.304170161485672, + "loss/reference_anchor": 0.22978965938091278, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7111921310424805, + "rewards/margins": 0.4084002375602722, + "rewards/rejected": 1.302791953086853, + "step": 635 + }, + { + "drift/chosen_abs": 0.1660601645708084, + "drift/rejected_abs": 0.06357360631227493, + "epoch": 0.6114893108802102, + "grad_norm": 10.0, + "learning_rate": 5.87600919708494e-08, + "logits/chosen": -1.9939651489257812, + "logits/rejected": -2.038660764694214, + "logps/chosen": -0.28754812479019165, + "logps/rejected": -0.2897412180900574, + "loss": 0.26881492137908936, + "loss/core": 0.2655784487724304, + "loss/preference_sft": 0.28754812479019165, + "loss/reference_anchor": 0.13306793570518494, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.6595392227172852, + "rewards/margins": 1.0255340337753296, + "rewards/rejected": 0.634005069732666, + "step": 640 + }, + { + "drift/chosen_abs": 0.20235376060009003, + "drift/rejected_abs": 0.1837695837020874, + "epoch": 0.6162665711214619, + "grad_norm": 200.0, + "learning_rate": 5.671694928216829e-08, + "logits/chosen": -2.2157418727874756, + "logits/rejected": -2.2581088542938232, + "logps/chosen": -0.2847815155982971, + "logps/rejected": -0.2754765748977661, + "loss": 0.5128556489944458, + "loss/core": 0.5072119832038879, + "loss/preference_sft": 0.2847815155982971, + "loss/reference_anchor": 0.25370898842811584, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.0232532024383545, + "rewards/margins": 0.18848243355751038, + "rewards/rejected": 1.8347707986831665, + "step": 645 + }, + { + "drift/chosen_abs": 0.31445202231407166, + "drift/rejected_abs": 0.2099848985671997, + "epoch": 0.6210438313627135, + "grad_norm": 152.0, + "learning_rate": 5.469948505629e-08, + "logits/chosen": -2.0116679668426514, + "logits/rejected": -2.073791980743408, + "logps/chosen": -0.2909887731075287, + "logps/rejected": -0.28746503591537476, + "loss": 1.5114305019378662, + "loss/core": 1.4958856105804443, + "loss/preference_sft": 0.2909887731075287, + "loss/reference_anchor": 0.748149573802948, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.143667697906494, + "rewards/margins": 1.044376254081726, + "rewards/rejected": 2.0992913246154785, + "step": 650 + }, + { + "drift/chosen_abs": 0.17652888596057892, + "drift/rejected_abs": 0.12569592893123627, + "epoch": 0.6258210916039652, + "grad_norm": 157.0, + "learning_rate": 5.270845797900168e-08, + "logits/chosen": -2.215951919555664, + "logits/rejected": -2.147303342819214, + "logps/chosen": -0.2824100852012634, + "logps/rejected": -0.2910049557685852, + "loss": 0.5359405279159546, + "loss/core": 0.5300716757774353, + "loss/preference_sft": 0.2824100852012634, + "loss/reference_anchor": 0.2652043402194977, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.7618181705474854, + "rewards/margins": 0.5078210234642029, + "rewards/rejected": 1.2539972066879272, + "step": 655 + }, + { + "drift/chosen_abs": 0.18071475625038147, + "drift/rejected_abs": 0.14076223969459534, + "epoch": 0.6305983518452167, + "grad_norm": 1.3203125, + "learning_rate": 5.0744616794160104e-08, + "logits/chosen": -2.0582504272460938, + "logits/rejected": -2.1387999057769775, + "logps/chosen": -0.2772466242313385, + "logps/rejected": -0.28088733553886414, + "loss": 0.2379394769668579, + "loss/core": 0.2350325882434845, + "loss/preference_sft": 0.2772466242313385, + "loss/reference_anchor": 0.11762037128210068, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.8062305450439453, + "rewards/margins": 0.40019816160202026, + "rewards/rejected": 1.4060323238372803, + "step": 660 + }, + { + "drift/chosen_abs": 0.22593644261360168, + "drift/rejected_abs": 0.1269337683916092, + "epoch": 0.6353756120864684, + "grad_norm": 8.125, + "learning_rate": 4.880870002211963e-08, + "logits/chosen": -2.240903854370117, + "logits/rejected": -2.2479407787323, + "logps/chosen": -0.27797046303749084, + "logps/rejected": -0.25145772099494934, + "loss": 0.524042546749115, + "loss/core": 0.5182952284812927, + "loss/preference_sft": 0.27797046303749084, + "loss/reference_anchor": 0.25957146286964417, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.2591378688812256, + "rewards/margins": 0.9982420802116394, + "rewards/rejected": 1.26089608669281, + "step": 665 + }, + { + "drift/chosen_abs": 0.13284215331077576, + "drift/rejected_abs": 0.09541095048189163, + "epoch": 0.64015287232772, + "grad_norm": 1.09375, + "learning_rate": 4.6901435682004996e-08, + "logits/chosen": -2.236567974090576, + "logits/rejected": -2.271984100341797, + "logps/chosen": -0.29236721992492676, + "logps/rejected": -0.2945176064968109, + "loss": 0.12218450009822845, + "loss/core": 0.12039539963006973, + "loss/preference_sft": 0.29236721992492676, + "loss/reference_anchor": 0.06021867319941521, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.3271570205688477, + "rewards/margins": 0.3746449947357178, + "rewards/rejected": 0.9525120854377747, + "step": 670 + }, + { + "drift/chosen_abs": 0.16276486217975616, + "drift/rejected_abs": 0.1297510266304016, + "epoch": 0.6449301325689717, + "grad_norm": 27.875, + "learning_rate": 4.502354101793314e-08, + "logits/chosen": -2.358452320098877, + "logits/rejected": -2.268037796020508, + "logps/chosen": -0.279884934425354, + "logps/rejected": -0.2925143837928772, + "loss": 0.35353174805641174, + "loss/core": 0.3494763970375061, + "loss/preference_sft": 0.279884934425354, + "loss/reference_anchor": 0.17477791011333466, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.6270040273666382, + "rewards/margins": 0.33251360058784485, + "rewards/rejected": 1.2944905757904053, + "step": 675 + }, + { + "drift/chosen_abs": 0.31575721502304077, + "drift/rejected_abs": 0.1288444995880127, + "epoch": 0.6497073928102234, + "grad_norm": 36.25, + "learning_rate": 4.3175722229287034e-08, + "logits/chosen": -2.114642381668091, + "logits/rejected": -2.1079599857330322, + "logps/chosen": -0.26992639899253845, + "logps/rejected": -0.27495431900024414, + "loss": 0.6401761174201965, + "loss/core": 0.6332933306694031, + "loss/preference_sft": 0.26992639899253845, + "loss/reference_anchor": 0.3171465992927551, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.1569862365722656, + "rewards/margins": 1.9471004009246826, + "rewards/rejected": 1.209885835647583, + "step": 680 + }, + { + "drift/chosen_abs": 0.23132996261119843, + "drift/rejected_abs": 0.23488974571228027, + "epoch": 0.654484653051475, + "grad_norm": 126.0, + "learning_rate": 4.135867420514276e-08, + "logits/chosen": -2.2158150672912598, + "logits/rejected": -2.220139741897583, + "logps/chosen": -0.29107457399368286, + "logps/rejected": -0.26147180795669556, + "loss": 0.934106707572937, + "loss/core": 0.9242784380912781, + "loss/preference_sft": 0.29107457399368286, + "loss/reference_anchor": 0.4623136520385742, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.3132710456848145, + "rewards/margins": -0.03536492586135864, + "rewards/rejected": 2.3486359119415283, + "step": 685 + }, + { + "drift/chosen_abs": 0.2907050848007202, + "drift/rejected_abs": 0.21539370715618134, + "epoch": 0.6592619132927267, + "grad_norm": 704.0, + "learning_rate": 3.957308026295035e-08, + "logits/chosen": -2.0876173973083496, + "logits/rejected": -2.159592390060425, + "logps/chosen": -0.3214958608150482, + "logps/rejected": -0.3213821053504944, + "loss": 1.7343063354492188, + "loss/core": 1.716496467590332, + "loss/preference_sft": 0.3214958608150482, + "loss/reference_anchor": 0.858340859413147, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.9058361053466797, + "rewards/margins": 0.7544411420822144, + "rewards/rejected": 2.151395082473755, + "step": 690 + }, + { + "drift/chosen_abs": 0.23619894683361053, + "drift/rejected_abs": 0.11816170066595078, + "epoch": 0.6640391735339782, + "grad_norm": 40.25, + "learning_rate": 3.7819611891566084e-08, + "logits/chosen": -1.9863860607147217, + "logits/rejected": -1.9883887767791748, + "logps/chosen": -0.26810505986213684, + "logps/rejected": -0.2822146713733673, + "loss": 0.27285757660865784, + "loss/core": 0.269618421792984, + "loss/preference_sft": 0.26810505986213684, + "loss/reference_anchor": 0.13514725863933563, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.361358880996704, + "rewards/margins": 1.190822720527649, + "rewards/rejected": 1.1705362796783447, + "step": 695 + }, + { + "drift/chosen_abs": 0.32739442586898804, + "drift/rejected_abs": 0.16697341203689575, + "epoch": 0.6688164337752299, + "grad_norm": 45.25, + "learning_rate": 3.609892849873286e-08, + "logits/chosen": -2.238140106201172, + "logits/rejected": -2.170215606689453, + "logps/chosen": -0.28792768716812134, + "logps/rejected": -0.26793545484542847, + "loss": 1.2909228801727295, + "loss/core": 1.2775663137435913, + "loss/preference_sft": 0.28792768716812134, + "loss/reference_anchor": 0.6390419006347656, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.2727417945861816, + "rewards/margins": 1.6051795482635498, + "rewards/rejected": 1.6675622463226318, + "step": 700 + }, + { + "drift/chosen_abs": 0.1326175481081009, + "drift/rejected_abs": 0.05240592360496521, + "epoch": 0.6735936940164815, + "grad_norm": 0.875, + "learning_rate": 3.4411677163103894e-08, + "logits/chosen": -2.1258556842803955, + "logits/rejected": -2.1329469680786133, + "logps/chosen": -0.3264055550098419, + "logps/rejected": -0.31279516220092773, + "loss": 0.1313130259513855, + "loss/core": 0.1293625682592392, + "loss/preference_sft": 0.3264055550098419, + "loss/reference_anchor": 0.06488193571567535, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.3231408596038818, + "rewards/margins": 0.7995508313179016, + "rewards/rejected": 0.5235900282859802, + "step": 705 + }, + { + "drift/chosen_abs": 0.24739420413970947, + "drift/rejected_abs": 0.08645138889551163, + "epoch": 0.6783709542577332, + "grad_norm": 39.0, + "learning_rate": 3.2758492390902945e-08, + "logits/chosen": -1.9009885787963867, + "logits/rejected": -1.9612804651260376, + "logps/chosen": -0.2778227627277374, + "logps/rejected": -0.28797098994255066, + "loss": 0.3639562726020813, + "loss/core": 0.359793096780777, + "loss/preference_sft": 0.2778227627277374, + "loss/reference_anchor": 0.18037569522857666, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.4713692665100098, + "rewards/margins": 1.64273202419281, + "rewards/rejected": 0.828637421131134, + "step": 710 + }, + { + "drift/chosen_abs": 0.23490512371063232, + "drift/rejected_abs": 0.0935138463973999, + "epoch": 0.6831482144989849, + "grad_norm": 10.5, + "learning_rate": 3.11399958773126e-08, + "logits/chosen": -2.0637762546539307, + "logits/rejected": -2.1122887134552, + "logps/chosen": -0.2840803563594818, + "logps/rejected": -0.2814682424068451, + "loss": 0.23643600940704346, + "loss/core": 0.23352782428264618, + "loss/preference_sft": 0.2840803563594818, + "loss/reference_anchor": 0.11700389534235, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.348367214202881, + "rewards/margins": 1.4155561923980713, + "rewards/rejected": 0.9328109622001648, + "step": 715 + }, + { + "drift/chosen_abs": 0.25520801544189453, + "drift/rejected_abs": 0.11677595227956772, + "epoch": 0.6879254747402365, + "grad_norm": 5.0625, + "learning_rate": 2.9556796272679972e-08, + "logits/chosen": -2.1001739501953125, + "logits/rejected": -2.1356372833251953, + "logps/chosen": -0.2884894907474518, + "logps/rejected": -0.31645768880844116, + "loss": 0.6001358032226562, + "loss/core": 0.593619167804718, + "loss/preference_sft": 0.2884894907474518, + "loss/reference_anchor": 0.29698267579078674, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.5518627166748047, + "rewards/margins": 1.5054526329040527, + "rewards/rejected": 1.0464102029800415, + "step": 720 + }, + { + "drift/chosen_abs": 0.11836670339107513, + "drift/rejected_abs": 0.11379599571228027, + "epoch": 0.6927027349814882, + "grad_norm": 2.453125, + "learning_rate": 2.8009488953628215e-08, + "logits/chosen": -2.0836520195007324, + "logits/rejected": -2.0259056091308594, + "logps/chosen": -0.28411412239074707, + "logps/rejected": -0.2758258581161499, + "loss": 0.23348887264728546, + "loss/core": 0.2306121587753296, + "loss/preference_sft": 0.28411412239074707, + "loss/reference_anchor": 0.11542298644781113, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.1802574396133423, + "rewards/margins": 0.044753752648830414, + "rewards/rejected": 1.1355037689208984, + "step": 725 + }, + { + "drift/chosen_abs": 0.280209481716156, + "drift/rejected_abs": 0.10185222327709198, + "epoch": 0.6974799952227397, + "grad_norm": 37.25, + "learning_rate": 2.649865579915976e-08, + "logits/chosen": -2.075974941253662, + "logits/rejected": -2.050661563873291, + "logps/chosen": -0.3032965660095215, + "logps/rejected": -0.31960076093673706, + "loss": 0.8421915769577026, + "loss/core": 0.8332451581954956, + "loss/preference_sft": 0.3032965660095215, + "loss/reference_anchor": 0.41699153184890747, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.8020949363708496, + "rewards/margins": 1.8411537408828735, + "rewards/rejected": 0.9609411954879761, + "step": 730 + }, + { + "drift/chosen_abs": 0.2798532247543335, + "drift/rejected_abs": 0.11052944511175156, + "epoch": 0.7022572554639914, + "grad_norm": 49.75, + "learning_rate": 2.5024864971835507e-08, + "logits/chosen": -2.1936328411102295, + "logits/rejected": -2.1866257190704346, + "logps/chosen": -0.266052782535553, + "logps/rejected": -0.27340036630630493, + "loss": 0.5987396240234375, + "loss/core": 0.5922771692276001, + "loss/preference_sft": 0.266052782535553, + "loss/reference_anchor": 0.29651200771331787, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.7965667247772217, + "rewards/margins": 1.6948391199111938, + "rewards/rejected": 1.1017276048660278, + "step": 735 + }, + { + "drift/chosen_abs": 0.21841701865196228, + "drift/rejected_abs": 0.16449300944805145, + "epoch": 0.707034515705243, + "grad_norm": 41.0, + "learning_rate": 2.3588670704111997e-08, + "logits/chosen": -1.9267356395721436, + "logits/rejected": -1.848046898841858, + "logps/chosen": -0.2917543947696686, + "logps/rejected": -0.3047102689743042, + "loss": 0.6575824022293091, + "loss/core": 0.6504920125007629, + "loss/preference_sft": 0.2917543947696686, + "loss/reference_anchor": 0.3253471255302429, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.181687831878662, + "rewards/margins": 0.5402703881263733, + "rewards/rejected": 1.6414177417755127, + "step": 740 + }, + { + "drift/chosen_abs": 0.2311057597398758, + "drift/rejected_abs": 0.12377341091632843, + "epoch": 0.7118117759464947, + "grad_norm": 2.3125, + "learning_rate": 2.219061308991721e-08, + "logits/chosen": -2.2026872634887695, + "logits/rejected": -2.2262043952941895, + "logps/chosen": -0.2942965626716614, + "logps/rejected": -0.2818772792816162, + "loss": 0.640469491481781, + "loss/core": 0.6335400342941284, + "loss/preference_sft": 0.2942965626716614, + "loss/reference_anchor": 0.3170434832572937, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.3100028038024902, + "rewards/margins": 1.0736254453659058, + "rewards/rejected": 1.2363771200180054, + "step": 745 + }, + { + "drift/chosen_abs": 0.2225804626941681, + "drift/rejected_abs": 0.14012092351913452, + "epoch": 0.7165890361877463, + "grad_norm": 147.0, + "learning_rate": 2.0831217881543557e-08, + "logits/chosen": -2.250375270843506, + "logits/rejected": -2.1884655952453613, + "logps/chosen": -0.27962955832481384, + "logps/rejected": -0.30086082220077515, + "loss": 0.2941736876964569, + "loss/core": 0.29070359468460083, + "loss/preference_sft": 0.27962955832481384, + "loss/reference_anchor": 0.14554303884506226, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.221367120742798, + "rewards/margins": 0.8238778114318848, + "rewards/rejected": 1.397489309310913, + "step": 750 + }, + { + "drift/chosen_abs": 0.35864901542663574, + "drift/rejected_abs": 0.1930883824825287, + "epoch": 0.721366296428998, + "grad_norm": 2.34375, + "learning_rate": 1.951099629193366e-08, + "logits/chosen": -2.1636476516723633, + "logits/rejected": -2.140977144241333, + "logps/chosen": -0.2771832346916199, + "logps/rejected": -0.28425145149230957, + "loss": 1.235163927078247, + "loss/core": 1.2223742008209229, + "loss/preference_sft": 0.2771832346916199, + "loss/reference_anchor": 0.6117671132087708, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.585937976837158, + "rewards/margins": 1.6583614349365234, + "rewards/rejected": 1.9275766611099243, + "step": 755 + }, + { + "drift/chosen_abs": 0.3538215160369873, + "drift/rejected_abs": 0.19870851933956146, + "epoch": 0.7261435566702497, + "grad_norm": 213.0, + "learning_rate": 1.823044480243431e-08, + "logits/chosen": -2.004826784133911, + "logits/rejected": -2.0009334087371826, + "logps/chosen": -0.30821096897125244, + "logps/rejected": -0.35763052105903625, + "loss": 1.0943807363510132, + "loss/core": 1.0829265117645264, + "loss/preference_sft": 0.30821096897125244, + "loss/reference_anchor": 0.5418857932090759, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.537405014038086, + "rewards/margins": 1.5527830123901367, + "rewards/rejected": 1.9846217632293701, + "step": 760 + }, + { + "drift/chosen_abs": 0.16350261867046356, + "drift/rejected_abs": 0.08384877443313599, + "epoch": 0.7309208169115012, + "grad_norm": 3.046875, + "learning_rate": 1.699004497608994e-08, + "logits/chosen": -2.3181560039520264, + "logits/rejected": -2.325047016143799, + "logps/chosen": -0.2780446410179138, + "logps/rejected": -0.26890766620635986, + "loss": 0.15251176059246063, + "loss/core": 0.15044714510440826, + "loss/preference_sft": 0.2780446410179138, + "loss/reference_anchor": 0.07542603462934494, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.6326892375946045, + "rewards/margins": 0.7961629629135132, + "rewards/rejected": 0.8365263938903809, + "step": 765 + }, + { + "drift/chosen_abs": 0.22837817668914795, + "drift/rejected_abs": 0.07515065371990204, + "epoch": 0.7356980771527529, + "grad_norm": 364.0, + "learning_rate": 1.5790263276546658e-08, + "logits/chosen": -2.214590549468994, + "logits/rejected": -2.287729263305664, + "logps/chosen": -0.2873011529445648, + "logps/rejected": -0.27970749139785767, + "loss": 0.502562940120697, + "loss/core": 0.4970094561576843, + "loss/preference_sft": 0.2873011529445648, + "loss/reference_anchor": 0.24894189834594727, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.281993865966797, + "rewards/margins": 1.531829833984375, + "rewards/rejected": 0.7501639127731323, + "step": 770 + }, + { + "drift/chosen_abs": 0.1840764731168747, + "drift/rejected_abs": 0.14043626189231873, + "epoch": 0.7404753373940045, + "grad_norm": 1.5859375, + "learning_rate": 1.4631550892634126e-08, + "logits/chosen": -2.0965075492858887, + "logits/rejected": -1.9921375513076782, + "logps/chosen": -0.2745542526245117, + "logps/rejected": -0.26717105507850647, + "loss": 0.3951452076435089, + "loss/core": 0.3906872868537903, + "loss/preference_sft": 0.2745542526245117, + "loss/reference_anchor": 0.19543880224227905, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.8390769958496094, + "rewards/margins": 0.4357059895992279, + "rewards/rejected": 1.4033708572387695, + "step": 775 + }, + { + "drift/chosen_abs": 0.2997458875179291, + "drift/rejected_abs": 0.19411607086658478, + "epoch": 0.7452525976352562, + "grad_norm": 464.0, + "learning_rate": 1.3514343568692132e-08, + "logits/chosen": -2.240328788757324, + "logits/rejected": -2.2070069313049316, + "logps/chosen": -0.2926557660102844, + "logps/rejected": -0.2933287024497986, + "loss": 1.8442484140396118, + "loss/core": 1.8254058361053467, + "loss/preference_sft": 0.2926557660102844, + "loss/reference_anchor": 0.9128621816635132, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.995630979537964, + "rewards/margins": 1.0562885999679565, + "rewards/rejected": 1.9393424987792969, + "step": 780 + }, + { + "drift/chosen_abs": 0.1899985373020172, + "drift/rejected_abs": 0.07741059362888336, + "epoch": 0.7500298578765078, + "grad_norm": 3.65625, + "learning_rate": 1.2439061440704724e-08, + "logits/chosen": -2.209710121154785, + "logits/rejected": -2.2208352088928223, + "logps/chosen": -0.28208765387535095, + "logps/rejected": -0.28293755650520325, + "loss": 0.6761240363121033, + "loss/core": 0.6688669323921204, + "loss/preference_sft": 0.28208765387535095, + "loss/reference_anchor": 0.3346477448940277, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.8997882604599, + "rewards/margins": 1.1262468099594116, + "rewards/rejected": 0.773541271686554, + "step": 785 + }, + { + "drift/chosen_abs": 0.3311009407043457, + "drift/rejected_abs": 0.1309339702129364, + "epoch": 0.7548071181177595, + "grad_norm": 173.0, + "learning_rate": 1.1406108878304468e-08, + "logits/chosen": -2.0325357913970947, + "logits/rejected": -2.026935577392578, + "logps/chosen": -0.27717894315719604, + "logps/rejected": -0.27080798149108887, + "loss": 0.6806409955024719, + "loss/core": 0.6733458638191223, + "loss/preference_sft": 0.27717894315719604, + "loss/reference_anchor": 0.33703652024269104, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.310328960418701, + "rewards/margins": 2.0351784229278564, + "rewards/rejected": 1.2751507759094238, + "step": 790 + }, + { + "drift/chosen_abs": 0.3741660416126251, + "drift/rejected_abs": 0.0975070595741272, + "epoch": 0.7595843783590112, + "grad_norm": 350.0, + "learning_rate": 1.0415874332705381e-08, + "logits/chosen": -2.150209903717041, + "logits/rejected": -2.0983715057373047, + "logps/chosen": -0.27431005239486694, + "logps/rejected": -0.2787257730960846, + "loss": 1.4984773397445679, + "loss/core": 1.4830867052078247, + "loss/preference_sft": 0.27431005239486694, + "loss/reference_anchor": 0.7421107888221741, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.738903760910034, + "rewards/margins": 2.7708230018615723, + "rewards/rejected": 0.9680808186531067, + "step": 795 + }, + { + "drift/chosen_abs": 0.2510735094547272, + "drift/rejected_abs": 0.1462600976228714, + "epoch": 0.7643616386002627, + "grad_norm": 18.625, + "learning_rate": 9.468730190622484e-09, + "logits/chosen": -2.4673285484313965, + "logits/rejected": -2.4519200325012207, + "logps/chosen": -0.29721131920814514, + "logps/rejected": -0.3075062036514282, + "loss": 1.2491406202316284, + "loss/core": 1.23618483543396, + "loss/preference_sft": 0.29721131920814514, + "loss/reference_anchor": 0.6180749535560608, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.509737730026245, + "rewards/margins": 1.0481786727905273, + "rewards/rejected": 1.4615590572357178, + "step": 800 + }, + { + "drift/chosen_abs": 0.09586870670318604, + "drift/rejected_abs": 0.0631202682852745, + "epoch": 0.7691388988415144, + "grad_norm": 5.0, + "learning_rate": 8.565032634232194e-09, + "logits/chosen": -2.117633104324341, + "logits/rejected": -2.0666775703430176, + "logps/chosen": -0.30181869864463806, + "logps/rejected": -0.3010576367378235, + "loss": 0.04290010407567024, + "loss/core": 0.04187611863017082, + "loss/preference_sft": 0.30181869864463806, + "loss/reference_anchor": 0.02101728692650795, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 0.9572784304618835, + "rewards/margins": 0.32730334997177124, + "rewards/rejected": 0.6299751400947571, + "step": 805 + }, + { + "drift/chosen_abs": 0.18182532489299774, + "drift/rejected_abs": 0.16493472456932068, + "epoch": 0.773916159082766, + "grad_norm": 90.5, + "learning_rate": 7.70512150722702e-09, + "logits/chosen": -2.0990633964538574, + "logits/rejected": -2.0376296043395996, + "logps/chosen": -0.26729169487953186, + "logps/rejected": -0.26321226358413696, + "loss": 0.49664896726608276, + "loss/core": 0.491199254989624, + "loss/preference_sft": 0.26729169487953186, + "loss/reference_anchor": 0.24575798213481903, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.8167064189910889, + "rewards/margins": 0.16888391971588135, + "rewards/rejected": 1.647822380065918, + "step": 810 + }, + { + "drift/chosen_abs": 0.24560102820396423, + "drift/rejected_abs": 0.10801911354064941, + "epoch": 0.7786934193240177, + "grad_norm": 1.7109375, + "learning_rate": 6.8893201870139915e-09, + "logits/chosen": -2.091200113296509, + "logits/rejected": -2.27951979637146, + "logps/chosen": -0.31718164682388306, + "logps/rejected": -0.3068375885486603, + "loss": 0.5775214433670044, + "loss/core": 0.5711715817451477, + "loss/preference_sft": 0.31718164682388306, + "loss/reference_anchor": 0.2857758402824402, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.4512245655059814, + "rewards/margins": 1.3882179260253906, + "rewards/rejected": 1.0630064010620117, + "step": 815 + }, + { + "drift/chosen_abs": 0.26311054825782776, + "drift/rejected_abs": 0.20147815346717834, + "epoch": 0.7834706795652693, + "grad_norm": 30.375, + "learning_rate": 6.117935463105808e-09, + "logits/chosen": -1.8164457082748413, + "logits/rejected": -1.7971694469451904, + "logps/chosen": -0.279045045375824, + "logps/rejected": -0.33144164085388184, + "loss": 0.9571006894111633, + "loss/core": 0.947068989276886, + "loss/preference_sft": 0.279045045375824, + "loss/reference_anchor": 0.4736693501472473, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.6302883625030518, + "rewards/margins": 0.6288776397705078, + "rewards/rejected": 2.001410961151123, + "step": 820 + }, + { + "drift/chosen_abs": 0.3537200093269348, + "drift/rejected_abs": 0.16787850856781006, + "epoch": 0.788247939806521, + "grad_norm": 14.1875, + "learning_rate": 5.391257421749826e-09, + "logits/chosen": -2.1316933631896973, + "logits/rejected": -2.1172828674316406, + "logps/chosen": -0.29390478134155273, + "logps/rejected": -0.2766962945461273, + "loss": 1.5460574626922607, + "loss/core": 1.530153512954712, + "loss/preference_sft": 0.29390478134155273, + "loss/reference_anchor": 0.7657959461212158, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.5372002124786377, + "rewards/margins": 1.859610915184021, + "rewards/rejected": 1.6775890588760376, + "step": 825 + }, + { + "drift/chosen_abs": 0.27988651394844055, + "drift/rejected_abs": 0.12321482598781586, + "epoch": 0.7930252000477725, + "grad_norm": 612.0, + "learning_rate": 4.709559336838462e-09, + "logits/chosen": -1.8963066339492798, + "logits/rejected": -1.9476807117462158, + "logps/chosen": -0.30415278673171997, + "logps/rejected": -0.3033929169178009, + "loss": 0.9329975247383118, + "loss/core": 0.9231513142585754, + "loss/preference_sft": 0.30415278673171997, + "loss/reference_anchor": 0.4618992209434509, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.798447370529175, + "rewards/margins": 1.5701082944869995, + "rewards/rejected": 1.2283390760421753, + "step": 830 + }, + { + "drift/chosen_abs": 0.16968384385108948, + "drift/rejected_abs": 0.1050095334649086, + "epoch": 0.7978024602890242, + "grad_norm": 1.7578125, + "learning_rate": 4.073097567142025e-09, + "logits/chosen": -2.2337470054626465, + "logits/rejected": -2.3164491653442383, + "logps/chosen": -0.2991805672645569, + "logps/rejected": -0.28817689418792725, + "loss": 0.22842451930046082, + "loss/core": 0.2255687713623047, + "loss/preference_sft": 0.2991805672645569, + "loss/reference_anchor": 0.11287200450897217, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.696568250656128, + "rewards/margins": 0.6477723121643066, + "rewards/rejected": 1.0487959384918213, + "step": 835 + }, + { + "drift/chosen_abs": 0.2029610425233841, + "drift/rejected_abs": 0.15892741084098816, + "epoch": 0.8025797205302759, + "grad_norm": 9.4375, + "learning_rate": 3.482111459902695e-09, + "logits/chosen": -2.029393196105957, + "logits/rejected": -2.1248087882995605, + "logps/chosen": -0.28962570428848267, + "logps/rejected": -0.3038334250450134, + "loss": 0.6708376407623291, + "loss/core": 0.6636188626289368, + "loss/preference_sft": 0.28962570428848267, + "loss/reference_anchor": 0.3319765329360962, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.0296101570129395, + "rewards/margins": 0.4415357708930969, + "rewards/rejected": 1.5880745649337769, + "step": 840 + }, + { + "drift/chosen_abs": 0.4587179124355316, + "drift/rejected_abs": 0.22257618606090546, + "epoch": 0.8073569807715275, + "grad_norm": 22.875, + "learning_rate": 2.9368232608258797e-09, + "logits/chosen": -2.164417028427124, + "logits/rejected": -2.176283121109009, + "logps/chosen": -0.27492496371269226, + "logps/rejected": -0.27344006299972534, + "loss": 2.1201024055480957, + "loss/core": 2.098555564880371, + "loss/preference_sft": 0.27492496371269226, + "loss/reference_anchor": 1.0498454570770264, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.5870513916015625, + "rewards/margins": 2.364668846130371, + "rewards/rejected": 2.222382068634033, + "step": 845 + }, + { + "drift/chosen_abs": 0.17442527413368225, + "drift/rejected_abs": 0.06823588907718658, + "epoch": 0.8121342410127792, + "grad_norm": 0.87109375, + "learning_rate": 2.4374380305025866e-09, + "logits/chosen": -2.2421109676361084, + "logits/rejected": -2.2991902828216553, + "logps/chosen": -0.25456729531288147, + "logps/rejected": -0.25157222151756287, + "loss": 0.12556889653205872, + "loss/core": 0.12381688505411148, + "loss/preference_sft": 0.25456729531288147, + "loss/reference_anchor": 0.06214389204978943, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7427303791046143, + "rewards/margins": 1.0707417726516724, + "rewards/rejected": 0.6719885468482971, + "step": 850 + }, + { + "drift/chosen_abs": 0.2316814661026001, + "drift/rejected_abs": 0.18873625993728638, + "epoch": 0.8169115012540308, + "grad_norm": 49.25, + "learning_rate": 1.984143567294594e-09, + "logits/chosen": -2.0805776119232178, + "logits/rejected": -2.073974609375, + "logps/chosen": -0.2674004137516022, + "logps/rejected": -0.29779064655303955, + "loss": 0.6567851305007935, + "loss/core": 0.6497505307197571, + "loss/preference_sft": 0.2674004137516022, + "loss/reference_anchor": 0.3249875009059906, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.3135294914245605, + "rewards/margins": 0.4317246377468109, + "rewards/rejected": 1.8818048238754272, + "step": 855 + }, + { + "drift/chosen_abs": 0.19219297170639038, + "drift/rejected_abs": 0.1073886901140213, + "epoch": 0.8216887614952825, + "grad_norm": 1.2421875, + "learning_rate": 1.577110336711096e-09, + "logits/chosen": -2.1786136627197266, + "logits/rejected": -2.251077175140381, + "logps/chosen": -0.29990866780281067, + "logps/rejected": -0.30773764848709106, + "loss": 0.4442000985145569, + "loss/core": 0.4392046332359314, + "loss/preference_sft": 0.29990866780281067, + "loss/reference_anchor": 0.21978160738945007, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.9197685718536377, + "rewards/margins": 0.8469489812850952, + "rewards/rejected": 1.072819471359253, + "step": 860 + }, + { + "drift/chosen_abs": 0.26136937737464905, + "drift/rejected_abs": 0.1216052770614624, + "epoch": 0.826466021736534, + "grad_norm": 3.8125, + "learning_rate": 1.2164914073037048e-09, + "logits/chosen": -2.1374754905700684, + "logits/rejected": -2.2176711559295654, + "logps/chosen": -0.29837021231651306, + "logps/rejected": -0.282470166683197, + "loss": 1.20570707321167, + "loss/core": 1.1931687593460083, + "loss/preference_sft": 0.29837021231651306, + "loss/reference_anchor": 0.5970777273178101, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.6126089096069336, + "rewards/margins": 1.4006534814834595, + "rewards/rejected": 1.2119553089141846, + "step": 865 + }, + { + "drift/chosen_abs": 0.4436754286289215, + "drift/rejected_abs": 0.2094762623310089, + "epoch": 0.8312432819777857, + "grad_norm": 46.75, + "learning_rate": 9.024223931035357e-10, + "logits/chosen": -1.9862560033798218, + "logits/rejected": -1.9510629177093506, + "logps/chosen": -0.29855862259864807, + "logps/rejected": -0.28159067034721375, + "loss": 1.7128210067749023, + "loss/core": 1.6952579021453857, + "loss/preference_sft": 0.29855862259864807, + "loss/reference_anchor": 0.8482988476753235, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.433295726776123, + "rewards/margins": 2.3404412269592285, + "rewards/rejected": 2.0928549766540527, + "step": 870 + }, + { + "drift/chosen_abs": 0.19628901779651642, + "drift/rejected_abs": 0.1381283849477768, + "epoch": 0.8360205422190374, + "grad_norm": 0.75390625, + "learning_rate": 6.350214026223516e-10, + "logits/chosen": -2.086601972579956, + "logits/rejected": -2.0887794494628906, + "logps/chosen": -0.3021940588951111, + "logps/rejected": -0.30490565299987793, + "loss": 0.4931362569332123, + "loss/core": 0.48765283823013306, + "loss/preference_sft": 0.3021940588951111, + "loss/reference_anchor": 0.24395246803760529, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9610573053359985, + "rewards/margins": 0.5812469720840454, + "rewards/rejected": 1.379810094833374, + "step": 875 + }, + { + "drift/chosen_abs": 0.37091922760009766, + "drift/rejected_abs": 0.14499905705451965, + "epoch": 0.840797802460289, + "grad_norm": 0.443359375, + "learning_rate": 4.143889944367429e-10, + "logits/chosen": -2.001990795135498, + "logits/rejected": -1.9710228443145752, + "logps/chosen": -0.28516069054603577, + "logps/rejected": -0.30990806221961975, + "loss": 1.4451124668121338, + "loss/core": 1.4302270412445068, + "loss/preference_sft": 0.28516069054603577, + "loss/reference_anchor": 0.7157606482505798, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.7089836597442627, + "rewards/margins": 2.2644705772399902, + "rewards/rejected": 1.444512963294983, + "step": 880 + }, + { + "drift/chosen_abs": 0.15534690022468567, + "drift/rejected_abs": 0.07884477078914642, + "epoch": 0.8455750627015407, + "grad_norm": 14.1875, + "learning_rate": 2.406081393722531e-10, + "logits/chosen": -2.0966742038726807, + "logits/rejected": -2.1267499923706055, + "logps/chosen": -0.2981413006782532, + "logps/rejected": -0.30681678652763367, + "loss": 0.11275926977396011, + "loss/core": 0.11104929447174072, + "loss/preference_sft": 0.2981413006782532, + "loss/reference_anchor": 0.05568493530154228, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.5524650812149048, + "rewards/margins": 0.7652949094772339, + "rewards/rejected": 0.7871701121330261, + "step": 885 + }, + { + "drift/chosen_abs": 0.18543264269828796, + "drift/rejected_abs": 0.16110344231128693, + "epoch": 0.8503523229427923, + "grad_norm": 14.9375, + "learning_rate": 1.1374418930135133e-10, + "logits/chosen": -2.1319258213043213, + "logits/rejected": -2.1853280067443848, + "logps/chosen": -0.26988840103149414, + "logps/rejected": -0.2897631525993347, + "loss": 0.5226936340332031, + "loss/core": 0.5169836282730103, + "loss/preference_sft": 0.26988840103149414, + "loss/reference_anchor": 0.25851255655288696, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.8515974283218384, + "rewards/margins": 0.24080824851989746, + "rewards/rejected": 1.6107892990112305, + "step": 890 + }, + { + "drift/chosen_abs": 0.21447238326072693, + "drift/rejected_abs": 0.09845264256000519, + "epoch": 0.855129583184044, + "grad_norm": 176.0, + "learning_rate": 3.3844852567285756e-11, + "logits/chosen": -2.092803478240967, + "logits/rejected": -2.0520870685577393, + "logps/chosen": -0.289115846157074, + "logps/rejected": -0.2914557456970215, + "loss": 0.28112271428108215, + "loss/core": 0.2777610719203949, + "loss/preference_sft": 0.289115846157074, + "loss/reference_anchor": 0.1391705721616745, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.144585132598877, + "rewards/margins": 1.1624013185501099, + "rewards/rejected": 0.9821839332580566, + "step": 895 + }, + { + "drift/chosen_abs": 0.29047515988349915, + "drift/rejected_abs": 0.14344081282615662, + "epoch": 0.8599068434252956, + "grad_norm": 89.0, + "learning_rate": 9.401760429905703e-13, + "logits/chosen": -2.0498125553131104, + "logits/rejected": -2.014761447906494, + "logps/chosen": -0.26901915669441223, + "logps/rejected": -0.2722063958644867, + "loss": 0.7836966514587402, + "loss/core": 0.7753987312316895, + "loss/preference_sft": 0.26901915669441223, + "loss/reference_anchor": 0.3879950940608978, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.9047515392303467, + "rewards/margins": 1.4703437089920044, + "rewards/rejected": 1.4344079494476318, + "step": 900 + }, + { + "epoch": 0.8599068434252956, + "step": 900, + "total_flos": 0.0, + "train_loss": 0.688900715245141, + "train_runtime": 7064.5608, + "train_samples_per_second": 2.038, + "train_steps_per_second": 0.127 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..e6fa22a --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09132e3eb4c33f3135f06a68a2d822cc9c604599f0e7703c7037fe12e529beaf +size 7057