commit 8a923bb2f23232fa423f1863b0c466446f786fa5 Author: ModelHub XC Date: Thu Jul 23 13:26:10 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: promotion/qwen3-8b-aaai27-flagship-inpo-avg-s43 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..94c3a67 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +--- +library_name: transformers +base_model: Qwen/Qwen3-8B +tags: +- ronpo +- mnpo +- preference-optimization +--- + +# qwen3-8b-aaai27-flagship-inpo-avg-s43 + +Research checkpoint for the RONPO AAAI revision experiments. + +- Method: inpo_avg +- Base model: `Qwen/Qwen3-8B` +- Seed: 43 +- Local source at upload time: `/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/full/inpo_avg/seed43/attempt2` +- Uploaded at UTC: 2026-07-13T13:52:08Z +- Run status metadata: `{"attempt": 2, "effective_batch_size": 16, "method": "inpo_avg", "objective_protocol": "objective_protocol.json", "optimizer_steps": 900, "resource_profile": "resource_profile.json", "scope_amendment": "scope_amendment_20260713.json", "seed": 43, "split_manifest": "split_manifest.json", "stability_gate": "stability_gate.json", "status": "completed", "wandb_run_id": "051016e9c42d", "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/051016e9c42d"}` + +AAAI-27 P1 matched budget: 900 optimizer steps, effective batch 16; passed non-thinking and collapse stability gates; sealed-test metrics are not used for checkpoint selection. + +Intended use: reproducibility and evaluation for the RONPO paper. This +checkpoint is not intended as a production assistant. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..ea4a1e6 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 4429.094605034722, + "train_runtime": 7055.131, + "train_samples": 16746, + "train_samples_per_second": 2.041, + "train_steps_per_second": 0.128 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..6bc768f --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +model_name_or_path: Qwen/Qwen3-8B +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/avg: 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: inpo +eta: 0.0075 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.02 +preference_sft_weight: 0.002 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 2.5e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 43 +gradient_accumulation_steps: 16 +gradient_checkpointing: true +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 1 +per_device_eval_batch_size: 1 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/full/inpo_avg/seed43/attempt2 +run_name: aaai27-flagship-full-inpo_avg-s43-a2 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/job_status.json b/job_status.json new file mode 100644 index 0000000..373cd41 --- /dev/null +++ b/job_status.json @@ -0,0 +1,23 @@ +{ + "status": "completed", + "stage": "full", + "method": "inpo_avg", + "seed": 43, + "attempt": 2, + "gpu": 0, + "gpus": [ + 0 + ], + "returncode": 0, + "finite_metrics": true, + "oom": false, + "traceback": false, + "reason": "passed", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "051016e9c42d", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/051016e9c42d", + "output_dir": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/full/inpo_avg/seed43/attempt2", + "log": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/logs/full_inpo_avg_s43_a2.log", + "completed_at": "2026-07-13T13:46:47Z" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..4cd59e2 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72897530323171bef02c6d06855476891239effab8260dace67100754fbe0f71 +size 16381517208 diff --git a/objective_protocol.json b/objective_protocol.json new file mode 100644 index 0000000..35b3b44 --- /dev/null +++ b/objective_protocol.json @@ -0,0 +1,69 @@ +{ + "schema_version": 1, + "frozen_at_kst": "2026-07-12T21:40:00+09:00", + "base_model": "Qwen/Qwen3-8B", + "primary_triple": { + "helpfulness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "ultrafeedback-helpfulness", + "transform": "identity" + }, + "safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "conciseness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "helpsteer-verbosity", + "transform": "negate" + } + }, + "gate_thresholds": { + "raw_score_std_strictly_greater_than": 0.01, + "aggregate_prompt_pair_spearman_median_strictly_less_than": 0.0, + "aggregate_pairwise_top1_mismatch_at_least": 0.5 + }, + "gate_pool": "source-train Qwen3-8B response pool split into deterministic train/validation; sealed source-test excluded", + "training_seeds": [42, 43, 44], + "optimizer_steps": 900, + "effective_batch_size": 16, + "decode": { + "enable_thinking": false, + "temperature": 0.7, + "top_p": 0.9, + "max_new_tokens": 2048, + "decode_seed": 42 + }, + "methods": [ + "ronpo_full_expect", + "ronpo_k_only", + "dpo", + "ipo", + "simpo", + "kto", + "sppo_avg", + "inpo_avg", + "ht_mnpo_helpfulness", + "ht_mnpo_safety", + "ht_mnpo_conciseness" + ], + "selection_policy": "No sealed-test selection. One preregistered configuration per method; failures use only the declared stability retry recipe, at most three attempts.", + "stability_gate": { + "validation_prompts": 128, + "min_candidate_base_mean_word_ratio": 0.33, + "max_candidate_base_mean_word_ratio": 2.0, + "max_consecutive_identical_word_run": 20, + "think_tag_count": 0, + "empty_response_count": 0, + "retry_recipe": "attempt 2 halves learning rate, enables gradient checkpointing, and adds reference=0.02/SFT=0.002 anchoring to unanchored unified losses; attempt 3 quarters learning rate and uses reference=0.05/SFT=0.005; RONPO uses reference=0.05/SFT=0.005 from attempt 1" + }, + "primary_metric": "mean prompt-level minimum of per-prompt min-max-normalized helpfulness, safety, conciseness", + "bootstrap": { + "unit": "prompt", + "paired": true, + "resamples": 2000, + "interval": "percentile_95", + "seed": 42 + } +} diff --git a/provenance.json b/provenance.json new file mode 100644 index 0000000..87e562e --- /dev/null +++ b/provenance.json @@ -0,0 +1,14 @@ +{ + "method": "inpo_avg", + "seed": 43, + "attempt": 2, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "051016e9c42d", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/051016e9c42d", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json", + "scope_amendment": "scope_amendment_20260713.json" +} diff --git a/resource_profile.json b/resource_profile.json new file mode 100644 index 0000000..18eec47 --- /dev/null +++ b/resource_profile.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "measured_at_kst": "2026-07-12T22:38:00+09:00", + "hardware": "NVIDIA B200 183359 MiB reported total; 178.35 GiB PyTorch-visible capacity", + "smoke_evidence": { + "attempted_profile": {"per_device_batch_size": 2, "gradient_accumulation_steps": 8, "gradient_checkpointing": false, "effective_batch_size": 16}, + "outcome": "OOM before completing the 20-step smoke across every launched unified baseline; representative process used 176.87 GiB and failed a 1.62 GiB allocation. KTO also OOMed at the optimizer step.", + "paper_metric": false + }, + "selected_full_training_profile": { + "per_device_batch_size": 2, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": false, + "attention_implementation": "PyTorch SDPA with cuDNN SDPA disabled", + "effective_batch_size": 16, + "applies_symmetrically_to_all_unified_pairwise_methods": true, + "kto_implementation_constraint": "TRL KTO requires actual batch size > 1. KTO uses batch 2 / accumulation 8 and identical AdamW with foreach=False. PyTorch SDPA is used with cuDNN SDPA disabled. The fixed base reference model is placed on a second B200 and its exact logits are returned to the policy device; the policy/optimizer remain on the first B200. This avoids a measured ~6-hour reference-precompute path while leaving the KTO loss, actual/effective batch, optimizer, and 900-step budget unchanged." + }, + "retry_semantics": "Resource selection is separated from method-stability attempts. Full attempt 1 uses the smoke-validated SDPA batch-2 profile and retains the preregistered method LR/anchor; attempts 2 and 3 fall back to batch 1 / accumulation 16 / gradient checkpointing and apply the declared LR/anchor stabilization recipe.", + "provenance_note": "No reward or sealed-evaluation result was available when this hardware profile was selected." +} diff --git a/run_status.json b/run_status.json new file mode 100644 index 0000000..2447e69 --- /dev/null +++ b/run_status.json @@ -0,0 +1,15 @@ +{ + "method": "inpo_avg", + "seed": 43, + "attempt": 2, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "051016e9c42d", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/051016e9c42d", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json", + "scope_amendment": "scope_amendment_20260713.json", + "status": "completed" +} diff --git a/split_manifest.json b/split_manifest.json new file mode 100644 index 0000000..4fb96bd --- /dev/null +++ b/split_manifest.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "split_rule": "sort source-train prompts by sha256('flagship-v1||'+prompt_id); first round(10%) validation", + "validation_fraction": 0.1, + "source_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/avg_oracle", + "base_model": "Qwen/Qwen3-8B", + "counts": { + "train": 16780, + "validation": 1864, + "sealed_test": 604 + }, + "prompt_id_sha256": { + "train": "fcf6f2ea5a0b7bcd41179e6f5288f4d850b2d04bdb729da91907bd18bf733a69", + "validation": "a87ba6f519192164daa487b109a6547dd8cbde46a27a373d6da201ca2d061b8f", + "sealed_test": "b398ced6af6cf73d731e1a5f32a0631d8060323d51eda1e375695566bf5f1867" + }, + "overlaps": { + "train_validation": 0, + "train_sealed_test": 0, + "validation_sealed_test": 0 + }, + "sealed_test_policy": "prompts only until S4a; no objective scores or model generations used in S0-S3" +} diff --git a/stability_gate.json b/stability_gate.json new file mode 100644 index 0000000..ab21b9b --- /dev/null +++ b/stability_gate.json @@ -0,0 +1,38 @@ +{ + "status": "passed", + "passed": true, + "base": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 388.1484375, + "max_words": 1379, + "max_repeat_run": 4 + }, + "candidate": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 393.96875, + "max_words": 1276, + "max_repeat_run": 3 + }, + "candidate_base_mean_word_ratio": 1.014995068735785, + "thresholds": { + "min_length_ratio": 0.33, + "max_length_ratio": 2.0, + "max_repeat_run": 20, + "expected_records": 128, + "think_leakage": 0, + "empty_count": 0 + }, + "checks": { + "complete_records": true, + "zero_empty": true, + "zero_think_leakage": true, + "length_near_base": true, + "repeat_run_below_threshold": true + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ea3c9e2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..ea4a1e6 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 4429.094605034722, + "train_runtime": 7055.131, + "train_samples": 16746, + "train_samples_per_second": 2.041, + "train_steps_per_second": 0.128 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..2977a1e --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,3643 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8599068434252956, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "drift/chosen_abs": 0.22064828872680664, + "drift/rejected_abs": 0.06916916370391846, + "epoch": 0.004777260241251642, + "grad_norm": 756.0, + "learning_rate": 1.111111111111111e-08, + "logits/chosen": -2.3183224201202393, + "logits/rejected": -2.3036041259765625, + "logps/chosen": -0.3117493987083435, + "logps/rejected": -0.2904360890388489, + "loss": 4424.53369140625, + "loss/core": 4424.529296875, + "loss/preference_sft": 0.3117493987083435, + "loss/reference_anchor": 0.1971215307712555, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.2059829235076904, + "rewards/margins": 1.5144869089126587, + "rewards/rejected": 0.6914961338043213, + "step": 5 + }, + { + "drift/chosen_abs": 0.30023816227912903, + "drift/rejected_abs": 0.26469460129737854, + "epoch": 0.009554520482503284, + "grad_norm": 2288.0, + "learning_rate": 2.5e-08, + "logits/chosen": -2.06440806388855, + "logits/rejected": -2.044948101043701, + "logps/chosen": -0.2865406572818756, + "logps/rejected": -0.3020057678222656, + "loss": 4440.205078125, + "loss/core": 4440.19384765625, + "loss/preference_sft": 0.2865406572818756, + "loss/reference_anchor": 0.5595167279243469, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.0006680488586426, + "rewards/margins": 0.35411542654037476, + "rewards/rejected": 2.646552324295044, + "step": 10 + }, + { + "drift/chosen_abs": 0.17687278985977173, + "drift/rejected_abs": 0.10483698546886444, + "epoch": 0.014331780723754926, + "grad_norm": 1672.0, + "learning_rate": 3.888888888888889e-08, + "logits/chosen": -2.229069709777832, + "logits/rejected": -2.2289915084838867, + "logps/chosen": -0.2580825984477997, + "logps/rejected": -0.27230164408683777, + "loss": 4434.91748046875, + "loss/core": 4434.9140625, + "loss/preference_sft": 0.2580825984477997, + "loss/reference_anchor": 0.13444273173809052, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.7682386636734009, + "rewards/margins": 0.7218483090400696, + "rewards/rejected": 1.046390414237976, + "step": 15 + }, + { + "drift/chosen_abs": 0.3651048541069031, + "drift/rejected_abs": 0.11754413694143295, + "epoch": 0.01910904096500657, + "grad_norm": 680.0, + "learning_rate": 5.2777777777777776e-08, + "logits/chosen": -2.1665427684783936, + "logits/rejected": -2.18465518951416, + "logps/chosen": -0.272116094827652, + "logps/rejected": -0.27083370089530945, + "loss": 4412.21240234375, + "loss/core": 4412.20166015625, + "loss/preference_sft": 0.272116094827652, + "loss/reference_anchor": 0.5144023895263672, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.6496434211730957, + "rewards/margins": 2.475973606109619, + "rewards/rejected": 1.1736698150634766, + "step": 20 + }, + { + "drift/chosen_abs": 0.2144162654876709, + "drift/rejected_abs": 0.09806983172893524, + "epoch": 0.02388630120625821, + "grad_norm": 960.0, + "learning_rate": 6.666666666666667e-08, + "logits/chosen": -2.008526563644409, + "logits/rejected": -2.124323844909668, + "logps/chosen": -0.27961307764053345, + "logps/rejected": -0.27083954215049744, + "loss": 4428.9775390625, + "loss/core": 4428.97509765625, + "loss/preference_sft": 0.27961307764053345, + "loss/reference_anchor": 0.08990418910980225, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 2.143292188644409, + "rewards/margins": 1.1632330417633057, + "rewards/rejected": 0.9800591468811035, + "step": 25 + }, + { + "drift/chosen_abs": 0.2801198363304138, + "drift/rejected_abs": 0.1081269234418869, + "epoch": 0.028663561447509853, + "grad_norm": 688.0, + "learning_rate": 8.055555555555555e-08, + "logits/chosen": -2.133108615875244, + "logits/rejected": -2.1352686882019043, + "logps/chosen": -0.2811334729194641, + "logps/rejected": -0.2618844211101532, + "loss": 4421.5927734375, + "loss/core": 4421.58642578125, + "loss/preference_sft": 0.2811334729194641, + "loss/reference_anchor": 0.2856946587562561, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.8003880977630615, + "rewards/margins": 1.7426341772079468, + "rewards/rejected": 1.0577540397644043, + "step": 30 + }, + { + "drift/chosen_abs": 0.252098023891449, + "drift/rejected_abs": 0.07856006920337677, + "epoch": 0.033440821688761495, + "grad_norm": 442.0, + "learning_rate": 9.444444444444444e-08, + "logits/chosen": -2.2716076374053955, + "logits/rejected": -2.2810473442077637, + "logps/chosen": -0.2923678755760193, + "logps/rejected": -0.2761971056461334, + "loss": 4421.9375, + "loss/core": 4421.9287109375, + "loss/preference_sft": 0.2923678755760193, + "loss/reference_anchor": 0.4119436740875244, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.5209803581237793, + "rewards/margins": 1.7362782955169678, + "rewards/rejected": 0.784701943397522, + "step": 35 + }, + { + "drift/chosen_abs": 0.2930517792701721, + "drift/rejected_abs": 0.09838008880615234, + "epoch": 0.03821808193001314, + "grad_norm": 1432.0, + "learning_rate": 1.0833333333333334e-07, + "logits/chosen": -2.2542996406555176, + "logits/rejected": -2.35125994682312, + "logps/chosen": -0.2655903697013855, + "logps/rejected": -0.2916432023048401, + "loss": 4415.17236328125, + "loss/core": 4415.16552734375, + "loss/preference_sft": 0.2655903697013855, + "loss/reference_anchor": 0.33125585317611694, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.9291985034942627, + "rewards/margins": 2.2448573112487793, + "rewards/rejected": 0.6843410730361938, + "step": 40 + }, + { + "drift/chosen_abs": 0.30555135011672974, + "drift/rejected_abs": 0.1278877556324005, + "epoch": 0.04299534217126478, + "grad_norm": 872.0, + "learning_rate": 1.2222222222222222e-07, + "logits/chosen": -2.0905721187591553, + "logits/rejected": -2.145535945892334, + "logps/chosen": -0.2696756422519684, + "logps/rejected": -0.27680477499961853, + "loss": 4421.12060546875, + "loss/core": 4421.1123046875, + "loss/preference_sft": 0.2696756422519684, + "loss/reference_anchor": 0.4080870747566223, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 3.0551581382751465, + "rewards/margins": 1.7777941226959229, + "rewards/rejected": 1.277363896369934, + "step": 45 + }, + { + "drift/chosen_abs": 0.1658797562122345, + "drift/rejected_abs": 0.12210337072610855, + "epoch": 0.04777260241251642, + "grad_norm": 2928.0, + "learning_rate": 1.3611111111111108e-07, + "logits/chosen": -2.1889004707336426, + "logits/rejected": -2.2628321647644043, + "logps/chosen": -0.2878100872039795, + "logps/rejected": -0.2691812217235565, + "loss": 4438.6357421875, + "loss/core": 4438.63232421875, + "loss/preference_sft": 0.2878100872039795, + "loss/reference_anchor": 0.10800884664058685, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6577980518341064, + "rewards/margins": 0.4397946894168854, + "rewards/rejected": 1.218003273010254, + "step": 50 + }, + { + "drift/chosen_abs": 0.23436591029167175, + "drift/rejected_abs": 0.07058925181627274, + "epoch": 0.05254986265376806, + "grad_norm": 3184.0, + "learning_rate": 1.5e-07, + "logits/chosen": -2.076425552368164, + "logits/rejected": -2.139699697494507, + "logps/chosen": -0.28844743967056274, + "logps/rejected": -0.28838062286376953, + "loss": 4422.97802734375, + "loss/core": 4422.97216796875, + "loss/preference_sft": 0.28844743967056274, + "loss/reference_anchor": 0.25497233867645264, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.3433704376220703, + "rewards/margins": 1.6407816410064697, + "rewards/rejected": 0.7025889158248901, + "step": 55 + }, + { + "drift/chosen_abs": 0.3793022632598877, + "drift/rejected_abs": 0.1703735888004303, + "epoch": 0.057327122895019705, + "grad_norm": 2256.0, + "learning_rate": 1.6388888888888888e-07, + "logits/chosen": -2.1773691177368164, + "logits/rejected": -2.124330520629883, + "logps/chosen": -0.2736962139606476, + "logps/rejected": -0.2769095301628113, + "loss": 4417.6767578125, + "loss/core": 4417.66015625, + "loss/preference_sft": 0.2736962139606476, + "loss/reference_anchor": 0.7732974886894226, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.7925236225128174, + "rewards/margins": 2.0935792922973633, + "rewards/rejected": 1.698943853378296, + "step": 60 + }, + { + "drift/chosen_abs": 0.32665160298347473, + "drift/rejected_abs": 0.11409376561641693, + "epoch": 0.06210438313627135, + "grad_norm": 3920.0, + "learning_rate": 1.7777777777777776e-07, + "logits/chosen": -2.108382225036621, + "logits/rejected": -2.1682333946228027, + "logps/chosen": -0.29594534635543823, + "logps/rejected": -0.2858942151069641, + "loss": 4416.6337890625, + "loss/core": 4416.6220703125, + "loss/preference_sft": 0.29594534635543823, + "loss/reference_anchor": 0.552937388420105, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.2631747722625732, + "rewards/margins": 2.1304943561553955, + "rewards/rejected": 1.1326806545257568, + "step": 65 + }, + { + "drift/chosen_abs": 0.2492091953754425, + "drift/rejected_abs": 0.1481395810842514, + "epoch": 0.06688164337752299, + "grad_norm": 1208.0, + "learning_rate": 1.9166666666666668e-07, + "logits/chosen": -2.1356945037841797, + "logits/rejected": -2.046206474304199, + "logps/chosen": -0.27981576323509216, + "logps/rejected": -0.2755919098854065, + "loss": 4431.11767578125, + "loss/core": 4431.11328125, + "loss/preference_sft": 0.27981576323509216, + "loss/reference_anchor": 0.20977923274040222, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.4919538497924805, + "rewards/margins": 1.011785864830017, + "rewards/rejected": 1.4801679849624634, + "step": 70 + }, + { + "drift/chosen_abs": 0.22614233195781708, + "drift/rejected_abs": 0.11433468014001846, + "epoch": 0.07165890361877464, + "grad_norm": 584.0, + "learning_rate": 2.0555555555555553e-07, + "logits/chosen": -2.0366768836975098, + "logits/rejected": -2.0689096450805664, + "logps/chosen": -0.30070480704307556, + "logps/rejected": -0.3065111041069031, + "loss": 4428.748046875, + "loss/core": 4428.7421875, + "loss/preference_sft": 0.30070480704307556, + "loss/reference_anchor": 0.25870731472969055, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.259551525115967, + "rewards/margins": 1.1922428607940674, + "rewards/rejected": 1.067308783531189, + "step": 75 + }, + { + "drift/chosen_abs": 0.13649195432662964, + "drift/rejected_abs": 0.0969230979681015, + "epoch": 0.07643616386002627, + "grad_norm": 392.0, + "learning_rate": 2.1944444444444442e-07, + "logits/chosen": -2.2698612213134766, + "logits/rejected": -2.2682509422302246, + "logps/chosen": -0.2945755422115326, + "logps/rejected": -0.28585803508758545, + "loss": 4439.20751953125, + "loss/core": 4439.205078125, + "loss/preference_sft": 0.2945755422115326, + "loss/reference_anchor": 0.09729237854480743, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.3633534908294678, + "rewards/margins": 0.39599722623825073, + "rewards/rejected": 0.9673563241958618, + "step": 80 + }, + { + "drift/chosen_abs": 0.1682918667793274, + "drift/rejected_abs": 0.16686959564685822, + "epoch": 0.08121342410127792, + "grad_norm": 9024.0, + "learning_rate": 2.3333333333333333e-07, + "logits/chosen": -2.2202513217926025, + "logits/rejected": -2.233330011367798, + "logps/chosen": -0.2740824520587921, + "logps/rejected": -0.2770374119281769, + "loss": 4444.6669921875, + "loss/core": 4444.66064453125, + "loss/preference_sft": 0.2740824520587921, + "loss/reference_anchor": 0.2943022847175598, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.6820385456085205, + "rewards/margins": 0.015296387486159801, + "rewards/rejected": 1.6667420864105225, + "step": 85 + }, + { + "drift/chosen_abs": 0.25606733560562134, + "drift/rejected_abs": 0.16503527760505676, + "epoch": 0.08599068434252956, + "grad_norm": 1632.0, + "learning_rate": 2.4722222222222224e-07, + "logits/chosen": -2.130919933319092, + "logits/rejected": -2.2292962074279785, + "logps/chosen": -0.2705443799495697, + "logps/rejected": -0.3077170252799988, + "loss": 4431.96630859375, + "loss/core": 4431.95703125, + "loss/preference_sft": 0.2705443799495697, + "loss/reference_anchor": 0.43878602981567383, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.5599331855773926, + "rewards/margins": 0.9912513494491577, + "rewards/rejected": 1.5686819553375244, + "step": 90 + }, + { + "drift/chosen_abs": 0.22851324081420898, + "drift/rejected_abs": 0.10233010351657867, + "epoch": 0.09076794458378121, + "grad_norm": 2288.0, + "learning_rate": 2.4998495746616845e-07, + "logits/chosen": -2.1402933597564697, + "logits/rejected": -2.06827974319458, + "logps/chosen": -0.28999412059783936, + "logps/rejected": -0.2968214154243469, + "loss": 4427.84716796875, + "loss/core": 4427.84228515625, + "loss/preference_sft": 0.28999412059783936, + "loss/reference_anchor": 0.24312031269073486, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.2837419509887695, + "rewards/margins": 1.2656266689300537, + "rewards/rejected": 1.0181152820587158, + "step": 95 + }, + { + "drift/chosen_abs": 0.37163451313972473, + "drift/rejected_abs": 0.1659991592168808, + "epoch": 0.09554520482503284, + "grad_norm": 836.0, + "learning_rate": 2.4992385337738696e-07, + "logits/chosen": -2.1171348094940186, + "logits/rejected": -2.160855531692505, + "logps/chosen": -0.268626868724823, + "logps/rejected": -0.28261902928352356, + "loss": 4417.1259765625, + "loss/core": 4417.11474609375, + "loss/preference_sft": 0.268626868724823, + "loss/reference_anchor": 0.558134138584137, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.716301679611206, + "rewards/margins": 2.0977909564971924, + "rewards/rejected": 1.6185109615325928, + "step": 100 + }, + { + "drift/chosen_abs": 0.3358571529388428, + "drift/rejected_abs": 0.08821113407611847, + "epoch": 0.10032246506628449, + "grad_norm": 1464.0, + "learning_rate": 2.4981577053636144e-07, + "logits/chosen": -2.0699539184570312, + "logits/rejected": -2.0407674312591553, + "logps/chosen": -0.31734392046928406, + "logps/rejected": -0.30329352617263794, + "loss": 4411.2470703125, + "loss/core": 4411.2373046875, + "loss/preference_sft": 0.31734392046928406, + "loss/reference_anchor": 0.4697941243648529, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.355891466140747, + "rewards/margins": 2.551638126373291, + "rewards/rejected": 0.8042529821395874, + "step": 105 + }, + { + "drift/chosen_abs": 0.16569358110427856, + "drift/rejected_abs": 0.0958634614944458, + "epoch": 0.10509972530753613, + "grad_norm": 584.0, + "learning_rate": 2.496607495886281e-07, + "logits/chosen": -2.129267930984497, + "logits/rejected": -2.1310436725616455, + "logps/chosen": -0.29805296659469604, + "logps/rejected": -0.3116980493068695, + "loss": 4435.1533203125, + "loss/core": 4435.1513671875, + "loss/preference_sft": 0.29805296659469604, + "loss/reference_anchor": 0.10324454307556152, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.6569359302520752, + "rewards/margins": 0.7000143527984619, + "rewards/rejected": 0.956921398639679, + "step": 110 + }, + { + "drift/chosen_abs": 0.29079359769821167, + "drift/rejected_abs": 0.1145281046628952, + "epoch": 0.10987698554878778, + "grad_norm": 1192.0, + "learning_rate": 2.4945884883122553e-07, + "logits/chosen": -2.0063235759735107, + "logits/rejected": -2.1220145225524902, + "logps/chosen": -0.3002547025680542, + "logps/rejected": -0.2964302897453308, + "loss": 4421.4990234375, + "loss/core": 4421.486328125, + "loss/preference_sft": 0.3002547025680542, + "loss/reference_anchor": 0.597941517829895, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.9075615406036377, + "rewards/margins": 1.7633129358291626, + "rewards/rejected": 1.1442487239837646, + "step": 115 + }, + { + "drift/chosen_abs": 0.2360951155424118, + "drift/rejected_abs": 0.12227632850408554, + "epoch": 0.11465424579003941, + "grad_norm": 2736.0, + "learning_rate": 2.492101441907714e-07, + "logits/chosen": -2.2496066093444824, + "logits/rejected": -2.261533498764038, + "logps/chosen": -0.304383784532547, + "logps/rejected": -0.30693715810775757, + "loss": 4428.9970703125, + "loss/core": 4428.9921875, + "loss/preference_sft": 0.304383784532547, + "loss/reference_anchor": 0.24186281859874725, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.359388828277588, + "rewards/margins": 1.1752779483795166, + "rewards/rejected": 1.1841111183166504, + "step": 120 + }, + { + "drift/chosen_abs": 0.1892295777797699, + "drift/rejected_abs": 0.22446322441101074, + "epoch": 0.11943150603129106, + "grad_norm": 808.0, + "learning_rate": 2.4891472919490977e-07, + "logits/chosen": -2.0349624156951904, + "logits/rejected": -1.9459272623062134, + "logps/chosen": -0.292393296957016, + "logps/rejected": -0.2992958724498749, + "loss": 4449.83154296875, + "loss/core": 4449.8212890625, + "loss/preference_sft": 0.292393296957016, + "loss/reference_anchor": 0.48200732469558716, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.8902618885040283, + "rewards/margins": -0.35318875312805176, + "rewards/rejected": 2.24345064163208, + "step": 125 + }, + { + "drift/chosen_abs": 0.22298288345336914, + "drift/rejected_abs": 0.20003148913383484, + "epoch": 0.1242087662725427, + "grad_norm": 2336.0, + "learning_rate": 2.4857271493713894e-07, + "logits/chosen": -2.3568854331970215, + "logits/rejected": -2.2751965522766113, + "logps/chosen": -0.2751834988594055, + "logps/rejected": -0.298502653837204, + "loss": 4441.8662109375, + "loss/core": 4441.8583984375, + "loss/preference_sft": 0.2751834988594055, + "loss/reference_anchor": 0.33170682191848755, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.2295897006988525, + "rewards/margins": 0.2301691472530365, + "rewards/rejected": 1.9994205236434937, + "step": 130 + }, + { + "drift/chosen_abs": 0.27084535360336304, + "drift/rejected_abs": 0.1473771631717682, + "epoch": 0.12898602651379434, + "grad_norm": 1432.0, + "learning_rate": 2.481842300350339e-07, + "logits/chosen": -2.086169958114624, + "logits/rejected": -2.182788848876953, + "logps/chosen": -0.34438925981521606, + "logps/rejected": -0.3138183355331421, + "loss": 4428.39990234375, + "loss/core": 4428.39208984375, + "loss/preference_sft": 0.34438925981521606, + "loss/reference_anchor": 0.338693231344223, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.7076194286346436, + "rewards/margins": 1.236877679824829, + "rewards/rejected": 1.4707417488098145, + "step": 135 + }, + { + "drift/chosen_abs": 0.3097183108329773, + "drift/rejected_abs": 0.24102067947387695, + "epoch": 0.13376328675504598, + "grad_norm": 1904.0, + "learning_rate": 2.4774942058187854e-07, + "logits/chosen": -1.950994849205017, + "logits/rejected": -1.937705397605896, + "logps/chosen": -0.3324321210384369, + "logps/rejected": -0.33997035026550293, + "loss": 4436.19677734375, + "loss/core": 4436.18359375, + "loss/preference_sft": 0.3324321210384369, + "loss/reference_anchor": 0.619308590888977, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.096101760864258, + "rewards/margins": 0.6867878437042236, + "rewards/rejected": 2.409313678741455, + "step": 140 + }, + { + "drift/chosen_abs": 0.17987477779388428, + "drift/rejected_abs": 0.08481304347515106, + "epoch": 0.13854054699629761, + "grad_norm": 1048.0, + "learning_rate": 2.472684500917257e-07, + "logits/chosen": -2.043088912963867, + "logits/rejected": -2.053077220916748, + "logps/chosen": -0.300198495388031, + "logps/rejected": -0.2859780192375183, + "loss": 4431.86328125, + "loss/core": 4431.8603515625, + "loss/preference_sft": 0.300198495388031, + "loss/reference_anchor": 0.10386216640472412, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.7945579290390015, + "rewards/margins": 0.9506572484970093, + "rewards/rejected": 0.8439006805419922, + "step": 145 + }, + { + "drift/chosen_abs": 0.2635027766227722, + "drift/rejected_abs": 0.12567314505577087, + "epoch": 0.14331780723754928, + "grad_norm": 3248.0, + "learning_rate": 2.467414994379065e-07, + "logits/chosen": -2.0115456581115723, + "logits/rejected": -2.035735845565796, + "logps/chosen": -0.28563544154167175, + "logps/rejected": -0.2719977796077728, + "loss": 4426.4873046875, + "loss/core": 4426.47998046875, + "loss/preference_sft": 0.28563544154167175, + "loss/reference_anchor": 0.3351196348667145, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.6349034309387207, + "rewards/margins": 1.3796746730804443, + "rewards/rejected": 1.2552287578582764, + "step": 150 + }, + { + "drift/chosen_abs": 0.26199209690093994, + "drift/rejected_abs": 0.14406320452690125, + "epoch": 0.1480950674788009, + "grad_norm": 1020.0, + "learning_rate": 2.4616876678501114e-07, + "logits/chosen": -1.9780941009521484, + "logits/rejected": -2.0333588123321533, + "logps/chosen": -0.24971485137939453, + "logps/rejected": -0.26529937982559204, + "loss": 4428.74267578125, + "loss/core": 4428.73779296875, + "loss/preference_sft": 0.24971485137939453, + "loss/reference_anchor": 0.19584576785564423, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.6188275814056396, + "rewards/margins": 1.1842237710952759, + "rewards/rejected": 1.4346040487289429, + "step": 155 + }, + { + "drift/chosen_abs": 0.2649013102054596, + "drift/rejected_abs": 0.11616607755422592, + "epoch": 0.15287232772005255, + "grad_norm": 1984.0, + "learning_rate": 2.4555046751436735e-07, + "logits/chosen": -2.1708004474639893, + "logits/rejected": -2.1445536613464355, + "logps/chosen": -0.2795725464820862, + "logps/rejected": -0.2928757071495056, + "loss": 4425.0234375, + "loss/core": 4425.0166015625, + "loss/preference_sft": 0.2795725464820862, + "loss/reference_anchor": 0.29672375321388245, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.647858142852783, + "rewards/margins": 1.4861972332000732, + "rewards/rejected": 1.1616607904434204, + "step": 160 + }, + { + "drift/chosen_abs": 0.17845085263252258, + "drift/rejected_abs": 0.09973239153623581, + "epoch": 0.15764958796130418, + "grad_norm": 848.0, + "learning_rate": 2.4488683414304425e-07, + "logits/chosen": -2.1181225776672363, + "logits/rejected": -2.0922858715057373, + "logps/chosen": -0.3071829080581665, + "logps/rejected": -0.30448269844055176, + "loss": 4434.00048828125, + "loss/core": 4433.998046875, + "loss/preference_sft": 0.3071829080581665, + "loss/reference_anchor": 0.09884973615407944, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.7831709384918213, + "rewards/margins": 0.7861735820770264, + "rewards/rejected": 0.9969974756240845, + "step": 165 + }, + { + "drift/chosen_abs": 0.18412050604820251, + "drift/rejected_abs": 0.0624062642455101, + "epoch": 0.16242684820255585, + "grad_norm": 450.0, + "learning_rate": 2.4417811623641203e-07, + "logits/chosen": -2.074202537536621, + "logits/rejected": -2.0019283294677734, + "logps/chosen": -0.30903059244155884, + "logps/rejected": -0.3260036110877991, + "loss": 4428.12548828125, + "loss/core": 4428.123046875, + "loss/preference_sft": 0.30903059244155884, + "loss/reference_anchor": 0.10582767426967621, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.8374769687652588, + "rewards/margins": 1.235172152519226, + "rewards/rejected": 0.6023045182228088, + "step": 170 + }, + { + "drift/chosen_abs": 0.17877274751663208, + "drift/rejected_abs": 0.07043075561523438, + "epoch": 0.16720410844380748, + "grad_norm": 2608.0, + "learning_rate": 2.4342458031429113e-07, + "logits/chosen": -2.2448172569274902, + "logits/rejected": -2.2653393745422363, + "logps/chosen": -0.2739713490009308, + "logps/rejected": -0.28165867924690247, + "loss": 4430.08447265625, + "loss/core": 4430.08154296875, + "loss/preference_sft": 0.2739713490009308, + "loss/reference_anchor": 0.09590987861156464, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.787602424621582, + "rewards/margins": 1.0849480628967285, + "rewards/rejected": 0.702654242515564, + "step": 175 + }, + { + "drift/chosen_abs": 0.2464297115802765, + "drift/rejected_abs": 0.13828793168067932, + "epoch": 0.17198136868505912, + "grad_norm": 1020.0, + "learning_rate": 2.4262650975072444e-07, + "logits/chosen": -2.204326629638672, + "logits/rejected": -2.1553001403808594, + "logps/chosen": -0.2618137300014496, + "logps/rejected": -0.28036749362945557, + "loss": 4430.1103515625, + "loss/core": 4430.10302734375, + "loss/preference_sft": 0.2618137300014496, + "loss/reference_anchor": 0.3428693115711212, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.463837146759033, + "rewards/margins": 1.0816283226013184, + "rewards/rejected": 1.3822085857391357, + "step": 180 + }, + { + "drift/chosen_abs": 0.19091588258743286, + "drift/rejected_abs": 0.15579600632190704, + "epoch": 0.17675862892631075, + "grad_norm": 6464.0, + "learning_rate": 2.417842046674122e-07, + "logits/chosen": -2.212608814239502, + "logits/rejected": -2.253899574279785, + "logps/chosen": -0.30273327231407166, + "logps/rejected": -0.302903950214386, + "loss": 4440.3125, + "loss/core": 4440.30517578125, + "loss/preference_sft": 0.30273327231407166, + "loss/reference_anchor": 0.3576907217502594, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.9086692333221436, + "rewards/margins": 0.3515912890434265, + "rewards/rejected": 1.5570778846740723, + "step": 185 + }, + { + "drift/chosen_abs": 0.2642178237438202, + "drift/rejected_abs": 0.11951635032892227, + "epoch": 0.18153588916756241, + "grad_norm": 7584.0, + "learning_rate": 2.4089798182084843e-07, + "logits/chosen": -2.1424639225006104, + "logits/rejected": -2.1239447593688965, + "logps/chosen": -0.28714102506637573, + "logps/rejected": -0.270624577999115, + "loss": 4425.54296875, + "loss/core": 4425.5361328125, + "loss/preference_sft": 0.28714102506637573, + "loss/reference_anchor": 0.32669633626937866, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.6386032104492188, + "rewards/margins": 1.4514687061309814, + "rewards/rejected": 1.1871345043182373, + "step": 190 + }, + { + "drift/chosen_abs": 0.1681932508945465, + "drift/rejected_abs": 0.13734407722949982, + "epoch": 0.18631314940881405, + "grad_norm": 752.0, + "learning_rate": 2.3996817448320195e-07, + "logits/chosen": -1.964268445968628, + "logits/rejected": -1.9507757425308228, + "logps/chosen": -0.28487104177474976, + "logps/rejected": -0.29054194688796997, + "loss": 4440.4189453125, + "loss/core": 4440.416015625, + "loss/preference_sft": 0.28487104177474976, + "loss/reference_anchor": 0.1267441213130951, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.6816270351409912, + "rewards/margins": 0.30894654989242554, + "rewards/rejected": 1.3726806640625, + "step": 195 + }, + { + "drift/chosen_abs": 0.25091999769210815, + "drift/rejected_abs": 0.14358346164226532, + "epoch": 0.19109040965006568, + "grad_norm": 1408.0, + "learning_rate": 2.3899513231698607e-07, + "logits/chosen": -2.1187353134155273, + "logits/rejected": -2.150465488433838, + "logps/chosen": -0.2739163041114807, + "logps/rejected": -0.27358219027519226, + "loss": 4429.14306640625, + "loss/core": 4429.1376953125, + "loss/preference_sft": 0.2739163041114807, + "loss/reference_anchor": 0.23309831321239471, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.507127523422241, + "rewards/margins": 1.161219835281372, + "rewards/rejected": 1.3459078073501587, + "step": 200 + }, + { + "drift/chosen_abs": 0.13701114058494568, + "drift/rejected_abs": 0.06575393676757812, + "epoch": 0.19586766989131732, + "grad_norm": 716.0, + "learning_rate": 2.3797922124356506e-07, + "logits/chosen": -2.0870461463928223, + "logits/rejected": -2.1504645347595215, + "logps/chosen": -0.2821192443370819, + "logps/rejected": -0.28834599256515503, + "loss": 4434.98291015625, + "loss/core": 4434.9814453125, + "loss/preference_sft": 0.2821192443370819, + "loss/reference_anchor": 0.043852418661117554, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.3701115846633911, + "rewards/margins": 0.7137712240219116, + "rewards/rejected": 0.6563402414321899, + "step": 205 + }, + { + "drift/chosen_abs": 0.25299134850502014, + "drift/rejected_abs": 0.1516466736793518, + "epoch": 0.20064493013256898, + "grad_norm": 1152.0, + "learning_rate": 2.3692082330554585e-07, + "logits/chosen": -2.03848934173584, + "logits/rejected": -2.016936779022217, + "logps/chosen": -0.28784194588661194, + "logps/rejected": -0.28754788637161255, + "loss": 4430.9228515625, + "loss/core": 4430.91357421875, + "loss/preference_sft": 0.28784194588661194, + "loss/reference_anchor": 0.4292024075984955, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.5295217037200928, + "rewards/margins": 1.0197639465332031, + "rewards/rejected": 1.5097577571868896, + "step": 210 + }, + { + "drift/chosen_abs": 0.20603258907794952, + "drift/rejected_abs": 0.10440722852945328, + "epoch": 0.20542219037382062, + "grad_norm": 760.0, + "learning_rate": 2.3582033652310765e-07, + "logits/chosen": -1.955307960510254, + "logits/rejected": -1.973205804824829, + "logps/chosen": -0.2961030900478363, + "logps/rejected": -0.27407750487327576, + "loss": 4430.9453125, + "loss/core": 4430.9423828125, + "loss/preference_sft": 0.2961030900478363, + "loss/reference_anchor": 0.12213130295276642, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.0588302612304688, + "rewards/margins": 1.0177305936813354, + "rewards/rejected": 1.0410997867584229, + "step": 215 + }, + { + "drift/chosen_abs": 0.1778123527765274, + "drift/rejected_abs": 0.08586692065000534, + "epoch": 0.21019945061507225, + "grad_norm": 656.0, + "learning_rate": 2.346781747443227e-07, + "logits/chosen": -2.037010669708252, + "logits/rejected": -2.0615968704223633, + "logps/chosen": -0.28160199522972107, + "logps/rejected": -0.2732716202735901, + "loss": 4432.251953125, + "loss/core": 4432.24951171875, + "loss/preference_sft": 0.28160199522972107, + "loss/reference_anchor": 0.08426757156848907, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.7759720087051392, + "rewards/margins": 0.9196584820747375, + "rewards/rejected": 0.8563135266304016, + "step": 220 + }, + { + "drift/chosen_abs": 0.3073180913925171, + "drift/rejected_abs": 0.20542486011981964, + "epoch": 0.2149767108563239, + "grad_norm": 1480.0, + "learning_rate": 2.3349476748952508e-07, + "logits/chosen": -2.0461509227752686, + "logits/rejected": -2.0668578147888184, + "logps/chosen": -0.3112167716026306, + "logps/rejected": -0.3018384873867035, + "loss": 4431.3017578125, + "loss/core": 4431.28515625, + "loss/preference_sft": 0.3112167716026306, + "loss/reference_anchor": 0.7777162194252014, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.071030855178833, + "rewards/margins": 1.018129825592041, + "rewards/rejected": 2.052901268005371, + "step": 225 + }, + { + "drift/chosen_abs": 0.3042348027229309, + "drift/rejected_abs": 0.16846933960914612, + "epoch": 0.21975397109757555, + "grad_norm": 1632.0, + "learning_rate": 2.3227055978978545e-07, + "logits/chosen": -2.130049467086792, + "logits/rejected": -2.120723247528076, + "logps/chosen": -0.27312982082366943, + "logps/rejected": -0.26595622301101685, + "loss": 4426.55078125, + "loss/core": 4426.54296875, + "loss/preference_sft": 0.27312982082366943, + "loss/reference_anchor": 0.3708053231239319, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.0423481464385986, + "rewards/margins": 1.3577412366867065, + "rewards/rejected": 1.6846065521240234, + "step": 230 + }, + { + "drift/chosen_abs": 0.17078641057014465, + "drift/rejected_abs": 0.06932120025157928, + "epoch": 0.2245312313388272, + "grad_norm": 544.0, + "learning_rate": 2.3100601201955321e-07, + "logits/chosen": -2.171781063079834, + "logits/rejected": -2.267810106277466, + "logps/chosen": -0.2851482927799225, + "logps/rejected": -0.279243528842926, + "loss": 4430.9599609375, + "loss/core": 4430.9580078125, + "loss/preference_sft": 0.2851482927799225, + "loss/reference_anchor": 0.10230743885040283, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.7078640460968018, + "rewards/margins": 1.018509864807129, + "rewards/rejected": 0.6893541216850281, + "step": 235 + }, + { + "drift/chosen_abs": 0.27906155586242676, + "drift/rejected_abs": 0.19857071340084076, + "epoch": 0.22930849158007882, + "grad_norm": 1012.0, + "learning_rate": 2.2970159972352864e-07, + "logits/chosen": -2.0736007690429688, + "logits/rejected": -2.1250550746917725, + "logps/chosen": -0.2709374725818634, + "logps/rejected": -0.28649747371673584, + "loss": 4434.0234375, + "loss/core": 4434.009765625, + "loss/preference_sft": 0.2709374725818634, + "loss/reference_anchor": 0.656843900680542, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.78991436958313, + "rewards/margins": 0.8058086633682251, + "rewards/rejected": 1.9841057062149048, + "step": 240 + }, + { + "drift/chosen_abs": 0.21625050902366638, + "drift/rejected_abs": 0.06605993956327438, + "epoch": 0.23408575182133046, + "grad_norm": 6624.0, + "learning_rate": 2.2835781343782966e-07, + "logits/chosen": -2.1297452449798584, + "logits/rejected": -2.2750277519226074, + "logps/chosen": -0.3038528859615326, + "logps/rejected": -0.29280510544776917, + "loss": 4424.94091796875, + "loss/core": 4424.9326171875, + "loss/preference_sft": 0.3038528859615326, + "loss/reference_anchor": 0.3740764260292053, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.1619296073913574, + "rewards/margins": 1.504833698272705, + "rewards/rejected": 0.6570958495140076, + "step": 245 + }, + { + "drift/chosen_abs": 0.33135589957237244, + "drift/rejected_abs": 0.16284796595573425, + "epoch": 0.23886301206258212, + "grad_norm": 916.0, + "learning_rate": 2.2697515850552152e-07, + "logits/chosen": -1.9698188304901123, + "logits/rejected": -1.9743677377700806, + "logps/chosen": -0.30887797474861145, + "logps/rejected": -0.29849907755851746, + "loss": 4422.46337890625, + "loss/core": 4422.45166015625, + "loss/preference_sft": 0.30887797474861145, + "loss/reference_anchor": 0.5555719137191772, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.312263011932373, + "rewards/margins": 1.6859710216522217, + "rewards/rejected": 1.6262918710708618, + "step": 250 + }, + { + "drift/chosen_abs": 0.27730509638786316, + "drift/rejected_abs": 0.11791691929101944, + "epoch": 0.24364027230383375, + "grad_norm": 600.0, + "learning_rate": 2.2555415488657775e-07, + "logits/chosen": -2.120657444000244, + "logits/rejected": -2.0884969234466553, + "logps/chosen": -0.28511613607406616, + "logps/rejected": -0.2897268831729889, + "loss": 4423.853515625, + "loss/core": 4423.83984375, + "loss/preference_sft": 0.28511613607406616, + "loss/reference_anchor": 0.6375155448913574, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.772818088531494, + "rewards/margins": 1.5964891910552979, + "rewards/rejected": 1.1763291358947754, + "step": 255 + }, + { + "drift/chosen_abs": 0.19447073340415955, + "drift/rejected_abs": 0.12607555091381073, + "epoch": 0.2484175325450854, + "grad_norm": 2720.0, + "learning_rate": 2.240953369623447e-07, + "logits/chosen": -2.07661771774292, + "logits/rejected": -2.0507664680480957, + "logps/chosen": -0.3053693175315857, + "logps/rejected": -0.307594358921051, + "loss": 4435.50927734375, + "loss/core": 4435.5048828125, + "loss/preference_sft": 0.3053693175315857, + "loss/reference_anchor": 0.21331624686717987, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9445161819458008, + "rewards/margins": 0.6846358180046082, + "rewards/rejected": 1.2598803043365479, + "step": 260 + }, + { + "drift/chosen_abs": 0.213128000497818, + "drift/rejected_abs": 0.10632689297199249, + "epoch": 0.25319479278633705, + "grad_norm": 724.0, + "learning_rate": 2.225992533345824e-07, + "logits/chosen": -2.0516610145568848, + "logits/rejected": -2.0290393829345703, + "logps/chosen": -0.2761135995388031, + "logps/rejected": -0.28632479906082153, + "loss": 4430.1591796875, + "loss/core": 4430.15673828125, + "loss/preference_sft": 0.2761135995388031, + "loss/reference_anchor": 0.1218862310051918, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.1307318210601807, + "rewards/margins": 1.079505205154419, + "rewards/rejected": 1.0512263774871826, + "step": 265 + }, + { + "drift/chosen_abs": 0.16613757610321045, + "drift/rejected_abs": 0.1417321413755417, + "epoch": 0.2579720530275887, + "grad_norm": 474.0, + "learning_rate": 2.210664666191579e-07, + "logits/chosen": -2.102410316467285, + "logits/rejected": -2.0517630577087402, + "logps/chosen": -0.30586427450180054, + "logps/rejected": -0.3037346303462982, + "loss": 4441.33203125, + "loss/core": 4441.328125, + "loss/preference_sft": 0.30586427450180054, + "loss/reference_anchor": 0.16727973520755768, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6612533330917358, + "rewards/margins": 0.24511925876140594, + "rewards/rejected": 1.416133999824524, + "step": 270 + }, + { + "drift/chosen_abs": 0.10398060083389282, + "drift/rejected_abs": 0.05893680453300476, + "epoch": 0.2627493132688403, + "grad_norm": 820.0, + "learning_rate": 2.1949755323446848e-07, + "logits/chosen": -2.2223498821258545, + "logits/rejected": -2.292362928390503, + "logps/chosen": -0.29613521695137024, + "logps/rejected": -0.2959131598472595, + "loss": 4438.3203125, + "loss/core": 4438.3193359375, + "loss/preference_sft": 0.29613521695137024, + "loss/reference_anchor": 0.0267405416816473, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.0383059978485107, + "rewards/margins": 0.46069198846817017, + "rewards/rejected": 0.5776140093803406, + "step": 275 + }, + { + "drift/chosen_abs": 0.18298956751823425, + "drift/rejected_abs": 0.08578231185674667, + "epoch": 0.26752657351009196, + "grad_norm": 1576.0, + "learning_rate": 2.1789310318467426e-07, + "logits/chosen": -2.0358364582061768, + "logits/rejected": -2.016176700592041, + "logps/chosen": -0.2997116446495056, + "logps/rejected": -0.29861459136009216, + "loss": 4431.60595703125, + "loss/core": 4431.60302734375, + "loss/preference_sft": 0.2997116446495056, + "loss/reference_anchor": 0.1133713498711586, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.8296573162078857, + "rewards/margins": 0.9720863103866577, + "rewards/rejected": 0.857571005821228, + "step": 280 + }, + { + "drift/chosen_abs": 0.29716697335243225, + "drift/rejected_abs": 0.20067524909973145, + "epoch": 0.2723038337513436, + "grad_norm": 972.0, + "learning_rate": 2.1625371983782182e-07, + "logits/chosen": -2.219372034072876, + "logits/rejected": -2.1742899417877197, + "logps/chosen": -0.28680914640426636, + "logps/rejected": -0.2844160497188568, + "loss": 4431.8857421875, + "loss/core": 4431.8740234375, + "loss/preference_sft": 0.28680914640426636, + "loss/reference_anchor": 0.5631648302078247, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.970261573791504, + "rewards/margins": 0.9673450589179993, + "rewards/rejected": 2.0029163360595703, + "step": 285 + }, + { + "drift/chosen_abs": 0.45473068952560425, + "drift/rejected_abs": 0.13007131218910217, + "epoch": 0.27708109399259523, + "grad_norm": 9920.0, + "learning_rate": 2.14580019698942e-07, + "logits/chosen": -2.151202440261841, + "logits/rejected": -2.2022032737731934, + "logps/chosen": -0.2817015051841736, + "logps/rejected": -0.27055931091308594, + "loss": 4402.37890625, + "loss/core": 4402.357421875, + "loss/preference_sft": 0.2817015051841736, + "loss/reference_anchor": 1.024298906326294, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.546250343322754, + "rewards/margins": 3.2471745014190674, + "rewards/rejected": 1.299075961112976, + "step": 290 + }, + { + "drift/chosen_abs": 0.2755797505378723, + "drift/rejected_abs": 0.1132490411400795, + "epoch": 0.28185835423384686, + "grad_norm": 1424.0, + "learning_rate": 2.1287263217820773e-07, + "logits/chosen": -1.9132686853408813, + "logits/rejected": -1.9894206523895264, + "logps/chosen": -0.31108710169792175, + "logps/rejected": -0.3019738793373108, + "loss": 4423.05029296875, + "loss/core": 4423.0380859375, + "loss/preference_sft": 0.31108710169792175, + "loss/reference_anchor": 0.5742016434669495, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.755403518676758, + "rewards/margins": 1.6267465353012085, + "rewards/rejected": 1.1286571025848389, + "step": 295 + }, + { + "drift/chosen_abs": 0.3510657846927643, + "drift/rejected_abs": 0.1981009691953659, + "epoch": 0.28663561447509855, + "grad_norm": 600.0, + "learning_rate": 2.111321993542385e-07, + "logits/chosen": -2.1004462242126465, + "logits/rejected": -2.165738582611084, + "logps/chosen": -0.2795126140117645, + "logps/rejected": -0.30045658349990845, + "loss": 4424.57080078125, + "loss/core": 4424.5498046875, + "loss/preference_sft": 0.2795126140117645, + "loss/reference_anchor": 1.045248031616211, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.509197235107422, + "rewards/margins": 1.5337461233139038, + "rewards/rejected": 1.9754512310028076, + "step": 300 + }, + { + "drift/chosen_abs": 0.3817795217037201, + "drift/rejected_abs": 0.25070327520370483, + "epoch": 0.2914128747163502, + "grad_norm": 4544.0, + "learning_rate": 2.0935937573264096e-07, + "logits/chosen": -1.98002028465271, + "logits/rejected": -2.0424513816833496, + "logps/chosen": -0.29999712109565735, + "logps/rejected": -0.30416327714920044, + "loss": 4427.5712890625, + "loss/core": 4427.548828125, + "loss/preference_sft": 0.29999712109565735, + "loss/reference_anchor": 1.0998529195785522, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.8161988258361816, + "rewards/margins": 1.3097256422042847, + "rewards/rejected": 2.5064730644226074, + "step": 305 + }, + { + "drift/chosen_abs": 0.23365557193756104, + "drift/rejected_abs": 0.14666993916034698, + "epoch": 0.2961901349576018, + "grad_norm": 2416.0, + "learning_rate": 2.0755482799987596e-07, + "logits/chosen": -1.9693409204483032, + "logits/rejected": -2.007472038269043, + "logps/chosen": -0.27722686529159546, + "logps/rejected": -0.27590957283973694, + "loss": 4433.0087890625, + "loss/core": 4433.00439453125, + "loss/preference_sft": 0.27722686529159546, + "loss/reference_anchor": 0.190281942486763, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.3363592624664307, + "rewards/margins": 0.8708029985427856, + "rewards/rejected": 1.4655563831329346, + "step": 310 + }, + { + "drift/chosen_abs": 0.11892326921224594, + "drift/rejected_abs": 0.12483175098896027, + "epoch": 0.30096739519885346, + "grad_norm": 1304.0, + "learning_rate": 2.0571923477254526e-07, + "logits/chosen": -2.1190714836120605, + "logits/rejected": -2.103590250015259, + "logps/chosen": -0.30928635597229004, + "logps/rejected": -0.30010607838630676, + "loss": 4445.4541015625, + "loss/core": 4445.4501953125, + "loss/preference_sft": 0.30928635597229004, + "loss/reference_anchor": 0.1733807623386383, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.1855305433273315, + "rewards/margins": -0.060359179973602295, + "rewards/rejected": 1.2458897829055786, + "step": 315 + }, + { + "drift/chosen_abs": 0.3176988959312439, + "drift/rejected_abs": 0.17362681031227112, + "epoch": 0.3057446554401051, + "grad_norm": 12032.0, + "learning_rate": 2.038532863421914e-07, + "logits/chosen": -1.9140937328338623, + "logits/rejected": -1.8793401718139648, + "logps/chosen": -0.315841943025589, + "logps/rejected": -0.31801360845565796, + "loss": 4425.9814453125, + "loss/core": 4425.97021484375, + "loss/preference_sft": 0.315841943025589, + "loss/reference_anchor": 0.5636979937553406, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.17467999458313, + "rewards/margins": 1.440582513809204, + "rewards/rejected": 1.7340972423553467, + "step": 320 + }, + { + "drift/chosen_abs": 0.1431046724319458, + "drift/rejected_abs": 0.0807650089263916, + "epoch": 0.31052191568135673, + "grad_norm": 510.0, + "learning_rate": 2.0195768441570726e-07, + "logits/chosen": -2.013561487197876, + "logits/rejected": -2.0425143241882324, + "logps/chosen": -0.2884654402732849, + "logps/rejected": -0.29259175062179565, + "loss": 4436.1591796875, + "loss/core": 4436.1572265625, + "loss/preference_sft": 0.2884654402732849, + "loss/reference_anchor": 0.061391688883304596, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.430469274520874, + "rewards/margins": 0.6241756677627563, + "rewards/rejected": 0.8062933683395386, + "step": 325 + }, + { + "drift/chosen_abs": 0.13373948633670807, + "drift/rejected_abs": 0.122031569480896, + "epoch": 0.31529917592260837, + "grad_norm": 11648.0, + "learning_rate": 2.0003314185145307e-07, + "logits/chosen": -2.183828830718994, + "logits/rejected": -2.1095242500305176, + "logps/chosen": -0.3171173930168152, + "logps/rejected": -0.3337666392326355, + "loss": 4443.197265625, + "loss/core": 4443.1923828125, + "loss/preference_sft": 0.3171173930168152, + "loss/reference_anchor": 0.21503393352031708, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.336255431175232, + "rewards/margins": 0.11729172617197037, + "rewards/rejected": 1.2189635038375854, + "step": 330 + }, + { + "drift/chosen_abs": 0.10823869705200195, + "drift/rejected_abs": 0.056395940482616425, + "epoch": 0.32007643616386, + "grad_norm": 812.0, + "learning_rate": 1.9808038239117913e-07, + "logits/chosen": -2.2766873836517334, + "logits/rejected": -2.3121063709259033, + "logps/chosen": -0.30512499809265137, + "logps/rejected": -0.306996613740921, + "loss": 4437.56005859375, + "loss/core": 4437.5595703125, + "loss/preference_sft": 0.30512499809265137, + "loss/reference_anchor": 0.025697598233819008, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.0811882019042969, + "rewards/margins": 0.5172288417816162, + "rewards/rejected": 0.5639593601226807, + "step": 335 + }, + { + "drift/chosen_abs": 0.22527499496936798, + "drift/rejected_abs": 0.09023512899875641, + "epoch": 0.3248536964051117, + "grad_norm": 764.0, + "learning_rate": 1.9610014038785646e-07, + "logits/chosen": -2.272291660308838, + "logits/rejected": -2.280327320098877, + "logps/chosen": -0.27914947271347046, + "logps/rejected": -0.28271421790122986, + "loss": 4426.8134765625, + "loss/core": 4426.8056640625, + "loss/preference_sft": 0.27914947271347046, + "loss/reference_anchor": 0.3431945741176605, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.2520639896392822, + "rewards/margins": 1.3524361848831177, + "rewards/rejected": 0.8996278047561646, + "step": 340 + }, + { + "drift/chosen_abs": 0.2175404578447342, + "drift/rejected_abs": 0.1755029261112213, + "epoch": 0.3296309566463633, + "grad_norm": 2672.0, + "learning_rate": 1.9409316052951657e-07, + "logits/chosen": -2.121755838394165, + "logits/rejected": -2.120198965072632, + "logps/chosen": -0.26820969581604004, + "logps/rejected": -0.2698619067668915, + "loss": 4439.0205078125, + "loss/core": 4439.01171875, + "loss/preference_sft": 0.26820969581604004, + "loss/reference_anchor": 0.3845630884170532, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.1744120121002197, + "rewards/margins": 0.42798107862472534, + "rewards/rejected": 1.7464309930801392, + "step": 345 + }, + { + "drift/chosen_abs": 0.23404574394226074, + "drift/rejected_abs": 0.14923684298992157, + "epoch": 0.33440821688761496, + "grad_norm": 992.0, + "learning_rate": 1.920601975592047e-07, + "logits/chosen": -2.22644305229187, + "logits/rejected": -2.2413718700408936, + "logps/chosen": -0.29749247431755066, + "logps/rejected": -0.3060537874698639, + "loss": 4432.6982421875, + "loss/core": 4432.6904296875, + "loss/preference_sft": 0.29749247431755066, + "loss/reference_anchor": 0.35366910696029663, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.3393826484680176, + "rewards/margins": 0.9169244766235352, + "rewards/rejected": 1.4224579334259033, + "step": 350 + }, + { + "drift/chosen_abs": 0.3508952260017395, + "drift/rejected_abs": 0.18243159353733063, + "epoch": 0.3391854771288666, + "grad_norm": 1576.0, + "learning_rate": 1.900020159911519e-07, + "logits/chosen": -2.1946253776550293, + "logits/rejected": -2.2092864513397217, + "logps/chosen": -0.285781592130661, + "logps/rejected": -0.26965421438217163, + "loss": 4422.45849609375, + "loss/core": 4422.4482421875, + "loss/preference_sft": 0.285781592130661, + "loss/reference_anchor": 0.4602643549442291, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.507547378540039, + "rewards/margins": 1.6848455667495728, + "rewards/rejected": 1.8227014541625977, + "step": 355 + }, + { + "drift/chosen_abs": 0.3240725100040436, + "drift/rejected_abs": 0.14371727406978607, + "epoch": 0.34396273737011823, + "grad_norm": 4448.0, + "learning_rate": 1.879193898232725e-07, + "logits/chosen": -1.9690821170806885, + "logits/rejected": -2.004427433013916, + "logps/chosen": -0.27598175406455994, + "logps/rejected": -0.2723207473754883, + "loss": 4420.2294921875, + "loss/core": 4420.2216796875, + "loss/preference_sft": 0.27598175406455994, + "loss/reference_anchor": 0.38521429896354675, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.24072527885437, + "rewards/margins": 1.8547264337539673, + "rewards/rejected": 1.3859989643096924, + "step": 360 + }, + { + "drift/chosen_abs": 0.250040739774704, + "drift/rejected_abs": 0.10423221439123154, + "epoch": 0.34873999761136987, + "grad_norm": 2208.0, + "learning_rate": 1.8581310224609496e-07, + "logits/chosen": -2.2583580017089844, + "logits/rejected": -2.328725576400757, + "logps/chosen": -0.3033362925052643, + "logps/rejected": -0.30249541997909546, + "loss": 4425.35693359375, + "loss/core": 4425.35009765625, + "loss/preference_sft": 0.3033362925052643, + "loss/reference_anchor": 0.2949478030204773, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.500340461730957, + "rewards/margins": 1.4580180644989014, + "rewards/rejected": 1.0423221588134766, + "step": 365 + }, + { + "drift/chosen_abs": 0.2433367520570755, + "drift/rejected_abs": 0.16708707809448242, + "epoch": 0.3535172578526215, + "grad_norm": 1004.0, + "learning_rate": 1.8368394534823635e-07, + "logits/chosen": -2.1375997066497803, + "logits/rejected": -2.0652060508728027, + "logps/chosen": -0.2894711196422577, + "logps/rejected": -0.3007884919643402, + "loss": 4434.6865234375, + "loss/core": 4434.677734375, + "loss/preference_sft": 0.2894711196422577, + "loss/reference_anchor": 0.397393137216568, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.4329514503479004, + "rewards/margins": 0.7622579336166382, + "rewards/rejected": 1.6706933975219727, + "step": 370 + }, + { + "drift/chosen_abs": 0.3843259811401367, + "drift/rejected_abs": 0.14180049300193787, + "epoch": 0.35829451809387314, + "grad_norm": 844.0, + "learning_rate": 1.8153271981852968e-07, + "logits/chosen": -2.014108419418335, + "logits/rejected": -1.938706398010254, + "logps/chosen": -0.28277072310447693, + "logps/rejected": -0.28435999155044556, + "loss": 4413.47509765625, + "loss/core": 4413.4560546875, + "loss/preference_sft": 0.28277072310447693, + "loss/reference_anchor": 0.9182009696960449, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 3.8399288654327393, + "rewards/margins": 2.425347089767456, + "rewards/rejected": 1.4145814180374146, + "step": 375 + }, + { + "drift/chosen_abs": 0.35180220007896423, + "drift/rejected_abs": 0.16935007274150848, + "epoch": 0.36307177833512483, + "grad_norm": 3824.0, + "learning_rate": 1.7936023464491812e-07, + "logits/chosen": -1.9925806522369385, + "logits/rejected": -2.077805995941162, + "logps/chosen": -0.2889358401298523, + "logps/rejected": -0.30222803354263306, + "loss": 4420.29736328125, + "loss/core": 4420.27685546875, + "loss/preference_sft": 0.2889358401298523, + "loss/reference_anchor": 0.9768502116203308, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.5166919231414795, + "rewards/margins": 1.8392245769500732, + "rewards/rejected": 1.6774673461914062, + "step": 380 + }, + { + "drift/chosen_abs": 0.311501145362854, + "drift/rejected_abs": 0.20911741256713867, + "epoch": 0.36784903857637646, + "grad_norm": 8704.0, + "learning_rate": 1.7716730681022723e-07, + "logits/chosen": -2.202134609222412, + "logits/rejected": -2.128066062927246, + "logps/chosen": -0.3021170496940613, + "logps/rejected": -0.3110412359237671, + "loss": 4431.4599609375, + "loss/core": 4431.4462890625, + "loss/preference_sft": 0.3021170496940613, + "loss/reference_anchor": 0.6745834350585938, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.115011692047119, + "rewards/margins": 1.0284936428070068, + "rewards/rejected": 2.086517810821533, + "step": 385 + }, + { + "drift/chosen_abs": 0.2097025215625763, + "drift/rejected_abs": 0.08994968235492706, + "epoch": 0.3726262988176281, + "grad_norm": 708.0, + "learning_rate": 1.7495476098493152e-07, + "logits/chosen": -2.065886974334717, + "logits/rejected": -2.094799757003784, + "logps/chosen": -0.29858630895614624, + "logps/rejected": -0.29905611276626587, + "loss": 4428.68310546875, + "loss/core": 4428.6787109375, + "loss/preference_sft": 0.29858630895614624, + "loss/reference_anchor": 0.24309007823467255, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.094350814819336, + "rewards/margins": 1.1971451044082642, + "rewards/rejected": 0.897205650806427, + "step": 390 + }, + { + "drift/chosen_abs": 0.20265540480613708, + "drift/rejected_abs": 0.09964494407176971, + "epoch": 0.37740355905887973, + "grad_norm": 824.0, + "learning_rate": 1.7272342921702937e-07, + "logits/chosen": -2.179865598678589, + "logits/rejected": -2.2556076049804688, + "logps/chosen": -0.2747688293457031, + "logps/rejected": -0.2691434919834137, + "loss": 4430.7685546875, + "loss/core": 4430.76611328125, + "loss/preference_sft": 0.2747688293457031, + "loss/reference_anchor": 0.09649205207824707, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.0265541076660156, + "rewards/margins": 1.0302097797393799, + "rewards/rejected": 0.9963444471359253, + "step": 395 + }, + { + "drift/chosen_abs": 0.20592239499092102, + "drift/rejected_abs": 0.16747477650642395, + "epoch": 0.38218081930013137, + "grad_norm": 1896.0, + "learning_rate": 1.7047415061914393e-07, + "logits/chosen": -2.1442654132843018, + "logits/rejected": -2.081714153289795, + "logps/chosen": -0.2744106352329254, + "logps/rejected": -0.2705535292625427, + "loss": 4439.44970703125, + "loss/core": 4439.44482421875, + "loss/preference_sft": 0.2744106352329254, + "loss/reference_anchor": 0.20020189881324768, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.0569989681243896, + "rewards/margins": 0.39103108644485474, + "rewards/rejected": 1.6659677028656006, + "step": 400 + }, + { + "drift/chosen_abs": 0.25720465183258057, + "drift/rejected_abs": 0.12404946982860565, + "epoch": 0.386958079541383, + "grad_norm": 7136.0, + "learning_rate": 1.6820777105296707e-07, + "logits/chosen": -2.3219733238220215, + "logits/rejected": -2.3814594745635986, + "logps/chosen": -0.2641887664794922, + "logps/rejected": -0.26516467332839966, + "loss": 4426.92333984375, + "loss/core": 4426.91162109375, + "loss/preference_sft": 0.2641887664794922, + "loss/reference_anchor": 0.5534130334854126, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.5716207027435303, + "rewards/margins": 1.331629753112793, + "rewards/rejected": 1.2399909496307373, + "step": 405 + }, + { + "drift/chosen_abs": 0.12248623371124268, + "drift/rejected_abs": 0.06730367243289948, + "epoch": 0.39173533978263464, + "grad_norm": 1352.0, + "learning_rate": 1.6592514281116553e-07, + "logits/chosen": -2.0958163738250732, + "logits/rejected": -2.1545522212982178, + "logps/chosen": -0.30971580743789673, + "logps/rejected": -0.3031691908836365, + "loss": 4437.1142578125, + "loss/core": 4437.1123046875, + "loss/preference_sft": 0.30971580743789673, + "loss/reference_anchor": 0.046301212161779404, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 1.2217724323272705, + "rewards/margins": 0.5523636937141418, + "rewards/rejected": 0.6694086790084839, + "step": 410 + }, + { + "drift/chosen_abs": 0.2457958459854126, + "drift/rejected_abs": 0.1320219784975052, + "epoch": 0.3965126000238863, + "grad_norm": 700.0, + "learning_rate": 1.636271242968684e-07, + "logits/chosen": -2.319655418395996, + "logits/rejected": -2.3182573318481445, + "logps/chosen": -0.2952980399131775, + "logps/rejected": -0.2870740294456482, + "loss": 4429.59912109375, + "loss/core": 4429.59375, + "loss/preference_sft": 0.2952980399131775, + "loss/reference_anchor": 0.270760178565979, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.457775831222534, + "rewards/margins": 1.1382719278335571, + "rewards/rejected": 1.3195040225982666, + "step": 415 + }, + { + "drift/chosen_abs": 0.3312036991119385, + "drift/rejected_abs": 0.10126861184835434, + "epoch": 0.40128986026513797, + "grad_norm": 7872.0, + "learning_rate": 1.6131457970085684e-07, + "logits/chosen": -2.119199752807617, + "logits/rejected": -2.087686061859131, + "logps/chosen": -0.32637646794319153, + "logps/rejected": -0.3032102882862091, + "loss": 4414.77880859375, + "loss/core": 4414.76318359375, + "loss/preference_sft": 0.32637646794319153, + "loss/reference_anchor": 0.7741028070449829, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.3084073066711426, + "rewards/margins": 2.3002755641937256, + "rewards/rejected": 1.0081310272216797, + "step": 420 + }, + { + "drift/chosen_abs": 0.3094877302646637, + "drift/rejected_abs": 0.10990305244922638, + "epoch": 0.4060671205063896, + "grad_norm": 1688.0, + "learning_rate": 1.5898837867657727e-07, + "logits/chosen": -2.1425862312316895, + "logits/rejected": -2.222062587738037, + "logps/chosen": -0.31150728464126587, + "logps/rejected": -0.31376224756240845, + "loss": 4418.671875, + "loss/core": 4418.65966796875, + "loss/preference_sft": 0.31150728464126587, + "loss/reference_anchor": 0.5772914886474609, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.094017744064331, + "rewards/margins": 1.9972339868545532, + "rewards/rejected": 1.0967837572097778, + "step": 425 + }, + { + "drift/chosen_abs": 0.14985668659210205, + "drift/rejected_abs": 0.0947737768292427, + "epoch": 0.41084438074764124, + "grad_norm": 3520.0, + "learning_rate": 1.5664939601310023e-07, + "logits/chosen": -2.170436143875122, + "logits/rejected": -2.2830162048339844, + "logps/chosen": -0.2657105326652527, + "logps/rejected": -0.2679697275161743, + "loss": 4437.12255859375, + "loss/core": 4437.1201171875, + "loss/preference_sft": 0.2657105326652527, + "loss/reference_anchor": 0.0813552588224411, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.4978549480438232, + "rewards/margins": 0.5538910627365112, + "rewards/rejected": 0.943963885307312, + "step": 430 + }, + { + "drift/chosen_abs": 0.39538705348968506, + "drift/rejected_abs": 0.13529789447784424, + "epoch": 0.41562164098889287, + "grad_norm": 11008.0, + "learning_rate": 1.5429851130614807e-07, + "logits/chosen": -1.9859920740127563, + "logits/rejected": -1.9977947473526, + "logps/chosen": -0.2817636728286743, + "logps/rejected": -0.2686322033405304, + "loss": 4410.4755859375, + "loss/core": 4410.4619140625, + "loss/preference_sft": 0.2817636728286743, + "loss/reference_anchor": 0.6513532400131226, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.953408718109131, + "rewards/margins": 2.6012117862701416, + "rewards/rejected": 1.3521969318389893, + "step": 435 + }, + { + "drift/chosen_abs": 0.21620866656303406, + "drift/rejected_abs": 0.12473054230213165, + "epoch": 0.4203989012301445, + "grad_norm": 1888.0, + "learning_rate": 1.51936608627315e-07, + "logits/chosen": -2.182278633117676, + "logits/rejected": -2.1586408615112305, + "logps/chosen": -0.2953268885612488, + "logps/rejected": -0.2857879102230072, + "loss": 4432.2978515625, + "loss/core": 4432.294921875, + "loss/preference_sft": 0.2953268885612488, + "loss/reference_anchor": 0.13938388228416443, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.161210060119629, + "rewards/margins": 0.9144028425216675, + "rewards/rejected": 1.2468074560165405, + "step": 440 + }, + { + "drift/chosen_abs": 0.30145135521888733, + "drift/rejected_abs": 0.1143740639090538, + "epoch": 0.42517616147139614, + "grad_norm": 3456.0, + "learning_rate": 1.4956457619160375e-07, + "logits/chosen": -2.11724853515625, + "logits/rejected": -2.1948139667510986, + "logps/chosen": -0.28283238410949707, + "logps/rejected": -0.27826812863349915, + "loss": 4419.79443359375, + "loss/core": 4419.7880859375, + "loss/preference_sft": 0.28283238410949707, + "loss/reference_anchor": 0.2919161915779114, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.012824296951294, + "rewards/margins": 1.8758718967437744, + "rewards/rejected": 1.1369524002075195, + "step": 445 + }, + { + "drift/chosen_abs": 0.2928563058376312, + "drift/rejected_abs": 0.1479603499174118, + "epoch": 0.4299534217126478, + "grad_norm": 15680.0, + "learning_rate": 1.471833060234044e-07, + "logits/chosen": -2.1485331058502197, + "logits/rejected": -2.2221622467041016, + "logps/chosen": -0.2605140507221222, + "logps/rejected": -0.27261584997177124, + "loss": 4425.9091796875, + "loss/core": 4425.8974609375, + "loss/preference_sft": 0.2605140507221222, + "loss/reference_anchor": 0.5446182489395142, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.9277219772338867, + "rewards/margins": 1.450683355331421, + "rewards/rejected": 1.4770386219024658, + "step": 450 + }, + { + "drift/chosen_abs": 0.17106688022613525, + "drift/rejected_abs": 0.08423759788274765, + "epoch": 0.4347306819538994, + "grad_norm": 552.0, + "learning_rate": 1.4479369362104037e-07, + "logits/chosen": -1.9175952672958374, + "logits/rejected": -1.9540889263153076, + "logps/chosen": -0.3066188097000122, + "logps/rejected": -0.30025577545166016, + "loss": 4432.8935546875, + "loss/core": 4432.89013671875, + "loss/preference_sft": 0.3066188097000122, + "loss/reference_anchor": 0.09201753884553909, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.7104876041412354, + "rewards/margins": 0.8717054128646851, + "rewards/rejected": 0.838782012462616, + "step": 455 + }, + { + "drift/chosen_abs": 0.22610139846801758, + "drift/rejected_abs": 0.1578260362148285, + "epoch": 0.4395079421951511, + "grad_norm": 1088.0, + "learning_rate": 1.4239663762000817e-07, + "logits/chosen": -2.0635833740234375, + "logits/rejected": -2.215299606323242, + "logps/chosen": -0.2846943736076355, + "logps/rejected": -0.2744455933570862, + "loss": 4435.564453125, + "loss/core": 4435.56005859375, + "loss/preference_sft": 0.2846943736076355, + "loss/reference_anchor": 0.2083575427532196, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.26090931892395, + "rewards/margins": 0.6843754053115845, + "rewards/rejected": 1.5765340328216553, + "step": 460 + }, + { + "drift/chosen_abs": 0.2833055555820465, + "drift/rejected_abs": 0.21140813827514648, + "epoch": 0.44428520243640274, + "grad_norm": 980.0, + "learning_rate": 1.3999303945503747e-07, + "logits/chosen": -2.1378350257873535, + "logits/rejected": -2.0690975189208984, + "logps/chosen": -0.27313750982284546, + "logps/rejected": -0.2805202007293701, + "loss": 4435.7646484375, + "loss/core": 4435.7529296875, + "loss/preference_sft": 0.27313750982284546, + "loss/reference_anchor": 0.5566142797470093, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.833055257797241, + "rewards/margins": 0.7193877100944519, + "rewards/rejected": 2.1136677265167236, + "step": 465 + }, + { + "drift/chosen_abs": 0.22237162292003632, + "drift/rejected_abs": 0.1690370738506317, + "epoch": 0.4490624626776544, + "grad_norm": 2528.0, + "learning_rate": 1.3758380302109808e-07, + "logits/chosen": -2.125699520111084, + "logits/rejected": -2.099730968475342, + "logps/chosen": -0.2701879143714905, + "logps/rejected": -0.28192803263664246, + "loss": 4437.58447265625, + "loss/core": 4437.5791015625, + "loss/preference_sft": 0.2701879143714905, + "loss/reference_anchor": 0.24922780692577362, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.223245859146118, + "rewards/margins": 0.5391066670417786, + "rewards/rejected": 1.6841392517089844, + "step": 470 + }, + { + "drift/chosen_abs": 0.23621515929698944, + "drift/rejected_abs": 0.16340488195419312, + "epoch": 0.453839722918906, + "grad_norm": 1424.0, + "learning_rate": 1.351698343334823e-07, + "logits/chosen": -2.234205961227417, + "logits/rejected": -2.2512259483337402, + "logps/chosen": -0.2768619954586029, + "logps/rejected": -0.2696452736854553, + "loss": 4434.79052734375, + "loss/core": 4434.787109375, + "loss/preference_sft": 0.2768619954586029, + "loss/reference_anchor": 0.17205794155597687, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.3621127605438232, + "rewards/margins": 0.7280641198158264, + "rewards/rejected": 1.6340488195419312, + "step": 475 + }, + { + "drift/chosen_abs": 0.530329704284668, + "drift/rejected_abs": 0.12030331045389175, + "epoch": 0.45861698316015764, + "grad_norm": 2736.0, + "learning_rate": 1.3275204118708942e-07, + "logits/chosen": -2.001190662384033, + "logits/rejected": -2.016195774078369, + "logps/chosen": -0.2734314799308777, + "logps/rejected": -0.27441081404685974, + "loss": 4392.0009765625, + "loss/core": 4391.97265625, + "loss/preference_sft": 0.2734314799308777, + "loss/reference_anchor": 1.3795697689056396, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 5.302812099456787, + "rewards/margins": 4.102025032043457, + "rewards/rejected": 1.2007869482040405, + "step": 480 + }, + { + "drift/chosen_abs": 0.2102138102054596, + "drift/rejected_abs": 0.11363495886325836, + "epoch": 0.4633942434014093, + "grad_norm": 2040.0, + "learning_rate": 1.3033133281504132e-07, + "logits/chosen": -2.2091434001922607, + "logits/rejected": -2.1927781105041504, + "logps/chosen": -0.2823958992958069, + "logps/rejected": -0.3023380637168884, + "loss": 4430.564453125, + "loss/core": 4430.5595703125, + "loss/preference_sft": 0.2823958992958069, + "loss/reference_anchor": 0.20998165011405945, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.102138042449951, + "rewards/margins": 1.0682824850082397, + "rewards/rejected": 1.033855676651001, + "step": 485 + }, + { + "drift/chosen_abs": 0.16725513339042664, + "drift/rejected_abs": 0.143391415476799, + "epoch": 0.4681715036426609, + "grad_norm": 6176.0, + "learning_rate": 1.2790861954675702e-07, + "logits/chosen": -2.198439598083496, + "logits/rejected": -2.145232677459717, + "logps/chosen": -0.2879284620285034, + "logps/rejected": -0.2975713610649109, + "loss": 4441.4736328125, + "loss/core": 4441.4697265625, + "loss/preference_sft": 0.2879284620285034, + "loss/reference_anchor": 0.16039147973060608, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.6691477298736572, + "rewards/margins": 0.2376086413860321, + "rewards/rejected": 1.4315390586853027, + "step": 490 + }, + { + "drift/chosen_abs": 0.3689550459384918, + "drift/rejected_abs": 0.17689982056617737, + "epoch": 0.47294876388391255, + "grad_norm": 12736.0, + "learning_rate": 1.2548481246561504e-07, + "logits/chosen": -2.104423761367798, + "logits/rejected": -2.1106228828430176, + "logps/chosen": -0.31167513132095337, + "logps/rejected": -0.30056482553482056, + "loss": 4420.04833984375, + "loss/core": 4420.02685546875, + "loss/preference_sft": 0.31167513132095337, + "loss/reference_anchor": 1.0426156520843506, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 3.6870017051696777, + "rewards/margins": 1.9218406677246094, + "rewards/rejected": 1.7651607990264893, + "step": 495 + }, + { + "drift/chosen_abs": 0.3468535840511322, + "drift/rejected_abs": 0.12408767640590668, + "epoch": 0.47772602412516424, + "grad_norm": 11392.0, + "learning_rate": 1.230608230663321e-07, + "logits/chosen": -2.1764144897460938, + "logits/rejected": -2.1404600143432617, + "logps/chosen": -0.28169992566108704, + "logps/rejected": -0.26774054765701294, + "loss": 4415.38818359375, + "loss/core": 4415.37548828125, + "loss/preference_sft": 0.28169992566108704, + "loss/reference_anchor": 0.5941214561462402, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.4685356616973877, + "rewards/margins": 2.2292885780334473, + "rewards/rejected": 1.2392469644546509, + "step": 500 + }, + { + "drift/chosen_abs": 0.32149267196655273, + "drift/rejected_abs": 0.13741762936115265, + "epoch": 0.4825032843664159, + "grad_norm": 1152.0, + "learning_rate": 1.206375629121874e-07, + "logits/chosen": -1.7544605731964111, + "logits/rejected": -1.77561354637146, + "logps/chosen": -0.30578163266181946, + "logps/rejected": -0.3066912889480591, + "loss": 4420.0146484375, + "loss/core": 4420.0068359375, + "loss/preference_sft": 0.30578163266181946, + "loss/reference_anchor": 0.36515378952026367, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.2140979766845703, + "rewards/margins": 1.852840781211853, + "rewards/rejected": 1.3612569570541382, + "step": 505 + }, + { + "drift/chosen_abs": 0.3091573417186737, + "drift/rejected_abs": 0.1424328237771988, + "epoch": 0.4872805446076675, + "grad_norm": 3552.0, + "learning_rate": 1.1821594329222079e-07, + "logits/chosen": -2.080624580383301, + "logits/rejected": -2.1759705543518066, + "logps/chosen": -0.2625710964202881, + "logps/rejected": -0.27721071243286133, + "loss": 4422.59716796875, + "loss/core": 4422.5908203125, + "loss/preference_sft": 0.2625710964202881, + "loss/reference_anchor": 0.3314869999885559, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.09027361869812, + "rewards/margins": 1.6671348810195923, + "rewards/rejected": 1.423138976097107, + "step": 510 + }, + { + "drift/chosen_abs": 0.22843725979328156, + "drift/rejected_abs": 0.13884204626083374, + "epoch": 0.49205780484891914, + "grad_norm": 912.0, + "learning_rate": 1.157968748785344e-07, + "logits/chosen": -1.9473901987075806, + "logits/rejected": -2.0520355701446533, + "logps/chosen": -0.2986038327217102, + "logps/rejected": -0.3280816972255707, + "loss": 4432.8408203125, + "loss/core": 4432.8349609375, + "loss/preference_sft": 0.2986038327217102, + "loss/reference_anchor": 0.27390003204345703, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.283982753753662, + "rewards/margins": 0.8998821973800659, + "rewards/rejected": 1.3841006755828857, + "step": 515 + }, + { + "drift/chosen_abs": 0.25142940878868103, + "drift/rejected_abs": 0.0938950628042221, + "epoch": 0.4968350650901708, + "grad_norm": 474.0, + "learning_rate": 1.1338126738382597e-07, + "logits/chosen": -2.0015084743499756, + "logits/rejected": -2.200745105743408, + "logps/chosen": -0.3057096600532532, + "logps/rejected": -0.29396018385887146, + "loss": 4423.95947265625, + "loss/core": 4423.9501953125, + "loss/preference_sft": 0.3057096600532532, + "loss/reference_anchor": 0.41009026765823364, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.513545513153076, + "rewards/margins": 1.5773802995681763, + "rewards/rejected": 0.9361652135848999, + "step": 520 + }, + { + "drift/chosen_abs": 0.3425474762916565, + "drift/rejected_abs": 0.14658579230308533, + "epoch": 0.5016123253314224, + "grad_norm": 328.0, + "learning_rate": 1.1097002921928316e-07, + "logits/chosen": -2.023900270462036, + "logits/rejected": -2.0547852516174316, + "logps/chosen": -0.2761421203613281, + "logps/rejected": -0.2680854797363281, + "loss": 4418.85009765625, + "loss/core": 4418.8408203125, + "loss/preference_sft": 0.2761421203613281, + "loss/reference_anchor": 0.4391368329524994, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.4254088401794434, + "rewards/margins": 1.96877920627594, + "rewards/rejected": 1.456629753112793, + "step": 525 + }, + { + "drift/chosen_abs": 0.1811850368976593, + "drift/rejected_abs": 0.08488638699054718, + "epoch": 0.5063895855726741, + "grad_norm": 684.0, + "learning_rate": 1.0856406715296717e-07, + "logits/chosen": -2.0869510173797607, + "logits/rejected": -2.206756830215454, + "logps/chosen": -0.29684141278266907, + "logps/rejected": -0.29523271322250366, + "loss": 4431.6162109375, + "loss/core": 4431.61376953125, + "loss/preference_sft": 0.29684141278266907, + "loss/reference_anchor": 0.08795829117298126, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.8088932037353516, + "rewards/margins": 0.9681621789932251, + "rewards/rejected": 0.8407310247421265, + "step": 530 + }, + { + "drift/chosen_abs": 0.3354306221008301, + "drift/rejected_abs": 0.177082821726799, + "epoch": 0.5111668458139257, + "grad_norm": 1024.0, + "learning_rate": 1.061642859688146e-07, + "logits/chosen": -2.060147762298584, + "logits/rejected": -2.065338373184204, + "logps/chosen": -0.2848697900772095, + "logps/rejected": -0.2815316915512085, + "loss": 4423.5537109375, + "loss/core": 4423.54052734375, + "loss/preference_sft": 0.2848697900772095, + "loss/reference_anchor": 0.5895854234695435, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.3528850078582764, + "rewards/margins": 1.5826117992401123, + "rewards/rejected": 1.770272970199585, + "step": 535 + }, + { + "drift/chosen_abs": 0.19800987839698792, + "drift/rejected_abs": 0.09570334106683731, + "epoch": 0.5159441060551774, + "grad_norm": 2848.0, + "learning_rate": 1.0377158812638491e-07, + "logits/chosen": -2.050477981567383, + "logits/rejected": -2.1051759719848633, + "logps/chosen": -0.29067665338516235, + "logps/rejected": -0.28211501240730286, + "loss": 4430.95361328125, + "loss/core": 4430.9501953125, + "loss/preference_sft": 0.29067665338516235, + "loss/reference_anchor": 0.14536850154399872, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.9791667461395264, + "rewards/margins": 1.0227887630462646, + "rewards/rejected": 0.9563776254653931, + "step": 540 + }, + { + "drift/chosen_abs": 0.37276729941368103, + "drift/rejected_abs": 0.08400270342826843, + "epoch": 0.520721366296429, + "grad_norm": 3744.0, + "learning_rate": 1.0138687342148249e-07, + "logits/chosen": -2.044623374938965, + "logits/rejected": -2.1050045490264893, + "logps/chosen": -0.28390389680862427, + "logps/rejected": -0.2720825672149658, + "loss": 4406.427734375, + "loss/core": 4406.41748046875, + "loss/preference_sft": 0.28390389680862427, + "loss/reference_anchor": 0.45570674538612366, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.7272789478302, + "rewards/margins": 2.9046859741210938, + "rewards/rejected": 0.8225927352905273, + "step": 545 + }, + { + "drift/chosen_abs": 0.2248731106519699, + "drift/rejected_abs": 0.11237964779138565, + "epoch": 0.5254986265376806, + "grad_norm": 1240.0, + "learning_rate": 9.90110386477801e-08, + "logits/chosen": -2.00352144241333, + "logits/rejected": -2.0287065505981445, + "logps/chosen": -0.28295984864234924, + "logps/rejected": -0.27802377939224243, + "loss": 4429.490234375, + "loss/core": 4429.48779296875, + "loss/preference_sft": 0.28295984864234924, + "loss/reference_anchor": 0.09104995429515839, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.246397018432617, + "rewards/margins": 1.1257827281951904, + "rewards/rejected": 1.1206141710281372, + "step": 550 + }, + { + "drift/chosen_abs": 0.3234367370605469, + "drift/rejected_abs": 0.15718965232372284, + "epoch": 0.5302758867789323, + "grad_norm": 1104.0, + "learning_rate": 9.664497725957164e-08, + "logits/chosen": -2.059795618057251, + "logits/rejected": -2.1331515312194824, + "logps/chosen": -0.31671643257141113, + "logps/rejected": -0.2889899015426636, + "loss": 4422.66943359375, + "loss/core": 4422.658203125, + "loss/preference_sft": 0.31671643257141113, + "loss/reference_anchor": 0.5569087266921997, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.2323079109191895, + "rewards/margins": 1.6611411571502686, + "rewards/rejected": 1.5711662769317627, + "step": 555 + }, + { + "drift/chosen_abs": 0.2606988251209259, + "drift/rejected_abs": 0.13210263848304749, + "epoch": 0.5350531470201839, + "grad_norm": 1032.0, + "learning_rate": 9.428957903578045e-08, + "logits/chosen": -1.871110200881958, + "logits/rejected": -1.8772865533828735, + "logps/chosen": -0.2984418272972107, + "logps/rejected": -0.2938275635242462, + "loss": 4427.8583984375, + "loss/core": 4427.85009765625, + "loss/preference_sft": 0.2984418272972107, + "loss/reference_anchor": 0.41203561425209045, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.606642007827759, + "rewards/margins": 1.286530613899231, + "rewards/rejected": 1.3201110363006592, + "step": 560 + }, + { + "drift/chosen_abs": 0.5179003477096558, + "drift/rejected_abs": 0.19176138937473297, + "epoch": 0.5398304072614356, + "grad_norm": 2688.0, + "learning_rate": 9.194572974534976e-08, + "logits/chosen": -1.9553916454315186, + "logits/rejected": -2.062887191772461, + "logps/chosen": -0.2773086130619049, + "logps/rejected": -0.2870504558086395, + "loss": 4402.384765625, + "loss/core": 4402.35546875, + "loss/preference_sft": 0.2773086130619049, + "loss/reference_anchor": 1.4593416452407837, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 5.178015232086182, + "rewards/margins": 3.264045000076294, + "rewards/rejected": 1.9139697551727295, + "step": 565 + }, + { + "drift/chosen_abs": 0.23666679859161377, + "drift/rejected_abs": 0.21036815643310547, + "epoch": 0.5446076675026872, + "grad_norm": 3984.0, + "learning_rate": 8.96143108141412e-08, + "logits/chosen": -2.098996639251709, + "logits/rejected": -2.055161952972412, + "logps/chosen": -0.31491559743881226, + "logps/rejected": -0.29813891649246216, + "loss": 4441.4765625, + "loss/core": 4441.46630859375, + "loss/preference_sft": 0.31491559743881226, + "loss/reference_anchor": 0.47641339898109436, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.3656527996063232, + "rewards/margins": 0.26370275020599365, + "rewards/rejected": 2.101950168609619, + "step": 570 + }, + { + "drift/chosen_abs": 0.24016256630420685, + "drift/rejected_abs": 0.10366284847259521, + "epoch": 0.5493849277439389, + "grad_norm": 1968.0, + "learning_rate": 8.72961989934668e-08, + "logits/chosen": -2.352053165435791, + "logits/rejected": -2.3277459144592285, + "logps/chosen": -0.28055113554000854, + "logps/rejected": -0.2906273305416107, + "loss": 4426.044921875, + "loss/core": 4426.0390625, + "loss/preference_sft": 0.28055113554000854, + "loss/reference_anchor": 0.2594422698020935, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.399277925491333, + "rewards/margins": 1.4035309553146362, + "rewards/rejected": 0.9957469701766968, + "step": 575 + }, + { + "drift/chosen_abs": 0.24877488613128662, + "drift/rejected_abs": 0.10169482231140137, + "epoch": 0.5541621879851905, + "grad_norm": 908.0, + "learning_rate": 8.499226603037854e-08, + "logits/chosen": -2.032081127166748, + "logits/rejected": -2.1548638343811035, + "logps/chosen": -0.30404120683670044, + "logps/rejected": -0.3054927885532379, + "loss": 4425.0419921875, + "loss/core": 4425.03759765625, + "loss/preference_sft": 0.30404120683670044, + "loss/reference_anchor": 0.2175559103488922, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.485950469970703, + "rewards/margins": 1.4695405960083008, + "rewards/rejected": 1.016409993171692, + "step": 580 + }, + { + "drift/chosen_abs": 0.26507624983787537, + "drift/rejected_abs": 0.19033315777778625, + "epoch": 0.5589394482264421, + "grad_norm": 2144.0, + "learning_rate": 8.270337833983987e-08, + "logits/chosen": -2.246429681777954, + "logits/rejected": -2.2616257667541504, + "logps/chosen": -0.27734872698783875, + "logps/rejected": -0.2799379229545593, + "loss": 4435.09521484375, + "loss/core": 4435.0849609375, + "loss/preference_sft": 0.27734872698783875, + "loss/reference_anchor": 0.45706719160079956, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.6497602462768555, + "rewards/margins": 0.7482293248176575, + "rewards/rejected": 1.9015309810638428, + "step": 585 + }, + { + "drift/chosen_abs": 0.27788597345352173, + "drift/rejected_abs": 0.17827074229717255, + "epoch": 0.5637167084676937, + "grad_norm": 3984.0, + "learning_rate": 8.04303966789025e-08, + "logits/chosen": -1.9625133275985718, + "logits/rejected": -1.9321056604385376, + "logps/chosen": -0.30193933844566345, + "logps/rejected": -0.29800906777381897, + "loss": 4431.5615234375, + "loss/core": 4431.5537109375, + "loss/preference_sft": 0.30193933844566345, + "loss/reference_anchor": 0.3806990385055542, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.774885892868042, + "rewards/margins": 0.9939848780632019, + "rewards/rejected": 1.7809009552001953, + "step": 590 + }, + { + "drift/chosen_abs": 0.1820889711380005, + "drift/rejected_abs": 0.23861515522003174, + "epoch": 0.5684939687089454, + "grad_norm": 728.0, + "learning_rate": 7.817417582301098e-08, + "logits/chosen": -2.0625860691070557, + "logits/rejected": -1.9918296337127686, + "logps/chosen": -0.27388039231300354, + "logps/rejected": -0.2751568555831909, + "loss": 4452.5673828125, + "loss/core": 4452.55859375, + "loss/preference_sft": 0.27388039231300354, + "loss/reference_anchor": 0.40798282623291016, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.8208898305892944, + "rewards/margins": -0.5628082752227783, + "rewards/rejected": 2.383697986602783, + "step": 595 + }, + { + "drift/chosen_abs": 0.17881783843040466, + "drift/rejected_abs": 0.11638796329498291, + "epoch": 0.5732712289501971, + "grad_norm": 2560.0, + "learning_rate": 7.59355642445566e-08, + "logits/chosen": -2.1608338356018066, + "logits/rejected": -2.074814558029175, + "logps/chosen": -0.2959844470024109, + "logps/rejected": -0.30641934275627136, + "loss": 4436.38525390625, + "loss/core": 4436.38037109375, + "loss/preference_sft": 0.2959844470024109, + "loss/reference_anchor": 0.1849345564842224, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.7870938777923584, + "rewards/margins": 0.6242033839225769, + "rewards/rejected": 1.1628906726837158, + "step": 600 + }, + { + "drift/chosen_abs": 0.19429127871990204, + "drift/rejected_abs": 0.06751377135515213, + "epoch": 0.5780484891914487, + "grad_norm": 1664.0, + "learning_rate": 7.37154037938015e-08, + "logits/chosen": -2.3095786571502686, + "logits/rejected": -2.3983094692230225, + "logps/chosen": -0.29155880212783813, + "logps/rejected": -0.3032078146934509, + "loss": 4426.8818359375, + "loss/core": 4426.87744140625, + "loss/preference_sft": 0.29155880212783813, + "loss/reference_anchor": 0.20911045372486115, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.9402952194213867, + "rewards/margins": 1.3450243473052979, + "rewards/rejected": 0.5952709913253784, + "step": 605 + }, + { + "drift/chosen_abs": 0.231297105550766, + "drift/rejected_abs": 0.15627989172935486, + "epoch": 0.5828257494327004, + "grad_norm": 700.0, + "learning_rate": 7.151452938229325e-08, + "logits/chosen": -2.0597474575042725, + "logits/rejected": -2.12882661819458, + "logps/chosen": -0.30136600136756897, + "logps/rejected": -0.310443252325058, + "loss": 4434.5126953125, + "loss/core": 4434.5048828125, + "loss/preference_sft": 0.30136600136756897, + "loss/reference_anchor": 0.3506472408771515, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.3125925064086914, + "rewards/margins": 0.750630259513855, + "rewards/rejected": 1.5619620084762573, + "step": 610 + }, + { + "drift/chosen_abs": 0.14156286418437958, + "drift/rejected_abs": 0.07604067772626877, + "epoch": 0.587603009673952, + "grad_norm": 394.0, + "learning_rate": 6.933376866888883e-08, + "logits/chosen": -2.1115915775299072, + "logits/rejected": -2.091139316558838, + "logps/chosen": -0.3194340765476227, + "logps/rejected": -0.3096611499786377, + "loss": 4435.76904296875, + "loss/core": 4435.7666015625, + "loss/preference_sft": 0.3194340765476227, + "loss/reference_anchor": 0.07679594308137894, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.4153246879577637, + "rewards/margins": 0.655189037322998, + "rewards/rejected": 0.7601357102394104, + "step": 615 + }, + { + "drift/chosen_abs": 0.1898563951253891, + "drift/rejected_abs": 0.13657966256141663, + "epoch": 0.5923802699152036, + "grad_norm": 908.0, + "learning_rate": 6.717394174850605e-08, + "logits/chosen": -2.2214183807373047, + "logits/rejected": -2.182659387588501, + "logps/chosen": -0.2612680494785309, + "logps/rejected": -0.27601683139801025, + "loss": 4437.4326171875, + "loss/core": 4437.42919921875, + "loss/preference_sft": 0.2612680494785309, + "loss/reference_anchor": 0.1394243687391281, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.8978593349456787, + "rewards/margins": 0.5342652797698975, + "rewards/rejected": 1.3635940551757812, + "step": 620 + }, + { + "drift/chosen_abs": 0.15214236080646515, + "drift/rejected_abs": 0.09313346445560455, + "epoch": 0.5971575301564552, + "grad_norm": 1688.0, + "learning_rate": 6.503586084371926e-08, + "logits/chosen": -2.0383543968200684, + "logits/rejected": -2.0106043815612793, + "logps/chosen": -0.29384806752204895, + "logps/rejected": -0.2968134880065918, + "loss": 4436.52734375, + "loss/core": 4436.52490234375, + "loss/preference_sft": 0.29384806752204895, + "loss/reference_anchor": 0.0648588091135025, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.5202014446258545, + "rewards/margins": 0.5949220061302185, + "rewards/rejected": 0.9252792596817017, + "step": 625 + }, + { + "drift/chosen_abs": 0.1255757361650467, + "drift/rejected_abs": 0.09434159100055695, + "epoch": 0.6019347903977069, + "grad_norm": 1464.0, + "learning_rate": 6.29203299993155e-08, + "logits/chosen": -2.274817705154419, + "logits/rejected": -2.197274684906006, + "logps/chosen": -0.2789786458015442, + "logps/rejected": -0.29018187522888184, + "loss": 4440.3251953125, + "loss/core": 4440.3232421875, + "loss/preference_sft": 0.2789786458015442, + "loss/reference_anchor": 0.05480783060193062, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.25403892993927, + "rewards/margins": 0.3123100697994232, + "rewards/rejected": 0.9417287707328796, + "step": 630 + }, + { + "drift/chosen_abs": 0.32034021615982056, + "drift/rejected_abs": 0.08672090619802475, + "epoch": 0.6067120506389586, + "grad_norm": 322.0, + "learning_rate": 6.082814477992656e-08, + "logits/chosen": -2.096656560897827, + "logits/rejected": -2.1229050159454346, + "logps/chosen": -0.30800944566726685, + "logps/rejected": -0.2923958897590637, + "loss": 4414.3359375, + "loss/core": 4414.32177734375, + "loss/preference_sft": 0.30800944566726685, + "loss/reference_anchor": 0.643897533416748, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.201916217803955, + "rewards/margins": 2.337477207183838, + "rewards/rejected": 0.8644390106201172, + "step": 635 + }, + { + "drift/chosen_abs": 0.24005039036273956, + "drift/rejected_abs": 0.06744923442602158, + "epoch": 0.6114893108802102, + "grad_norm": 4000.0, + "learning_rate": 5.87600919708494e-08, + "logits/chosen": -2.0245227813720703, + "logits/rejected": -2.164236545562744, + "logps/chosen": -0.2961650490760803, + "logps/rejected": -0.28276434540748596, + "loss": 4421.81494140625, + "loss/core": 4421.8095703125, + "loss/preference_sft": 0.2961650490760803, + "loss/reference_anchor": 0.24256105720996857, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.400282621383667, + "rewards/margins": 1.7267650365829468, + "rewards/rejected": 0.673517644405365, + "step": 640 + }, + { + "drift/chosen_abs": 0.4588896632194519, + "drift/rejected_abs": 0.15720048546791077, + "epoch": 0.6162665711214619, + "grad_norm": 1512.0, + "learning_rate": 5.671694928216829e-08, + "logits/chosen": -2.078514575958252, + "logits/rejected": -2.1803154945373535, + "logps/chosen": -0.2796784043312073, + "logps/rejected": -0.2615285813808441, + "loss": 4405.5849609375, + "loss/core": 4405.5576171875, + "loss/preference_sft": 0.2796784043312073, + "loss/reference_anchor": 1.3149816989898682, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.588896751403809, + "rewards/margins": 3.0177664756774902, + "rewards/rejected": 1.5711300373077393, + "step": 645 + }, + { + "drift/chosen_abs": 0.34622058272361755, + "drift/rejected_abs": 0.09166648238897324, + "epoch": 0.6210438313627135, + "grad_norm": 1000.0, + "learning_rate": 5.469948505629e-08, + "logits/chosen": -2.298170566558838, + "logits/rejected": -2.3131611347198486, + "logps/chosen": -0.29029572010040283, + "logps/rejected": -0.2863155007362366, + "loss": 4411.43017578125, + "loss/core": 4411.4130859375, + "loss/preference_sft": 0.29029572010040283, + "loss/reference_anchor": 0.8013036847114563, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.4620628356933594, + "rewards/margins": 2.561060905456543, + "rewards/rejected": 0.9010015726089478, + "step": 650 + }, + { + "drift/chosen_abs": 0.28553155064582825, + "drift/rejected_abs": 0.17389452457427979, + "epoch": 0.6258210916039652, + "grad_norm": 1688.0, + "learning_rate": 5.270845797900168e-08, + "logits/chosen": -1.9482574462890625, + "logits/rejected": -1.9404575824737549, + "logps/chosen": -0.2796974182128906, + "logps/rejected": -0.30062079429626465, + "loss": 4429.71923828125, + "loss/core": 4429.712890625, + "loss/preference_sft": 0.2796974182128906, + "loss/reference_anchor": 0.2853711247444153, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.8548009395599365, + "rewards/margins": 1.118624210357666, + "rewards/rejected": 1.73617684841156, + "step": 655 + }, + { + "drift/chosen_abs": 0.2055063545703888, + "drift/rejected_abs": 0.10912273079156876, + "epoch": 0.6305983518452167, + "grad_norm": 1080.0, + "learning_rate": 5.0744616794160104e-08, + "logits/chosen": -2.2815113067626953, + "logits/rejected": -2.3383548259735107, + "logps/chosen": -0.2732482850551605, + "logps/rejected": -0.2792763113975525, + "loss": 4431.6318359375, + "loss/core": 4431.62744140625, + "loss/preference_sft": 0.2732482850551605, + "loss/reference_anchor": 0.17782394587993622, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.055063486099243, + "rewards/margins": 0.9659796953201294, + "rewards/rejected": 1.0890839099884033, + "step": 660 + }, + { + "drift/chosen_abs": 0.1800767034292221, + "drift/rejected_abs": 0.12754087150096893, + "epoch": 0.6353756120864684, + "grad_norm": 1536.0, + "learning_rate": 4.880870002211963e-08, + "logits/chosen": -2.1525440216064453, + "logits/rejected": -2.227254867553711, + "logps/chosen": -0.29735860228538513, + "logps/rejected": -0.2968386113643646, + "loss": 4437.6201171875, + "loss/core": 4437.6162109375, + "loss/preference_sft": 0.29735860228538513, + "loss/reference_anchor": 0.21875838935375214, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.7989108562469482, + "rewards/margins": 0.5276275873184204, + "rewards/rejected": 1.2712833881378174, + "step": 665 + }, + { + "drift/chosen_abs": 0.2005433738231659, + "drift/rejected_abs": 0.12166965007781982, + "epoch": 0.64015287232772, + "grad_norm": 876.0, + "learning_rate": 4.6901435682004996e-08, + "logits/chosen": -2.293273448944092, + "logits/rejected": -2.3247194290161133, + "logps/chosen": -0.28415507078170776, + "logps/rejected": -0.2904112935066223, + "loss": 4434.0439453125, + "loss/core": 4434.03857421875, + "loss/preference_sft": 0.28415507078170776, + "loss/reference_anchor": 0.20630235970020294, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.002830982208252, + "rewards/margins": 0.7875508069992065, + "rewards/rejected": 1.2152799367904663, + "step": 670 + }, + { + "drift/chosen_abs": 0.19195790588855743, + "drift/rejected_abs": 0.10611511766910553, + "epoch": 0.6449301325689717, + "grad_norm": 976.0, + "learning_rate": 4.502354101793314e-08, + "logits/chosen": -2.1044974327087402, + "logits/rejected": -2.231261730194092, + "logps/chosen": -0.2956617772579193, + "logps/rejected": -0.28804105520248413, + "loss": 4433.052734375, + "loss/core": 4433.0478515625, + "loss/preference_sft": 0.2956617772579193, + "loss/reference_anchor": 0.15880243480205536, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.9193670749664307, + "rewards/margins": 0.8593287467956543, + "rewards/rejected": 1.0600383281707764, + "step": 675 + }, + { + "drift/chosen_abs": 0.12193242460489273, + "drift/rejected_abs": 0.14080289006233215, + "epoch": 0.6497073928102234, + "grad_norm": 4992.0, + "learning_rate": 4.3175722229287034e-08, + "logits/chosen": -2.1528353691101074, + "logits/rejected": -2.117715358734131, + "logps/chosen": -0.2846739888191223, + "logps/rejected": -0.27910977602005005, + "loss": 4447.14404296875, + "loss/core": 4447.1396484375, + "loss/preference_sft": 0.2846739888191223, + "loss/reference_anchor": 0.1636960357427597, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.2183479070663452, + "rewards/margins": -0.1849554032087326, + "rewards/rejected": 1.4033033847808838, + "step": 680 + }, + { + "drift/chosen_abs": 0.14501406252384186, + "drift/rejected_abs": 0.1529538929462433, + "epoch": 0.654484653051475, + "grad_norm": 1912.0, + "learning_rate": 4.135867420514276e-08, + "logits/chosen": -2.2104806900024414, + "logits/rejected": -2.2139463424682617, + "logps/chosen": -0.29691627621650696, + "logps/rejected": -0.29713374376296997, + "loss": 4445.89892578125, + "loss/core": 4445.89208984375, + "loss/preference_sft": 0.29691627621650696, + "loss/reference_anchor": 0.28842589259147644, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.4487031698226929, + "rewards/margins": -0.07711444050073624, + "rewards/rejected": 1.525817632675171, + "step": 685 + }, + { + "drift/chosen_abs": 0.3045670986175537, + "drift/rejected_abs": 0.17205330729484558, + "epoch": 0.6592619132927267, + "grad_norm": 1632.0, + "learning_rate": 3.957308026295035e-08, + "logits/chosen": -1.9819128513336182, + "logits/rejected": -1.9787757396697998, + "logps/chosen": -0.26376795768737793, + "logps/rejected": -0.27589425444602966, + "loss": 4426.86669921875, + "loss/core": 4426.86083984375, + "loss/preference_sft": 0.26376795768737793, + "loss/reference_anchor": 0.3071330487728119, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.0436840057373047, + "rewards/margins": 1.3387844562530518, + "rewards/rejected": 1.7048994302749634, + "step": 690 + }, + { + "drift/chosen_abs": 0.14530780911445618, + "drift/rejected_abs": 0.08450115472078323, + "epoch": 0.6640391735339782, + "grad_norm": 828.0, + "learning_rate": 3.7819611891566084e-08, + "logits/chosen": -2.2839925289154053, + "logits/rejected": -2.149257183074951, + "logps/chosen": -0.29438167810440063, + "logps/rejected": -0.3029247224330902, + "loss": 4435.900390625, + "loss/core": 4435.8994140625, + "loss/preference_sft": 0.29438167810440063, + "loss/reference_anchor": 0.06458897143602371, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.4513713121414185, + "rewards/margins": 0.645209550857544, + "rewards/rejected": 0.8061617612838745, + "step": 695 + }, + { + "drift/chosen_abs": 0.30958184599876404, + "drift/rejected_abs": 0.14133182168006897, + "epoch": 0.6688164337752299, + "grad_norm": 3184.0, + "learning_rate": 3.609892849873286e-08, + "logits/chosen": -2.1187186241149902, + "logits/rejected": -2.1548023223876953, + "logps/chosen": -0.28238362073898315, + "logps/rejected": -0.268256813287735, + "loss": 4422.44287109375, + "loss/core": 4422.43359375, + "loss/preference_sft": 0.28238362073898315, + "loss/reference_anchor": 0.4544747471809387, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.094094753265381, + "rewards/margins": 1.6822659969329834, + "rewards/rejected": 1.411828875541687, + "step": 700 + }, + { + "drift/chosen_abs": 0.1411522626876831, + "drift/rejected_abs": 0.12172625958919525, + "epoch": 0.6735936940164815, + "grad_norm": 5376.0, + "learning_rate": 3.4411677163103894e-08, + "logits/chosen": -2.031183958053589, + "logits/rejected": -2.114187479019165, + "logps/chosen": -0.27659520506858826, + "logps/rejected": -0.2737487256526947, + "loss": 4441.9150390625, + "loss/core": 4441.912109375, + "loss/preference_sft": 0.27659520506858826, + "loss/reference_anchor": 0.11837001889944077, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.4112656116485596, + "rewards/margins": 0.19559058547019958, + "rewards/rejected": 1.2156749963760376, + "step": 705 + }, + { + "drift/chosen_abs": 0.2575318515300751, + "drift/rejected_abs": 0.13676753640174866, + "epoch": 0.6783709542577332, + "grad_norm": 684.0, + "learning_rate": 3.2758492390902945e-08, + "logits/chosen": -2.029358148574829, + "logits/rejected": -2.0675930976867676, + "logps/chosen": -0.28309616446495056, + "logps/rejected": -0.28654059767723083, + "loss": 4428.4951171875, + "loss/core": 4428.49072265625, + "loss/preference_sft": 0.28309616446495056, + "loss/reference_anchor": 0.1890076845884323, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.574094533920288, + "rewards/margins": 1.2082717418670654, + "rewards/rejected": 1.3658227920532227, + "step": 710 + }, + { + "drift/chosen_abs": 0.2860047519207001, + "drift/rejected_abs": 0.1380338966846466, + "epoch": 0.6831482144989849, + "grad_norm": 508.0, + "learning_rate": 3.11399958773126e-08, + "logits/chosen": -2.116067886352539, + "logits/rejected": -2.1624484062194824, + "logps/chosen": -0.2840300500392914, + "logps/rejected": -0.29530325531959534, + "loss": 4424.92333984375, + "loss/core": 4424.91796875, + "loss/preference_sft": 0.2840300500392914, + "loss/reference_anchor": 0.24914495646953583, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.857480764389038, + "rewards/margins": 1.4812989234924316, + "rewards/rejected": 1.376182198524475, + "step": 715 + }, + { + "drift/chosen_abs": 0.22536201775074005, + "drift/rejected_abs": 0.10613832622766495, + "epoch": 0.6879254747402365, + "grad_norm": 632.0, + "learning_rate": 2.9556796272679972e-08, + "logits/chosen": -2.2809996604919434, + "logits/rejected": -2.2536635398864746, + "logps/chosen": -0.29658135771751404, + "logps/rejected": -0.29319730401039124, + "loss": 4428.8095703125, + "loss/core": 4428.8037109375, + "loss/preference_sft": 0.29658135771751404, + "loss/reference_anchor": 0.24946120381355286, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.252169132232666, + "rewards/margins": 1.1916897296905518, + "rewards/rejected": 1.0604794025421143, + "step": 720 + }, + { + "drift/chosen_abs": 0.11407455056905746, + "drift/rejected_abs": 0.07727280259132385, + "epoch": 0.6927027349814882, + "grad_norm": 840.0, + "learning_rate": 2.8009488953628215e-08, + "logits/chosen": -2.1286325454711914, + "logits/rejected": -2.1653316020965576, + "logps/chosen": -0.318909227848053, + "logps/rejected": -0.31315773725509644, + "loss": 4439.55126953125, + "loss/core": 4439.5498046875, + "loss/preference_sft": 0.318909227848053, + "loss/reference_anchor": 0.04532434046268463, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.1400318145751953, + "rewards/margins": 0.3686821460723877, + "rewards/rejected": 0.7713496685028076, + "step": 725 + }, + { + "drift/chosen_abs": 0.2567460536956787, + "drift/rejected_abs": 0.08764373511075974, + "epoch": 0.6974799952227397, + "grad_norm": 1560.0, + "learning_rate": 2.649865579915976e-08, + "logits/chosen": -2.2115912437438965, + "logits/rejected": -2.209638833999634, + "logps/chosen": -0.29488250613212585, + "logps/rejected": -0.2987883985042572, + "loss": 4422.2177734375, + "loss/core": 4422.21142578125, + "loss/preference_sft": 0.29488250613212585, + "loss/reference_anchor": 0.2926619052886963, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.567460775375366, + "rewards/margins": 1.693508505821228, + "rewards/rejected": 0.8739525079727173, + "step": 730 + }, + { + "drift/chosen_abs": 0.2040727585554123, + "drift/rejected_abs": 0.13325053453445435, + "epoch": 0.7022572554639914, + "grad_norm": 1256.0, + "learning_rate": 2.5024864971835507e-08, + "logits/chosen": -2.227663516998291, + "logits/rejected": -2.234522581100464, + "logps/chosen": -0.26899558305740356, + "logps/rejected": -0.28572389483451843, + "loss": 4435.130859375, + "loss/core": 4435.1279296875, + "loss/preference_sft": 0.26899558305740356, + "loss/reference_anchor": 0.1256342977285385, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.039360523223877, + "rewards/margins": 0.708215594291687, + "rewards/rejected": 1.3311448097229004, + "step": 735 + }, + { + "drift/chosen_abs": 0.2198764532804489, + "drift/rejected_abs": 0.1182849183678627, + "epoch": 0.707034515705243, + "grad_norm": 948.0, + "learning_rate": 2.3588670704111997e-08, + "logits/chosen": -2.0449001789093018, + "logits/rejected": -2.077310800552368, + "logps/chosen": -0.2672049403190613, + "logps/rejected": -0.28309932351112366, + "loss": 4430.99560546875, + "loss/core": 4430.9921875, + "loss/preference_sft": 0.2672049403190613, + "loss/reference_anchor": 0.17148524522781372, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.195984363555908, + "rewards/margins": 1.0152232646942139, + "rewards/rejected": 1.1807608604431152, + "step": 740 + }, + { + "drift/chosen_abs": 0.318195104598999, + "drift/rejected_abs": 0.21116037666797638, + "epoch": 0.7118117759464947, + "grad_norm": 2128.0, + "learning_rate": 2.219061308991721e-08, + "logits/chosen": -2.014070987701416, + "logits/rejected": -2.0407216548919678, + "logps/chosen": -0.28043967485427856, + "logps/rejected": -0.2645443081855774, + "loss": 4431.36669921875, + "loss/core": 4431.3525390625, + "loss/preference_sft": 0.28043967485427856, + "loss/reference_anchor": 0.6777635812759399, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.1789755821228027, + "rewards/margins": 1.0692497491836548, + "rewards/rejected": 2.1097254753112793, + "step": 745 + }, + { + "drift/chosen_abs": 0.25101321935653687, + "drift/rejected_abs": 0.15045872330665588, + "epoch": 0.7165890361877463, + "grad_norm": 912.0, + "learning_rate": 2.0831217881543557e-08, + "logits/chosen": -2.0724101066589355, + "logits/rejected": -2.101039409637451, + "logps/chosen": -0.25144457817077637, + "logps/rejected": -0.27629944682121277, + "loss": 4431.208984375, + "loss/core": 4431.20361328125, + "loss/preference_sft": 0.25144457817077637, + "loss/reference_anchor": 0.23283448815345764, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.5094053745269775, + "rewards/margins": 1.0123567581176758, + "rewards/rejected": 1.4970487356185913, + "step": 750 + }, + { + "drift/chosen_abs": 0.4007979929447174, + "drift/rejected_abs": 0.2181040346622467, + "epoch": 0.721366296428998, + "grad_norm": 412.0, + "learning_rate": 1.951099629193366e-08, + "logits/chosen": -2.212562084197998, + "logits/rejected": -2.208326816558838, + "logps/chosen": -0.36083754897117615, + "logps/rejected": -0.28320175409317017, + "loss": 4421.8203125, + "loss/core": 4421.7998046875, + "loss/preference_sft": 0.36083754897117615, + "loss/reference_anchor": 1.0116541385650635, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.007124423980713, + "rewards/margins": 1.8269456624984741, + "rewards/rejected": 2.180178642272949, + "step": 755 + }, + { + "drift/chosen_abs": 0.1610032469034195, + "drift/rejected_abs": 0.07328177988529205, + "epoch": 0.7261435566702497, + "grad_norm": 836.0, + "learning_rate": 1.823044480243431e-08, + "logits/chosen": -2.1159017086029053, + "logits/rejected": -2.1593313217163086, + "logps/chosen": -0.29487186670303345, + "logps/rejected": -0.29895296692848206, + "loss": 4432.8603515625, + "loss/core": 4432.85693359375, + "loss/preference_sft": 0.29487186670303345, + "loss/reference_anchor": 0.1201050877571106, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.608341932296753, + "rewards/margins": 0.8831087350845337, + "rewards/rejected": 0.7252334356307983, + "step": 760 + }, + { + "drift/chosen_abs": 0.14802543818950653, + "drift/rejected_abs": 0.09172812104225159, + "epoch": 0.7309208169115012, + "grad_norm": 968.0, + "learning_rate": 1.699004497608994e-08, + "logits/chosen": -2.2781732082366943, + "logits/rejected": -2.3036012649536133, + "logps/chosen": -0.27361828088760376, + "logps/rejected": -0.2817005515098572, + "loss": 4436.9404296875, + "loss/core": 4436.93896484375, + "loss/preference_sft": 0.27361828088760376, + "loss/reference_anchor": 0.04991341382265091, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.478417158126831, + "rewards/margins": 0.5637811422348022, + "rewards/rejected": 0.914635956287384, + "step": 765 + }, + { + "drift/chosen_abs": 0.2051977664232254, + "drift/rejected_abs": 0.10878431797027588, + "epoch": 0.7356980771527529, + "grad_norm": 2448.0, + "learning_rate": 1.5790263276546658e-08, + "logits/chosen": -2.144563674926758, + "logits/rejected": -2.1687493324279785, + "logps/chosen": -0.2964807152748108, + "logps/rejected": -0.28825658559799194, + "loss": 4431.6572265625, + "loss/core": 4431.6533203125, + "loss/preference_sft": 0.2964807152748108, + "loss/reference_anchor": 0.15945471823215485, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.049602508544922, + "rewards/margins": 0.9648269414901733, + "rewards/rejected": 1.0847753286361694, + "step": 770 + }, + { + "drift/chosen_abs": 0.20840251445770264, + "drift/rejected_abs": 0.09104669094085693, + "epoch": 0.7404753373940045, + "grad_norm": 2640.0, + "learning_rate": 1.4631550892634126e-08, + "logits/chosen": -2.2751145362854004, + "logits/rejected": -2.253349781036377, + "logps/chosen": -0.26738208532333374, + "logps/rejected": -0.26774922013282776, + "loss": 4428.873046875, + "loss/core": 4428.869140625, + "loss/preference_sft": 0.26738208532333374, + "loss/reference_anchor": 0.17535707354545593, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.0836570262908936, + "rewards/margins": 1.174851655960083, + "rewards/rejected": 0.908805251121521, + "step": 775 + }, + { + "drift/chosen_abs": 0.28770846128463745, + "drift/rejected_abs": 0.17277604341506958, + "epoch": 0.7452525976352562, + "grad_norm": 1576.0, + "learning_rate": 1.3514343568692132e-08, + "logits/chosen": -2.053027391433716, + "logits/rejected": -2.161233425140381, + "logps/chosen": -0.2801096737384796, + "logps/rejected": -0.2799223065376282, + "loss": 4429.0986328125, + "loss/core": 4429.0927734375, + "loss/preference_sft": 0.2801096737384796, + "loss/reference_anchor": 0.25047561526298523, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.874798536300659, + "rewards/margins": 1.1635240316390991, + "rewards/rejected": 1.71127450466156, + "step": 780 + }, + { + "drift/chosen_abs": 0.24193139374256134, + "drift/rejected_abs": 0.15544036030769348, + "epoch": 0.7500298578765078, + "grad_norm": 1832.0, + "learning_rate": 1.2439061440704724e-08, + "logits/chosen": -1.9472965002059937, + "logits/rejected": -1.9168212413787842, + "logps/chosen": -0.2860909104347229, + "logps/rejected": -0.2906906306743622, + "loss": 4433.3818359375, + "loss/core": 4433.375, + "loss/preference_sft": 0.2860909104347229, + "loss/reference_anchor": 0.28603917360305786, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.4173223972320557, + "rewards/margins": 0.8639556765556335, + "rewards/rejected": 1.5533664226531982, + "step": 785 + }, + { + "drift/chosen_abs": 0.17823731899261475, + "drift/rejected_abs": 0.08329329639673233, + "epoch": 0.7548071181177595, + "grad_norm": 764.0, + "learning_rate": 1.1406108878304468e-08, + "logits/chosen": -2.137251377105713, + "logits/rejected": -2.205977439880371, + "logps/chosen": -0.3069455027580261, + "logps/rejected": -0.3350505530834198, + "loss": 4431.853515625, + "loss/core": 4431.8515625, + "loss/preference_sft": 0.3069455027580261, + "loss/reference_anchor": 0.0917288064956665, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.7821671962738037, + "rewards/margins": 0.9510431289672852, + "rewards/rejected": 0.8311241269111633, + "step": 790 + }, + { + "drift/chosen_abs": 0.24515828490257263, + "drift/rejected_abs": 0.21105511486530304, + "epoch": 0.7595843783590112, + "grad_norm": 4512.0, + "learning_rate": 1.0415874332705381e-08, + "logits/chosen": -2.0945167541503906, + "logits/rejected": -2.070615291595459, + "logps/chosen": -0.2601197361946106, + "logps/rejected": -0.26042866706848145, + "loss": 4440.27392578125, + "loss/core": 4440.26611328125, + "loss/preference_sft": 0.2601197361946106, + "loss/reference_anchor": 0.35151952505111694, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.451155424118042, + "rewards/margins": 0.34183400869369507, + "rewards/rejected": 2.109321355819702, + "step": 795 + }, + { + "drift/chosen_abs": 0.35011768341064453, + "drift/rejected_abs": 0.09590541571378708, + "epoch": 0.7643616386002627, + "grad_norm": 880.0, + "learning_rate": 9.468730190622484e-09, + "logits/chosen": -2.0071511268615723, + "logits/rejected": -2.0719618797302246, + "logps/chosen": -0.31157320737838745, + "logps/rejected": -0.2820609211921692, + "loss": 4411.7822265625, + "loss/core": 4411.767578125, + "loss/preference_sft": 0.31157320737838745, + "loss/reference_anchor": 0.7128745317459106, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.5011768341064453, + "rewards/margins": 2.5428547859191895, + "rewards/rejected": 0.958321750164032, + "step": 800 + }, + { + "drift/chosen_abs": 0.20914137363433838, + "drift/rejected_abs": 0.09575755894184113, + "epoch": 0.7691388988415144, + "grad_norm": 1840.0, + "learning_rate": 8.565032634232194e-09, + "logits/chosen": -2.109005928039551, + "logits/rejected": -2.2536232471466064, + "logps/chosen": -0.2827692925930023, + "logps/rejected": -0.2778649926185608, + "loss": 4429.55029296875, + "loss/core": 4429.54638671875, + "loss/preference_sft": 0.2827692925930023, + "loss/reference_anchor": 0.17659203708171844, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.0893187522888184, + "rewards/margins": 1.1325154304504395, + "rewards/rejected": 0.9568031430244446, + "step": 805 + }, + { + "drift/chosen_abs": 0.2004438191652298, + "drift/rejected_abs": 0.08489207923412323, + "epoch": 0.773916159082766, + "grad_norm": 1696.0, + "learning_rate": 7.70512150722702e-09, + "logits/chosen": -2.398157835006714, + "logits/rejected": -2.384509563446045, + "logps/chosen": -0.27724185585975647, + "logps/rejected": -0.2638983428478241, + "loss": 4429.2548828125, + "loss/core": 4429.25, + "loss/preference_sft": 0.27724185585975647, + "loss/reference_anchor": 0.17924687266349792, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.0037176609039307, + "rewards/margins": 1.1569417715072632, + "rewards/rejected": 0.8467755317687988, + "step": 810 + }, + { + "drift/chosen_abs": 0.2000669687986374, + "drift/rejected_abs": 0.12345165014266968, + "epoch": 0.7786934193240177, + "grad_norm": 8064.0, + "learning_rate": 6.8893201870139915e-09, + "logits/chosen": -2.0791401863098145, + "logits/rejected": -2.0309667587280273, + "logps/chosen": -0.28644925355911255, + "logps/rejected": -0.30002540349960327, + "loss": 4434.1357421875, + "loss/core": 4434.1328125, + "loss/preference_sft": 0.28644925355911255, + "loss/reference_anchor": 0.10940603911876678, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.9967515468597412, + "rewards/margins": 0.7785899043083191, + "rewards/rejected": 1.2181618213653564, + "step": 815 + }, + { + "drift/chosen_abs": 0.2549762427806854, + "drift/rejected_abs": 0.12483543157577515, + "epoch": 0.7834706795652693, + "grad_norm": 1976.0, + "learning_rate": 6.117935463105808e-09, + "logits/chosen": -2.0269968509674072, + "logits/rejected": -2.0181238651275635, + "logps/chosen": -0.289524108171463, + "logps/rejected": -0.2884969115257263, + "loss": 4427.1259765625, + "loss/core": 4427.119140625, + "loss/preference_sft": 0.289524108171463, + "loss/reference_anchor": 0.3114100396633148, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.548508405685425, + "rewards/margins": 1.3080947399139404, + "rewards/rejected": 1.2404136657714844, + "step": 820 + }, + { + "drift/chosen_abs": 0.22770798206329346, + "drift/rejected_abs": 0.07756448537111282, + "epoch": 0.788247939806521, + "grad_norm": 540.0, + "learning_rate": 5.391257421749826e-09, + "logits/chosen": -2.086367130279541, + "logits/rejected": -2.2468793392181396, + "logps/chosen": -0.30201080441474915, + "logps/rejected": -0.342225044965744, + "loss": 4424.48291015625, + "loss/core": 4424.47705078125, + "loss/preference_sft": 0.30201080441474915, + "loss/reference_anchor": 0.25235939025878906, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.2739996910095215, + "rewards/margins": 1.5204788446426392, + "rewards/rejected": 0.7535209059715271, + "step": 825 + }, + { + "drift/chosen_abs": 0.18832604587078094, + "drift/rejected_abs": 0.07938660681247711, + "epoch": 0.7930252000477725, + "grad_norm": 1336.0, + "learning_rate": 4.709559336838462e-09, + "logits/chosen": -2.1965444087982178, + "logits/rejected": -2.2099156379699707, + "logps/chosen": -0.2985237240791321, + "logps/rejected": -0.2869217097759247, + "loss": 4430.0537109375, + "loss/core": 4430.05078125, + "loss/preference_sft": 0.2985237240791321, + "loss/reference_anchor": 0.11851098388433456, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8832508325576782, + "rewards/margins": 1.0893845558166504, + "rewards/rejected": 0.7938660979270935, + "step": 830 + }, + { + "drift/chosen_abs": 0.21169748902320862, + "drift/rejected_abs": 0.0859481543302536, + "epoch": 0.7978024602890242, + "grad_norm": 1048.0, + "learning_rate": 4.073097567142025e-09, + "logits/chosen": -2.2265546321868896, + "logits/rejected": -2.2355074882507324, + "logps/chosen": -0.253441721200943, + "logps/rejected": -0.27110329270362854, + "loss": 4427.9150390625, + "loss/core": 4427.91064453125, + "loss/preference_sft": 0.253441721200943, + "loss/reference_anchor": 0.21069765090942383, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.1147894859313965, + "rewards/margins": 1.2563145160675049, + "rewards/rejected": 0.8584749102592468, + "step": 835 + }, + { + "drift/chosen_abs": 0.3630954623222351, + "drift/rejected_abs": 0.11634461581707001, + "epoch": 0.8025797205302759, + "grad_norm": 6592.0, + "learning_rate": 3.482111459902695e-09, + "logits/chosen": -2.0539486408233643, + "logits/rejected": -2.109269857406616, + "logps/chosen": -0.2834726870059967, + "logps/rejected": -0.2910478711128235, + "loss": 4412.4912109375, + "loss/core": 4412.4755859375, + "loss/preference_sft": 0.2834726870059967, + "loss/reference_anchor": 0.7740469574928284, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.6292176246643066, + "rewards/margins": 2.487287998199463, + "rewards/rejected": 1.1419297456741333, + "step": 840 + }, + { + "drift/chosen_abs": 0.2685644328594208, + "drift/rejected_abs": 0.23484978079795837, + "epoch": 0.8073569807715275, + "grad_norm": 2464.0, + "learning_rate": 2.9368232608258797e-09, + "logits/chosen": -1.914307951927185, + "logits/rejected": -1.9539330005645752, + "logps/chosen": -0.3026157021522522, + "logps/rejected": -0.30641502141952515, + "loss": 4440.4267578125, + "loss/core": 4440.4130859375, + "loss/preference_sft": 0.3026157021522522, + "loss/reference_anchor": 0.6450716257095337, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.6856446266174316, + "rewards/margins": 0.3389059603214264, + "rewards/rejected": 2.346738576889038, + "step": 845 + }, + { + "drift/chosen_abs": 0.24007916450500488, + "drift/rejected_abs": 0.11487044394016266, + "epoch": 0.8121342410127792, + "grad_norm": 1616.0, + "learning_rate": 2.4374380305025866e-09, + "logits/chosen": -1.966447114944458, + "logits/rejected": -2.0533547401428223, + "logps/chosen": -0.286264568567276, + "logps/rejected": -0.2925958037376404, + "loss": 4427.82861328125, + "loss/core": 4427.8251953125, + "loss/preference_sft": 0.286264568567276, + "loss/reference_anchor": 0.16112756729125977, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.4002561569213867, + "rewards/margins": 1.252990961074829, + "rewards/rejected": 1.1472651958465576, + "step": 850 + }, + { + "drift/chosen_abs": 0.2156863957643509, + "drift/rejected_abs": 0.12507030367851257, + "epoch": 0.8169115012540308, + "grad_norm": 1456.0, + "learning_rate": 1.984143567294594e-09, + "logits/chosen": -1.9173269271850586, + "logits/rejected": -1.8677393198013306, + "logps/chosen": -0.2849237322807312, + "logps/rejected": -0.2810590863227844, + "loss": 4432.4453125, + "loss/core": 4432.4404296875, + "loss/preference_sft": 0.2849237322807312, + "loss/reference_anchor": 0.21630068123340607, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.156566619873047, + "rewards/margins": 0.9074234962463379, + "rewards/rejected": 1.2491432428359985, + "step": 855 + }, + { + "drift/chosen_abs": 0.19415462017059326, + "drift/rejected_abs": 0.12510396540164948, + "epoch": 0.8216887614952825, + "grad_norm": 916.0, + "learning_rate": 1.577110336711096e-09, + "logits/chosen": -2.001447916030884, + "logits/rejected": -2.0701100826263428, + "logps/chosen": -0.28479576110839844, + "logps/rejected": -0.3271367847919464, + "loss": 4435.3583984375, + "loss/core": 4435.35302734375, + "loss/preference_sft": 0.28479576110839844, + "loss/reference_anchor": 0.26654165983200073, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.9405349493026733, + "rewards/margins": 0.6905874013900757, + "rewards/rejected": 1.2499475479125977, + "step": 860 + }, + { + "drift/chosen_abs": 0.22797951102256775, + "drift/rejected_abs": 0.13979032635688782, + "epoch": 0.826466021736534, + "grad_norm": 4576.0, + "learning_rate": 1.2164914073037048e-09, + "logits/chosen": -1.9181400537490845, + "logits/rejected": -1.976419448852539, + "logps/chosen": -0.25987738370895386, + "logps/rejected": -0.28612470626831055, + "loss": 4431.91015625, + "loss/core": 4431.9052734375, + "loss/preference_sft": 0.25987738370895386, + "loss/reference_anchor": 0.20739074051380157, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.2794971466064453, + "rewards/margins": 0.950373649597168, + "rewards/rejected": 1.3291234970092773, + "step": 865 + }, + { + "drift/chosen_abs": 0.24134759604930878, + "drift/rejected_abs": 0.13137051463127136, + "epoch": 0.8312432819777857, + "grad_norm": 7136.0, + "learning_rate": 9.024223931035357e-10, + "logits/chosen": -2.3466553688049316, + "logits/rejected": -2.3163647651672363, + "logps/chosen": -0.3023620545864105, + "logps/rejected": -0.30314552783966064, + "loss": 4430.294921875, + "loss/core": 4430.28369140625, + "loss/preference_sft": 0.3023620545864105, + "loss/reference_anchor": 0.53348708152771, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.4105780124664307, + "rewards/margins": 1.0996638536453247, + "rewards/rejected": 1.3109138011932373, + "step": 870 + }, + { + "drift/chosen_abs": 0.169838547706604, + "drift/rejected_abs": 0.1980375051498413, + "epoch": 0.8360205422190374, + "grad_norm": 2000.0, + "learning_rate": 6.350214026223516e-10, + "logits/chosen": -2.20188570022583, + "logits/rejected": -2.0546789169311523, + "logps/chosen": -0.27584901452064514, + "logps/rejected": -0.2932647168636322, + "loss": 4448.4404296875, + "loss/core": 4448.4365234375, + "loss/preference_sft": 0.27584901452064514, + "loss/reference_anchor": 0.2003796100616455, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.6970628499984741, + "rewards/margins": -0.28227338194847107, + "rewards/rejected": 1.979336142539978, + "step": 875 + }, + { + "drift/chosen_abs": 0.2957570552825928, + "drift/rejected_abs": 0.15153485536575317, + "epoch": 0.840797802460289, + "grad_norm": 1232.0, + "learning_rate": 4.143889944367429e-10, + "logits/chosen": -2.06880521774292, + "logits/rejected": -2.0875306129455566, + "logps/chosen": -0.30057868361473083, + "logps/rejected": -0.2992440164089203, + "loss": 4425.47802734375, + "loss/core": 4425.4697265625, + "loss/preference_sft": 0.30057868361473083, + "loss/reference_anchor": 0.42805910110473633, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.956110954284668, + "rewards/margins": 1.4640635251998901, + "rewards/rejected": 1.4920471906661987, + "step": 880 + }, + { + "drift/chosen_abs": 0.11761297285556793, + "drift/rejected_abs": 0.054164666682481766, + "epoch": 0.8455750627015407, + "grad_norm": 876.0, + "learning_rate": 2.406081393722531e-10, + "logits/chosen": -2.1882359981536865, + "logits/rejected": -2.2277369499206543, + "logps/chosen": -0.30893659591674805, + "logps/rejected": -0.300253301858902, + "loss": 4435.9912109375, + "loss/core": 4435.990234375, + "loss/preference_sft": 0.30893659591674805, + "loss/reference_anchor": 0.027945518493652344, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.1728156805038452, + "rewards/margins": 0.6360281109809875, + "rewards/rejected": 0.5367876291275024, + "step": 885 + }, + { + "drift/chosen_abs": 0.20437291264533997, + "drift/rejected_abs": 0.07434792816638947, + "epoch": 0.8503523229427923, + "grad_norm": 4288.0, + "learning_rate": 1.1374418930135133e-10, + "logits/chosen": -2.072187662124634, + "logits/rejected": -2.139744997024536, + "logps/chosen": -0.33536940813064575, + "logps/rejected": -0.32796454429626465, + "loss": 4427.37646484375, + "loss/core": 4427.3720703125, + "loss/preference_sft": 0.33536940813064575, + "loss/reference_anchor": 0.1952354460954666, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.042240619659424, + "rewards/margins": 1.299561619758606, + "rewards/rejected": 0.7426789402961731, + "step": 890 + }, + { + "drift/chosen_abs": 0.2200593650341034, + "drift/rejected_abs": 0.10724915564060211, + "epoch": 0.855129583184044, + "grad_norm": 1336.0, + "learning_rate": 3.3844852567285756e-11, + "logits/chosen": -2.085082530975342, + "logits/rejected": -2.0716962814331055, + "logps/chosen": -0.28429746627807617, + "logps/rejected": -0.27482885122299194, + "loss": 4429.484375, + "loss/core": 4429.48046875, + "loss/preference_sft": 0.28429746627807617, + "loss/reference_anchor": 0.15885330736637115, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.200329542160034, + "rewards/margins": 1.1366140842437744, + "rewards/rejected": 1.0637153387069702, + "step": 895 + }, + { + "drift/chosen_abs": 0.4483451843261719, + "drift/rejected_abs": 0.1666850745677948, + "epoch": 0.8599068434252956, + "grad_norm": 2816.0, + "learning_rate": 9.401760429905703e-13, + "logits/chosen": -2.159799814224243, + "logits/rejected": -2.1224822998046875, + "logps/chosen": -0.3090743124485016, + "logps/rejected": -0.3085591197013855, + "loss": 4408.07568359375, + "loss/core": 4408.0546875, + "loss/preference_sft": 0.3090743124485016, + "loss/reference_anchor": 1.011836051940918, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.482909202575684, + "rewards/margins": 2.827746629714966, + "rewards/rejected": 1.6551628112792969, + "step": 900 + }, + { + "epoch": 0.8599068434252956, + "step": 900, + "total_flos": 0.0, + "train_loss": 4429.094605034722, + "train_runtime": 7055.131, + "train_samples_per_second": 2.041, + "train_steps_per_second": 0.128 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..e0e02aa --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:463d41d824cd1165d2f2fdbc8b546d0b94ce4a7d8722d2682ddfda3db4a8f657 +size 6993