From 11536274bc40ca559d2cc79f046dece962ab592e Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Thu, 23 Jul 2026 12:30:10 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: promotion/qwen3-8b-aaai27-flagship-inpo-avg-s44 Source: Original Platform --- .gitattributes | 36 + README.md | 24 + all_results.json | 9 + chat_template.jinja | 89 + config.json | 71 + config.yaml | 58 + generation_config.json | 12 + job_status.json | 23 + model.safetensors | 3 + objective_protocol.json | 69 + provenance.json | 14 + resource_profile.json | 21 + run_status.json | 15 + split_manifest.json | 23 + stability_gate.json | 38 + tokenizer.json | 3 + tokenizer_config.json | 30 + train_results.json | 9 + trainer_state.json | 3643 +++++++++++++++++++++++++++++++++++++++ training_args.bin | 3 + 20 files changed, 4193 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 config.yaml create mode 100644 generation_config.json create mode 100644 job_status.json create mode 100644 model.safetensors create mode 100644 objective_protocol.json create mode 100644 provenance.json create mode 100644 resource_profile.json create mode 100644 run_status.json create mode 100644 split_manifest.json create mode 100644 stability_gate.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..55ab685 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +--- +library_name: transformers +base_model: Qwen/Qwen3-8B +tags: +- ronpo +- mnpo +- preference-optimization +--- + +# qwen3-8b-aaai27-flagship-inpo-avg-s44 + +Research checkpoint for the RONPO AAAI revision experiments. + +- Method: inpo_avg +- Base model: `Qwen/Qwen3-8B` +- Seed: 44 +- Local source at upload time: `/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/full/inpo_avg/seed44/attempt2` +- Uploaded at UTC: 2026-07-13T13:54:35Z +- Run status metadata: `{"attempt": 2, "effective_batch_size": 16, "method": "inpo_avg", "objective_protocol": "objective_protocol.json", "optimizer_steps": 900, "resource_profile": "resource_profile.json", "scope_amendment": "scope_amendment_20260713.json", "seed": 44, "split_manifest": "split_manifest.json", "stability_gate": "stability_gate.json", "status": "completed", "wandb_run_id": "a7c32666324c", "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/a7c32666324c"}` + +AAAI-27 P1 matched budget: 900 optimizer steps, effective batch 16; passed non-thinking and collapse stability gates; sealed-test metrics are not used for checkpoint selection. + +Intended use: reproducibility and evaluation for the RONPO paper. This +checkpoint is not intended as a production assistant. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..aab52c8 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 4429.236100260417, + "train_runtime": 7053.367, + "train_samples": 16746, + "train_samples_per_second": 2.042, + "train_steps_per_second": 0.128 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..ebf7589 --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +model_name_or_path: Qwen/Qwen3-8B +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/avg: 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: inpo +eta: 0.0075 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.02 +preference_sft_weight: 0.002 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 2.5e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 44 +gradient_accumulation_steps: 16 +gradient_checkpointing: true +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 1 +per_device_eval_batch_size: 1 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/full/inpo_avg/seed44/attempt2 +run_name: aaai27-flagship-full-inpo_avg-s44-a2 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/job_status.json b/job_status.json new file mode 100644 index 0000000..d2aed7c --- /dev/null +++ b/job_status.json @@ -0,0 +1,23 @@ +{ + "status": "completed", + "stage": "full", + "method": "inpo_avg", + "seed": 44, + "attempt": 2, + "gpu": 1, + "gpus": [ + 1 + ], + "returncode": 0, + "finite_metrics": true, + "oom": false, + "traceback": false, + "reason": "passed", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "a7c32666324c", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/a7c32666324c", + "output_dir": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/full/inpo_avg/seed44/attempt2", + "log": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/logs/full_inpo_avg_s44_a2.log", + "completed_at": "2026-07-13T13:45:56Z" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..25d5dda --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53865855e167880535428d71c53bddf6ae0f4ece617dbe994cada06e10bd7486 +size 16381517208 diff --git a/objective_protocol.json b/objective_protocol.json new file mode 100644 index 0000000..35b3b44 --- /dev/null +++ b/objective_protocol.json @@ -0,0 +1,69 @@ +{ + "schema_version": 1, + "frozen_at_kst": "2026-07-12T21:40:00+09:00", + "base_model": "Qwen/Qwen3-8B", + "primary_triple": { + "helpfulness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "ultrafeedback-helpfulness", + "transform": "identity" + }, + "safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "conciseness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "helpsteer-verbosity", + "transform": "negate" + } + }, + "gate_thresholds": { + "raw_score_std_strictly_greater_than": 0.01, + "aggregate_prompt_pair_spearman_median_strictly_less_than": 0.0, + "aggregate_pairwise_top1_mismatch_at_least": 0.5 + }, + "gate_pool": "source-train Qwen3-8B response pool split into deterministic train/validation; sealed source-test excluded", + "training_seeds": [42, 43, 44], + "optimizer_steps": 900, + "effective_batch_size": 16, + "decode": { + "enable_thinking": false, + "temperature": 0.7, + "top_p": 0.9, + "max_new_tokens": 2048, + "decode_seed": 42 + }, + "methods": [ + "ronpo_full_expect", + "ronpo_k_only", + "dpo", + "ipo", + "simpo", + "kto", + "sppo_avg", + "inpo_avg", + "ht_mnpo_helpfulness", + "ht_mnpo_safety", + "ht_mnpo_conciseness" + ], + "selection_policy": "No sealed-test selection. One preregistered configuration per method; failures use only the declared stability retry recipe, at most three attempts.", + "stability_gate": { + "validation_prompts": 128, + "min_candidate_base_mean_word_ratio": 0.33, + "max_candidate_base_mean_word_ratio": 2.0, + "max_consecutive_identical_word_run": 20, + "think_tag_count": 0, + "empty_response_count": 0, + "retry_recipe": "attempt 2 halves learning rate, enables gradient checkpointing, and adds reference=0.02/SFT=0.002 anchoring to unanchored unified losses; attempt 3 quarters learning rate and uses reference=0.05/SFT=0.005; RONPO uses reference=0.05/SFT=0.005 from attempt 1" + }, + "primary_metric": "mean prompt-level minimum of per-prompt min-max-normalized helpfulness, safety, conciseness", + "bootstrap": { + "unit": "prompt", + "paired": true, + "resamples": 2000, + "interval": "percentile_95", + "seed": 42 + } +} diff --git a/provenance.json b/provenance.json new file mode 100644 index 0000000..75fc5b3 --- /dev/null +++ b/provenance.json @@ -0,0 +1,14 @@ +{ + "method": "inpo_avg", + "seed": 44, + "attempt": 2, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "a7c32666324c", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/a7c32666324c", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json", + "scope_amendment": "scope_amendment_20260713.json" +} diff --git a/resource_profile.json b/resource_profile.json new file mode 100644 index 0000000..18eec47 --- /dev/null +++ b/resource_profile.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "measured_at_kst": "2026-07-12T22:38:00+09:00", + "hardware": "NVIDIA B200 183359 MiB reported total; 178.35 GiB PyTorch-visible capacity", + "smoke_evidence": { + "attempted_profile": {"per_device_batch_size": 2, "gradient_accumulation_steps": 8, "gradient_checkpointing": false, "effective_batch_size": 16}, + "outcome": "OOM before completing the 20-step smoke across every launched unified baseline; representative process used 176.87 GiB and failed a 1.62 GiB allocation. KTO also OOMed at the optimizer step.", + "paper_metric": false + }, + "selected_full_training_profile": { + "per_device_batch_size": 2, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": false, + "attention_implementation": "PyTorch SDPA with cuDNN SDPA disabled", + "effective_batch_size": 16, + "applies_symmetrically_to_all_unified_pairwise_methods": true, + "kto_implementation_constraint": "TRL KTO requires actual batch size > 1. KTO uses batch 2 / accumulation 8 and identical AdamW with foreach=False. PyTorch SDPA is used with cuDNN SDPA disabled. The fixed base reference model is placed on a second B200 and its exact logits are returned to the policy device; the policy/optimizer remain on the first B200. This avoids a measured ~6-hour reference-precompute path while leaving the KTO loss, actual/effective batch, optimizer, and 900-step budget unchanged." + }, + "retry_semantics": "Resource selection is separated from method-stability attempts. Full attempt 1 uses the smoke-validated SDPA batch-2 profile and retains the preregistered method LR/anchor; attempts 2 and 3 fall back to batch 1 / accumulation 16 / gradient checkpointing and apply the declared LR/anchor stabilization recipe.", + "provenance_note": "No reward or sealed-evaluation result was available when this hardware profile was selected." +} diff --git a/run_status.json b/run_status.json new file mode 100644 index 0000000..9ff9008 --- /dev/null +++ b/run_status.json @@ -0,0 +1,15 @@ +{ + "method": "inpo_avg", + "seed": 44, + "attempt": 2, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "a7c32666324c", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/a7c32666324c", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json", + "scope_amendment": "scope_amendment_20260713.json", + "status": "completed" +} diff --git a/split_manifest.json b/split_manifest.json new file mode 100644 index 0000000..4fb96bd --- /dev/null +++ b/split_manifest.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "split_rule": "sort source-train prompts by sha256('flagship-v1||'+prompt_id); first round(10%) validation", + "validation_fraction": 0.1, + "source_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/avg_oracle", + "base_model": "Qwen/Qwen3-8B", + "counts": { + "train": 16780, + "validation": 1864, + "sealed_test": 604 + }, + "prompt_id_sha256": { + "train": "fcf6f2ea5a0b7bcd41179e6f5288f4d850b2d04bdb729da91907bd18bf733a69", + "validation": "a87ba6f519192164daa487b109a6547dd8cbde46a27a373d6da201ca2d061b8f", + "sealed_test": "b398ced6af6cf73d731e1a5f32a0631d8060323d51eda1e375695566bf5f1867" + }, + "overlaps": { + "train_validation": 0, + "train_sealed_test": 0, + "validation_sealed_test": 0 + }, + "sealed_test_policy": "prompts only until S4a; no objective scores or model generations used in S0-S3" +} diff --git a/stability_gate.json b/stability_gate.json new file mode 100644 index 0000000..52a41bb --- /dev/null +++ b/stability_gate.json @@ -0,0 +1,38 @@ +{ + "status": "passed", + "passed": true, + "base": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 388.1484375, + "max_words": 1379, + "max_repeat_run": 4 + }, + "candidate": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 393.484375, + "max_words": 1249, + "max_repeat_run": 3 + }, + "candidate_base_mean_word_ratio": 1.013747156975223, + "thresholds": { + "min_length_ratio": 0.33, + "max_length_ratio": 2.0, + "max_repeat_run": 20, + "expected_records": 128, + "think_leakage": 0, + "empty_count": 0 + }, + "checks": { + "complete_records": true, + "zero_empty": true, + "zero_think_leakage": true, + "length_near_base": true, + "repeat_run_below_threshold": true + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ea3c9e2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..aab52c8 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 4429.236100260417, + "train_runtime": 7053.367, + "train_samples": 16746, + "train_samples_per_second": 2.042, + "train_steps_per_second": 0.128 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..c3a16a8 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,3643 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8599068434252956, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "drift/chosen_abs": 0.3419838547706604, + "drift/rejected_abs": 0.14085355401039124, + "epoch": 0.004777260241251642, + "grad_norm": 7168.0, + "learning_rate": 1.111111111111111e-08, + "logits/chosen": -2.274840831756592, + "logits/rejected": -2.238874912261963, + "logps/chosen": -0.30753669142723083, + "logps/rejected": -0.3066186010837555, + "loss": 4418.6552734375, + "loss/core": 4418.6337890625, + "loss/preference_sft": 0.30753669142723083, + "loss/reference_anchor": 1.068562388420105, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.4167866706848145, + "rewards/margins": 2.012580156326294, + "rewards/rejected": 1.4042068719863892, + "step": 5 + }, + { + "drift/chosen_abs": 0.29194900393486023, + "drift/rejected_abs": 0.13348758220672607, + "epoch": 0.009554520482503284, + "grad_norm": 1528.0, + "learning_rate": 2.5e-08, + "logits/chosen": -2.2050859928131104, + "logits/rejected": -2.1969170570373535, + "logps/chosen": -0.29071494936943054, + "logps/rejected": -0.2801753282546997, + "loss": 4423.58544921875, + "loss/core": 4423.5791015625, + "loss/preference_sft": 0.29071494936943054, + "loss/reference_anchor": 0.28804466128349304, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.919328212738037, + "rewards/margins": 1.5846813917160034, + "rewards/rejected": 1.3346469402313232, + "step": 10 + }, + { + "drift/chosen_abs": 0.1922224462032318, + "drift/rejected_abs": 0.08871154487133026, + "epoch": 0.014331780723754926, + "grad_norm": 700.0, + "learning_rate": 3.888888888888889e-08, + "logits/chosen": -1.8447561264038086, + "logits/rejected": -1.9001595973968506, + "logps/chosen": -0.29011744260787964, + "logps/rejected": -0.28691497445106506, + "loss": 4430.5283203125, + "loss/core": 4430.525390625, + "loss/preference_sft": 0.29011744260787964, + "loss/reference_anchor": 0.11491169780492783, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.9212669134140015, + "rewards/margins": 1.0505340099334717, + "rewards/rejected": 0.8707327842712402, + "step": 15 + }, + { + "drift/chosen_abs": 0.16317619383335114, + "drift/rejected_abs": 0.05642751604318619, + "epoch": 0.01910904096500657, + "grad_norm": 676.0, + "learning_rate": 5.2777777777777776e-08, + "logits/chosen": -2.1695523262023926, + "logits/rejected": -2.097202777862549, + "logps/chosen": -0.2871994376182556, + "logps/rejected": -0.28375035524368286, + "loss": 4430.1630859375, + "loss/core": 4430.16064453125, + "loss/preference_sft": 0.2871994376182556, + "loss/reference_anchor": 0.07742591947317123, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.6311792135238647, + "rewards/margins": 1.0771574974060059, + "rewards/rejected": 0.5540218353271484, + "step": 20 + }, + { + "drift/chosen_abs": 0.18135179579257965, + "drift/rejected_abs": 0.11895482242107391, + "epoch": 0.02388630120625821, + "grad_norm": 1336.0, + "learning_rate": 6.666666666666667e-08, + "logits/chosen": -2.2033162117004395, + "logits/rejected": -2.218423366546631, + "logps/chosen": -0.27016180753707886, + "logps/rejected": -0.2789779007434845, + "loss": 4436.3212890625, + "loss/core": 4436.31787109375, + "loss/preference_sft": 0.27016180753707886, + "loss/reference_anchor": 0.13152866065502167, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.8121395111083984, + "rewards/margins": 0.6226418018341064, + "rewards/rejected": 1.1894975900650024, + "step": 25 + }, + { + "drift/chosen_abs": 0.26979273557662964, + "drift/rejected_abs": 0.18020370602607727, + "epoch": 0.028663561447509853, + "grad_norm": 3728.0, + "learning_rate": 8.055555555555555e-08, + "logits/chosen": -2.1017913818359375, + "logits/rejected": -2.063615083694458, + "logps/chosen": -0.3164452612400055, + "logps/rejected": -0.2725190818309784, + "loss": 4433.0703125, + "loss/core": 4433.05859375, + "loss/preference_sft": 0.3164452612400055, + "loss/reference_anchor": 0.557770848274231, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.6958069801330566, + "rewards/margins": 0.8953614234924316, + "rewards/rejected": 1.800445318222046, + "step": 30 + }, + { + "drift/chosen_abs": 0.18841537833213806, + "drift/rejected_abs": 0.08009415119886398, + "epoch": 0.033440821688761495, + "grad_norm": 912.0, + "learning_rate": 9.444444444444444e-08, + "logits/chosen": -2.1271777153015137, + "logits/rejected": -2.22860050201416, + "logps/chosen": -0.2820509970188141, + "logps/rejected": -0.2648252844810486, + "loss": 4429.8935546875, + "loss/core": 4429.8916015625, + "loss/preference_sft": 0.2820509970188141, + "loss/reference_anchor": 0.08949244022369385, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.883462905883789, + "rewards/margins": 1.0993850231170654, + "rewards/rejected": 0.7840781211853027, + "step": 35 + }, + { + "drift/chosen_abs": 0.2343977689743042, + "drift/rejected_abs": 0.09806372225284576, + "epoch": 0.03821808193001314, + "grad_norm": 1048.0, + "learning_rate": 1.0833333333333334e-07, + "logits/chosen": -2.1154208183288574, + "logits/rejected": -2.1933438777923584, + "logps/chosen": -0.29859572649002075, + "logps/rejected": -0.29969868063926697, + "loss": 4426.380859375, + "loss/core": 4426.3779296875, + "loss/preference_sft": 0.29859572649002075, + "loss/reference_anchor": 0.15946529805660248, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.3425917625427246, + "rewards/margins": 1.366955280303955, + "rewards/rejected": 0.9756364822387695, + "step": 40 + }, + { + "drift/chosen_abs": 0.28305283188819885, + "drift/rejected_abs": 0.1835290640592575, + "epoch": 0.04299534217126478, + "grad_norm": 4864.0, + "learning_rate": 1.2222222222222222e-07, + "logits/chosen": -2.038555383682251, + "logits/rejected": -2.0794358253479004, + "logps/chosen": -0.28872132301330566, + "logps/rejected": -0.29276594519615173, + "loss": 4432.30322265625, + "loss/core": 4432.2890625, + "loss/preference_sft": 0.28872132301330566, + "loss/reference_anchor": 0.6807764172554016, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.8277623653411865, + "rewards/margins": 0.9989973306655884, + "rewards/rejected": 1.8287651538848877, + "step": 45 + }, + { + "drift/chosen_abs": 0.18194088339805603, + "drift/rejected_abs": 0.13804134726524353, + "epoch": 0.04777260241251642, + "grad_norm": 430.0, + "learning_rate": 1.3611111111111108e-07, + "logits/chosen": -2.107499122619629, + "logits/rejected": -2.1948068141937256, + "logps/chosen": -0.28583410382270813, + "logps/rejected": -0.289579838514328, + "loss": 4438.6875, + "loss/core": 4438.68359375, + "loss/preference_sft": 0.28583410382270813, + "loss/reference_anchor": 0.17273597419261932, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.818098783493042, + "rewards/margins": 0.4417288303375244, + "rewards/rejected": 1.3763700723648071, + "step": 50 + }, + { + "drift/chosen_abs": 0.16760578751564026, + "drift/rejected_abs": 0.09244702756404877, + "epoch": 0.05254986265376806, + "grad_norm": 660.0, + "learning_rate": 1.5e-07, + "logits/chosen": -2.1269633769989014, + "logits/rejected": -2.2465131282806396, + "logps/chosen": -0.2968710958957672, + "logps/rejected": -0.29279255867004395, + "loss": 4434.4453125, + "loss/core": 4434.4443359375, + "loss/preference_sft": 0.2968710958957672, + "loss/reference_anchor": 0.05782536417245865, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.6760451793670654, + "rewards/margins": 0.7522833347320557, + "rewards/rejected": 0.9237619638442993, + "step": 55 + }, + { + "drift/chosen_abs": 0.14607155323028564, + "drift/rejected_abs": 0.10737241804599762, + "epoch": 0.057327122895019705, + "grad_norm": 2272.0, + "learning_rate": 1.6388888888888888e-07, + "logits/chosen": -2.11145281791687, + "logits/rejected": -2.1462674140930176, + "logps/chosen": -0.29008573293685913, + "logps/rejected": -0.2873132824897766, + "loss": 4439.5068359375, + "loss/core": 4439.5029296875, + "loss/preference_sft": 0.29008573293685913, + "loss/reference_anchor": 0.1456277072429657, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.459886908531189, + "rewards/margins": 0.387167751789093, + "rewards/rejected": 1.0727192163467407, + "step": 60 + }, + { + "drift/chosen_abs": 0.24463272094726562, + "drift/rejected_abs": 0.09055524319410324, + "epoch": 0.06210438313627135, + "grad_norm": 2912.0, + "learning_rate": 1.7777777777777776e-07, + "logits/chosen": -2.143834114074707, + "logits/rejected": -2.2185416221618652, + "logps/chosen": -0.3164866268634796, + "logps/rejected": -0.3099859356880188, + "loss": 4424.44091796875, + "loss/core": 4424.4326171875, + "loss/preference_sft": 0.3164866268634796, + "loss/reference_anchor": 0.37436532974243164, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.4456965923309326, + "rewards/margins": 1.5431140661239624, + "rewards/rejected": 0.9025824666023254, + "step": 65 + }, + { + "drift/chosen_abs": 0.23293833434581757, + "drift/rejected_abs": 0.10600552707910538, + "epoch": 0.06688164337752299, + "grad_norm": 8192.0, + "learning_rate": 1.9166666666666668e-07, + "logits/chosen": -2.1075563430786133, + "logits/rejected": -2.165278434753418, + "logps/chosen": -0.3008618950843811, + "logps/rejected": -0.30584263801574707, + "loss": 4426.07080078125, + "loss/core": 4426.06494140625, + "loss/preference_sft": 0.3008618950843811, + "loss/reference_anchor": 0.24336305260658264, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.329383373260498, + "rewards/margins": 1.4130064249038696, + "rewards/rejected": 0.9163768887519836, + "step": 70 + }, + { + "drift/chosen_abs": 0.1538292020559311, + "drift/rejected_abs": 0.10185450315475464, + "epoch": 0.07165890361877464, + "grad_norm": 804.0, + "learning_rate": 2.0555555555555553e-07, + "logits/chosen": -2.0664281845092773, + "logits/rejected": -2.1418495178222656, + "logps/chosen": -0.2909933924674988, + "logps/rejected": -0.2969188988208771, + "loss": 4437.65087890625, + "loss/core": 4437.6474609375, + "loss/preference_sft": 0.2909933924674988, + "loss/reference_anchor": 0.10792021453380585, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.5364402532577515, + "rewards/margins": 0.5191212296485901, + "rewards/rejected": 1.0173190832138062, + "step": 75 + }, + { + "drift/chosen_abs": 0.12916477024555206, + "drift/rejected_abs": 0.07108369469642639, + "epoch": 0.07643616386002627, + "grad_norm": 892.0, + "learning_rate": 2.1944444444444442e-07, + "logits/chosen": -2.2707951068878174, + "logits/rejected": -2.3466694355010986, + "logps/chosen": -0.30908656120300293, + "logps/rejected": -0.30672696232795715, + "loss": 4436.71728515625, + "loss/core": 4436.7158203125, + "loss/preference_sft": 0.30908656120300293, + "loss/reference_anchor": 0.04678858071565628, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.291474461555481, + "rewards/margins": 0.5815840363502502, + "rewards/rejected": 0.7098905444145203, + "step": 80 + }, + { + "drift/chosen_abs": 0.24453523755073547, + "drift/rejected_abs": 0.1695018708705902, + "epoch": 0.08121342410127792, + "grad_norm": 556.0, + "learning_rate": 2.3333333333333333e-07, + "logits/chosen": -2.1169114112854004, + "logits/rejected": -2.1277921199798584, + "logps/chosen": -0.2921332120895386, + "logps/rejected": -0.28643548488616943, + "loss": 4435.0263671875, + "loss/core": 4435.017578125, + "loss/preference_sft": 0.2921332120895386, + "loss/reference_anchor": 0.4582666754722595, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.4450902938842773, + "rewards/margins": 0.7517923712730408, + "rewards/rejected": 1.6932977437973022, + "step": 85 + }, + { + "drift/chosen_abs": 0.28909215331077576, + "drift/rejected_abs": 0.06680949032306671, + "epoch": 0.08599068434252956, + "grad_norm": 1080.0, + "learning_rate": 2.4722222222222224e-07, + "logits/chosen": -2.001685380935669, + "logits/rejected": -2.1864590644836426, + "logps/chosen": -0.3068524897098541, + "logps/rejected": -0.30106669664382935, + "loss": 4415.2734375, + "loss/core": 4415.2666015625, + "loss/preference_sft": 0.3068524897098541, + "loss/reference_anchor": 0.31496793031692505, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.889475107192993, + "rewards/margins": 2.222256898880005, + "rewards/rejected": 0.6672185659408569, + "step": 90 + }, + { + "drift/chosen_abs": 0.2167942225933075, + "drift/rejected_abs": 0.13755588233470917, + "epoch": 0.09076794458378121, + "grad_norm": 2912.0, + "learning_rate": 2.4998495746616845e-07, + "logits/chosen": -2.0368285179138184, + "logits/rejected": -1.9507942199707031, + "logps/chosen": -0.28156915307044983, + "logps/rejected": -0.30304011702537537, + "loss": 4434.26171875, + "loss/core": 4434.25439453125, + "loss/preference_sft": 0.28156915307044983, + "loss/reference_anchor": 0.33130401372909546, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.167041063308716, + "rewards/margins": 0.7925023436546326, + "rewards/rejected": 1.3745384216308594, + "step": 95 + }, + { + "drift/chosen_abs": 0.29789382219314575, + "drift/rejected_abs": 0.1836269050836563, + "epoch": 0.09554520482503284, + "grad_norm": 600.0, + "learning_rate": 2.4992385337738696e-07, + "logits/chosen": -2.0101962089538574, + "logits/rejected": -2.0421595573425293, + "logps/chosen": -0.2820836007595062, + "logps/rejected": -0.31330883502960205, + "loss": 4426.01904296875, + "loss/core": 4426.0087890625, + "loss/preference_sft": 0.2820836007595062, + "loss/reference_anchor": 0.4864688813686371, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.9787633419036865, + "rewards/margins": 1.4464952945709229, + "rewards/rejected": 1.5322680473327637, + "step": 100 + }, + { + "drift/chosen_abs": 0.3174459934234619, + "drift/rejected_abs": 0.13966134190559387, + "epoch": 0.10032246506628449, + "grad_norm": 812.0, + "learning_rate": 2.4981577053636144e-07, + "logits/chosen": -1.9345197677612305, + "logits/rejected": -1.9873930215835571, + "logps/chosen": -0.2772332727909088, + "logps/rejected": -0.2635360360145569, + "loss": 4421.1513671875, + "loss/core": 4421.1435546875, + "loss/preference_sft": 0.2772332727909088, + "loss/reference_anchor": 0.37949052453041077, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.1736035346984863, + "rewards/margins": 1.7784216403961182, + "rewards/rejected": 1.3951818943023682, + "step": 105 + }, + { + "drift/chosen_abs": 0.21933257579803467, + "drift/rejected_abs": 0.10228520631790161, + "epoch": 0.10509972530753613, + "grad_norm": 556.0, + "learning_rate": 2.496607495886281e-07, + "logits/chosen": -2.123267412185669, + "logits/rejected": -2.0964272022247314, + "logps/chosen": -0.2958363890647888, + "logps/rejected": -0.30722880363464355, + "loss": 4429.02294921875, + "loss/core": 4429.0166015625, + "loss/preference_sft": 0.2958363890647888, + "loss/reference_anchor": 0.31792789697647095, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.1888623237609863, + "rewards/margins": 1.1705862283706665, + "rewards/rejected": 1.0182762145996094, + "step": 110 + }, + { + "drift/chosen_abs": 0.3494819402694702, + "drift/rejected_abs": 0.1267239898443222, + "epoch": 0.10987698554878778, + "grad_norm": 5792.0, + "learning_rate": 2.4945884883122553e-07, + "logits/chosen": -2.022082805633545, + "logits/rejected": -2.070870876312256, + "logps/chosen": -0.2908271253108978, + "logps/rejected": -0.28468307852745056, + "loss": 4415.51806640625, + "loss/core": 4415.5029296875, + "loss/preference_sft": 0.2908271253108978, + "loss/reference_anchor": 0.6909618377685547, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.49409556388855, + "rewards/margins": 2.2313833236694336, + "rewards/rejected": 1.2627122402191162, + "step": 115 + }, + { + "drift/chosen_abs": 0.18104346096515656, + "drift/rejected_abs": 0.140670508146286, + "epoch": 0.11465424579003941, + "grad_norm": 9856.0, + "learning_rate": 2.492101441907714e-07, + "logits/chosen": -2.241240978240967, + "logits/rejected": -2.1032841205596924, + "logps/chosen": -0.2601907551288605, + "logps/rejected": -0.28034669160842896, + "loss": 4439.17724609375, + "loss/core": 4439.17431640625, + "loss/preference_sft": 0.2601907551288605, + "loss/reference_anchor": 0.14795775711536407, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.8099110126495361, + "rewards/margins": 0.4032059609889984, + "rewards/rejected": 1.4067050218582153, + "step": 120 + }, + { + "drift/chosen_abs": 0.408181756734848, + "drift/rejected_abs": 0.10417692363262177, + "epoch": 0.11943150603129106, + "grad_norm": 422.0, + "learning_rate": 2.4891472919490977e-07, + "logits/chosen": -1.9139236211776733, + "logits/rejected": -1.9596039056777954, + "logps/chosen": -0.2813839018344879, + "logps/rejected": -0.2793610095977783, + "loss": 4404.7919921875, + "loss/core": 4404.77392578125, + "loss/preference_sft": 0.2813839018344879, + "loss/reference_anchor": 0.8949528932571411, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 4.081476211547852, + "rewards/margins": 3.0719616413116455, + "rewards/rejected": 1.0095146894454956, + "step": 125 + }, + { + "drift/chosen_abs": 0.2684691548347473, + "drift/rejected_abs": 0.1741405725479126, + "epoch": 0.1242087662725427, + "grad_norm": 8576.0, + "learning_rate": 2.4857271493713894e-07, + "logits/chosen": -2.075075626373291, + "logits/rejected": -2.073489189147949, + "logps/chosen": -0.29706940054893494, + "logps/rejected": -0.2922501266002655, + "loss": 4431.94091796875, + "loss/core": 4431.92919921875, + "loss/preference_sft": 0.29706940054893494, + "loss/reference_anchor": 0.525195300579071, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.6840920448303223, + "rewards/margins": 0.9443261027336121, + "rewards/rejected": 1.7397658824920654, + "step": 130 + }, + { + "drift/chosen_abs": 0.17242693901062012, + "drift/rejected_abs": 0.0899103656411171, + "epoch": 0.12898602651379434, + "grad_norm": 1224.0, + "learning_rate": 2.481842300350339e-07, + "logits/chosen": -2.03088641166687, + "logits/rejected": -2.094935894012451, + "logps/chosen": -0.27687835693359375, + "logps/rejected": -0.26349860429763794, + "loss": 4433.5263671875, + "loss/core": 4433.5234375, + "loss/preference_sft": 0.27687835693359375, + "loss/reference_anchor": 0.09716491401195526, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.7209184169769287, + "rewards/margins": 0.8248888850212097, + "rewards/rejected": 0.8960295915603638, + "step": 135 + }, + { + "drift/chosen_abs": 0.22251293063163757, + "drift/rejected_abs": 0.09145991504192352, + "epoch": 0.13376328675504598, + "grad_norm": 500.0, + "learning_rate": 2.4774942058187854e-07, + "logits/chosen": -2.1904947757720947, + "logits/rejected": -2.24971079826355, + "logps/chosen": -0.2990642189979553, + "logps/rejected": -0.2923958897590637, + "loss": 4427.3486328125, + "loss/core": 4427.3408203125, + "loss/preference_sft": 0.2990642189979553, + "loss/reference_anchor": 0.39207085967063904, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.223792314529419, + "rewards/margins": 1.3103091716766357, + "rewards/rejected": 0.9134831428527832, + "step": 140 + }, + { + "drift/chosen_abs": 0.18590529263019562, + "drift/rejected_abs": 0.10894735902547836, + "epoch": 0.13854054699629761, + "grad_norm": 1584.0, + "learning_rate": 2.472684500917257e-07, + "logits/chosen": -1.9672470092773438, + "logits/rejected": -1.9949979782104492, + "logps/chosen": -0.33192163705825806, + "logps/rejected": -0.3151020109653473, + "loss": 4434.21435546875, + "loss/core": 4434.2119140625, + "loss/preference_sft": 0.33192163705825806, + "loss/reference_anchor": 0.10941161960363388, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8581035137176514, + "rewards/margins": 0.773113489151001, + "rewards/rejected": 1.0849900245666504, + "step": 145 + }, + { + "drift/chosen_abs": 0.15106205642223358, + "drift/rejected_abs": 0.05738229304552078, + "epoch": 0.14331780723754928, + "grad_norm": 624.0, + "learning_rate": 2.467414994379065e-07, + "logits/chosen": -2.0708765983581543, + "logits/rejected": -2.1375513076782227, + "logps/chosen": -0.30995962023735046, + "logps/rejected": -0.31707248091697693, + "loss": 4432.21533203125, + "loss/core": 4432.2119140625, + "loss/preference_sft": 0.30995962023735046, + "loss/reference_anchor": 0.15558083355426788, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.5100862979888916, + "rewards/margins": 0.9373763799667358, + "rewards/rejected": 0.5727099776268005, + "step": 150 + }, + { + "drift/chosen_abs": 0.15896198153495789, + "drift/rejected_abs": 0.08986209332942963, + "epoch": 0.1480950674788009, + "grad_norm": 1496.0, + "learning_rate": 2.4616876678501114e-07, + "logits/chosen": -2.2561638355255127, + "logits/rejected": -2.3034205436706543, + "logps/chosen": -0.31711262464523315, + "logps/rejected": -0.302999347448349, + "loss": 4435.2392578125, + "loss/core": 4435.23681640625, + "loss/preference_sft": 0.31711262464523315, + "loss/reference_anchor": 0.06949084997177124, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.588337779045105, + "rewards/margins": 0.6926823258399963, + "rewards/rejected": 0.8956555128097534, + "step": 155 + }, + { + "drift/chosen_abs": 0.3718644082546234, + "drift/rejected_abs": 0.11488336324691772, + "epoch": 0.15287232772005255, + "grad_norm": 13056.0, + "learning_rate": 2.4555046751436735e-07, + "logits/chosen": -2.1315560340881348, + "logits/rejected": -2.117034912109375, + "logps/chosen": -0.29637980461120605, + "logps/rejected": -0.282162606716156, + "loss": 4410.93359375, + "loss/core": 4410.9189453125, + "loss/preference_sft": 0.29637980461120605, + "loss/reference_anchor": 0.6615738868713379, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.717616558074951, + "rewards/margins": 2.570777416229248, + "rewards/rejected": 1.1468391418457031, + "step": 160 + }, + { + "drift/chosen_abs": 0.18394318222999573, + "drift/rejected_abs": 0.09417982399463654, + "epoch": 0.15764958796130418, + "grad_norm": 944.0, + "learning_rate": 2.4488683414304425e-07, + "logits/chosen": -2.132768392562866, + "logits/rejected": -2.2345738410949707, + "logps/chosen": -0.31837791204452515, + "logps/rejected": -0.31375187635421753, + "loss": 4432.53857421875, + "loss/core": 4432.53564453125, + "loss/preference_sft": 0.31837791204452515, + "loss/reference_anchor": 0.1169905811548233, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.8377997875213623, + "rewards/margins": 0.896827220916748, + "rewards/rejected": 0.940972626209259, + "step": 165 + }, + { + "drift/chosen_abs": 0.15466679632663727, + "drift/rejected_abs": 0.09628058969974518, + "epoch": 0.16242684820255585, + "grad_norm": 1496.0, + "learning_rate": 2.4417811623641203e-07, + "logits/chosen": -2.274855136871338, + "logits/rejected": -2.3352713584899902, + "logps/chosen": -0.27421072125434875, + "logps/rejected": -0.2842170298099518, + "loss": 4436.6865234375, + "loss/core": 4436.68359375, + "loss/preference_sft": 0.27421072125434875, + "loss/reference_anchor": 0.11561869084835052, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.5460879802703857, + "rewards/margins": 0.5835286378860474, + "rewards/rejected": 0.9625593423843384, + "step": 170 + }, + { + "drift/chosen_abs": 0.5257049798965454, + "drift/rejected_abs": 0.19930550456047058, + "epoch": 0.16720410844380748, + "grad_norm": 1896.0, + "learning_rate": 2.4342458031429113e-07, + "logits/chosen": -1.8325287103652954, + "logits/rejected": -1.8387473821640015, + "logps/chosen": -0.2876504957675934, + "logps/rejected": -0.29608818888664246, + "loss": 4401.90185546875, + "loss/core": 4401.876953125, + "loss/preference_sft": 0.2876504957675934, + "loss/reference_anchor": 1.251502275466919, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 5.2567338943481445, + "rewards/margins": 3.286668300628662, + "rewards/rejected": 1.9700653553009033, + "step": 175 + }, + { + "drift/chosen_abs": 0.3301013112068176, + "drift/rejected_abs": 0.15288423001766205, + "epoch": 0.17198136868505912, + "grad_norm": 458.0, + "learning_rate": 2.4262650975072444e-07, + "logits/chosen": -2.189413070678711, + "logits/rejected": -2.3054258823394775, + "logps/chosen": -0.2844337224960327, + "logps/rejected": -0.2936963140964508, + "loss": 4420.90087890625, + "loss/core": 4420.8896484375, + "loss/preference_sft": 0.2844337224960327, + "loss/reference_anchor": 0.5586980581283569, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 3.300053119659424, + "rewards/margins": 1.8004553318023682, + "rewards/rejected": 1.4995979070663452, + "step": 180 + }, + { + "drift/chosen_abs": 0.1918279230594635, + "drift/rejected_abs": 0.07734932750463486, + "epoch": 0.17675862892631075, + "grad_norm": 704.0, + "learning_rate": 2.417842046674122e-07, + "logits/chosen": -2.3335790634155273, + "logits/rejected": -2.3553173542022705, + "logps/chosen": -0.2591295838356018, + "logps/rejected": -0.26806166768074036, + "loss": 4429.29296875, + "loss/core": 4429.2900390625, + "loss/preference_sft": 0.2591295838356018, + "loss/reference_anchor": 0.09960640221834183, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9182792901992798, + "rewards/margins": 1.1459853649139404, + "rewards/rejected": 0.7722936868667603, + "step": 185 + }, + { + "drift/chosen_abs": 0.19113460183143616, + "drift/rejected_abs": 0.09281430393457413, + "epoch": 0.18153588916756241, + "grad_norm": 564.0, + "learning_rate": 2.4089798182084843e-07, + "logits/chosen": -2.0862205028533936, + "logits/rejected": -2.1600217819213867, + "logps/chosen": -0.30776557326316833, + "logps/rejected": -0.30735090374946594, + "loss": 4431.3701171875, + "loss/core": 4431.36669921875, + "loss/preference_sft": 0.30776557326316833, + "loss/reference_anchor": 0.1401655375957489, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9096482992172241, + "rewards/margins": 0.9876093864440918, + "rewards/rejected": 0.9220390319824219, + "step": 190 + }, + { + "drift/chosen_abs": 0.21678169071674347, + "drift/rejected_abs": 0.13430018723011017, + "epoch": 0.18631314940881405, + "grad_norm": 704.0, + "learning_rate": 2.3996817448320195e-07, + "logits/chosen": -2.158468246459961, + "logits/rejected": -2.112973213195801, + "logps/chosen": -0.28618916869163513, + "logps/rejected": -0.29950863122940063, + "loss": 4432.5107421875, + "loss/core": 4432.50537109375, + "loss/preference_sft": 0.28618916869163513, + "loss/reference_anchor": 0.24302752315998077, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.1672415733337402, + "rewards/margins": 0.914328932762146, + "rewards/rejected": 1.2529126405715942, + "step": 195 + }, + { + "drift/chosen_abs": 0.28183844685554504, + "drift/rejected_abs": 0.15761585533618927, + "epoch": 0.19109040965006568, + "grad_norm": 1928.0, + "learning_rate": 2.3899513231698607e-07, + "logits/chosen": -2.1264212131500244, + "logits/rejected": -2.0478241443634033, + "logps/chosen": -0.292626291513443, + "logps/rejected": -0.28839462995529175, + "loss": 4428.09912109375, + "loss/core": 4428.0927734375, + "loss/preference_sft": 0.292626291513443, + "loss/reference_anchor": 0.28291893005371094, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.8163602352142334, + "rewards/margins": 1.2429499626159668, + "rewards/rejected": 1.5734103918075562, + "step": 200 + }, + { + "drift/chosen_abs": 0.26241064071655273, + "drift/rejected_abs": 0.14250826835632324, + "epoch": 0.19586766989131732, + "grad_norm": 2240.0, + "learning_rate": 2.3797922124356506e-07, + "logits/chosen": -2.029865264892578, + "logits/rejected": -2.0273633003234863, + "logps/chosen": -0.39078933000564575, + "logps/rejected": -0.30789244174957275, + "loss": 4427.798828125, + "loss/core": 4427.78955078125, + "loss/preference_sft": 0.39078933000564575, + "loss/reference_anchor": 0.41654643416404724, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.6241068840026855, + "rewards/margins": 1.2998729944229126, + "rewards/rejected": 1.3242336511611938, + "step": 205 + }, + { + "drift/chosen_abs": 0.17145346105098724, + "drift/rejected_abs": 0.07196025550365448, + "epoch": 0.20064493013256898, + "grad_norm": 532.0, + "learning_rate": 2.3692082330554585e-07, + "logits/chosen": -2.2569234371185303, + "logits/rejected": -2.352635145187378, + "logps/chosen": -0.25879400968551636, + "logps/rejected": -0.2546376585960388, + "loss": 4431.224609375, + "loss/core": 4431.22265625, + "loss/preference_sft": 0.25879400968551636, + "loss/reference_anchor": 0.05695471167564392, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.7141616344451904, + "rewards/margins": 0.9953814744949341, + "rewards/rejected": 0.7187802791595459, + "step": 210 + }, + { + "drift/chosen_abs": 0.2754186987876892, + "drift/rejected_abs": 0.17294225096702576, + "epoch": 0.20542219037382062, + "grad_norm": 6752.0, + "learning_rate": 2.3582033652310765e-07, + "logits/chosen": -1.9878219366073608, + "logits/rejected": -1.9628711938858032, + "logps/chosen": -0.2865174412727356, + "logps/rejected": -0.27714353799819946, + "loss": 4431.0849609375, + "loss/core": 4431.0791015625, + "loss/preference_sft": 0.2865174412727356, + "loss/reference_anchor": 0.26327061653137207, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.7541871070861816, + "rewards/margins": 1.025450587272644, + "rewards/rejected": 1.7287365198135376, + "step": 215 + }, + { + "drift/chosen_abs": 0.3495247960090637, + "drift/rejected_abs": 0.07642688602209091, + "epoch": 0.21019945061507225, + "grad_norm": 544.0, + "learning_rate": 2.346781747443227e-07, + "logits/chosen": -2.042937755584717, + "logits/rejected": -2.1352648735046387, + "logps/chosen": -0.29404938220977783, + "logps/rejected": -0.28113672137260437, + "loss": 4409.341796875, + "loss/core": 4409.32666015625, + "loss/preference_sft": 0.29404938220977783, + "loss/reference_anchor": 0.7147288918495178, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.494548797607422, + "rewards/margins": 2.73149037361145, + "rewards/rejected": 0.7630582451820374, + "step": 220 + }, + { + "drift/chosen_abs": 0.3245381712913513, + "drift/rejected_abs": 0.10127030313014984, + "epoch": 0.2149767108563239, + "grad_norm": 3024.0, + "learning_rate": 2.3349476748952508e-07, + "logits/chosen": -2.1589510440826416, + "logits/rejected": -2.2461776733398438, + "logps/chosen": -0.27290159463882446, + "logps/rejected": -0.27267274260520935, + "loss": 4415.03759765625, + "loss/core": 4415.03125, + "loss/preference_sft": 0.27290159463882446, + "loss/reference_anchor": 0.2985621392726898, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.2441680431365967, + "rewards/margins": 2.2339510917663574, + "rewards/rejected": 1.0102171897888184, + "step": 225 + }, + { + "drift/chosen_abs": 0.36413469910621643, + "drift/rejected_abs": 0.13577529788017273, + "epoch": 0.21975397109757555, + "grad_norm": 9600.0, + "learning_rate": 2.3227055978978545e-07, + "logits/chosen": -2.0139591693878174, + "logits/rejected": -2.053098678588867, + "logps/chosen": -0.2517145872116089, + "logps/rejected": -0.25124287605285645, + "loss": 4415.037109375, + "loss/core": 4415.02294921875, + "loss/preference_sft": 0.2517145872116089, + "loss/reference_anchor": 0.6684554815292358, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.639683246612549, + "rewards/margins": 2.2833621501922607, + "rewards/rejected": 1.3563209772109985, + "step": 230 + }, + { + "drift/chosen_abs": 0.2075643092393875, + "drift/rejected_abs": 0.12083862721920013, + "epoch": 0.2245312313388272, + "grad_norm": 616.0, + "learning_rate": 2.3100601201955321e-07, + "logits/chosen": -2.329841136932373, + "logits/rejected": -2.312140464782715, + "logps/chosen": -0.25918227434158325, + "logps/rejected": -0.2679867744445801, + "loss": 4433.1220703125, + "loss/core": 4433.1142578125, + "loss/preference_sft": 0.25918227434158325, + "loss/reference_anchor": 0.35554802417755127, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.0753586292266846, + "rewards/margins": 0.868099570274353, + "rewards/rejected": 1.207258939743042, + "step": 235 + }, + { + "drift/chosen_abs": 0.3381517231464386, + "drift/rejected_abs": 0.19172975420951843, + "epoch": 0.22930849158007882, + "grad_norm": 1488.0, + "learning_rate": 2.2970159972352864e-07, + "logits/chosen": -2.123835325241089, + "logits/rejected": -2.1424546241760254, + "logps/chosen": -0.2800813317298889, + "logps/rejected": -0.2956727147102356, + "loss": 4425.13037109375, + "loss/core": 4425.1181640625, + "loss/preference_sft": 0.2800813317298889, + "loss/reference_anchor": 0.6074963808059692, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.381516695022583, + "rewards/margins": 1.4686692953109741, + "rewards/rejected": 1.9128472805023193, + "step": 240 + }, + { + "drift/chosen_abs": 0.3294285535812378, + "drift/rejected_abs": 0.16067305207252502, + "epoch": 0.23408575182133046, + "grad_norm": 5056.0, + "learning_rate": 2.2835781343782966e-07, + "logits/chosen": -1.8609027862548828, + "logits/rejected": -1.8787086009979248, + "logps/chosen": -0.2937183976173401, + "logps/rejected": -0.2992231547832489, + "loss": 4422.87890625, + "loss/core": 4422.8642578125, + "loss/preference_sft": 0.2937183976173401, + "loss/reference_anchor": 0.6773186922073364, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.292163848876953, + "rewards/margins": 1.6896095275878906, + "rewards/rejected": 1.6025537252426147, + "step": 245 + }, + { + "drift/chosen_abs": 0.15980662405490875, + "drift/rejected_abs": 0.10799809545278549, + "epoch": 0.23886301206258212, + "grad_norm": 418.0, + "learning_rate": 2.2697515850552152e-07, + "logits/chosen": -2.347092866897583, + "logits/rejected": -2.329366683959961, + "logps/chosen": -0.288370281457901, + "logps/rejected": -0.29249584674835205, + "loss": 4437.6669921875, + "loss/core": 4437.66357421875, + "loss/preference_sft": 0.288370281457901, + "loss/reference_anchor": 0.1452062875032425, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.5972728729248047, + "rewards/margins": 0.5199095606803894, + "rewards/rejected": 1.07736337184906, + "step": 250 + }, + { + "drift/chosen_abs": 0.1499588042497635, + "drift/rejected_abs": 0.0562698058784008, + "epoch": 0.24364027230383375, + "grad_norm": 376.0, + "learning_rate": 2.2555415488657775e-07, + "logits/chosen": -2.120940685272217, + "logits/rejected": -2.184211254119873, + "logps/chosen": -0.30225870013237, + "logps/rejected": -0.3060735762119293, + "loss": 4431.9951171875, + "loss/core": 4431.9931640625, + "loss/preference_sft": 0.30225870013237, + "loss/reference_anchor": 0.07390942424535751, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.499276876449585, + "rewards/margins": 0.9399444460868835, + "rewards/rejected": 0.5593323707580566, + "step": 255 + }, + { + "drift/chosen_abs": 0.23949389159679413, + "drift/rejected_abs": 0.10860034078359604, + "epoch": 0.2484175325450854, + "grad_norm": 482.0, + "learning_rate": 2.240953369623447e-07, + "logits/chosen": -2.2790729999542236, + "logits/rejected": -2.3324427604675293, + "logps/chosen": -0.27358928322792053, + "logps/rejected": -0.2760041356086731, + "loss": 4427.1962890625, + "loss/core": 4427.19140625, + "loss/preference_sft": 0.27358928322792053, + "loss/reference_anchor": 0.19914691150188446, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.3933863639831543, + "rewards/margins": 1.307618260383606, + "rewards/rejected": 1.0857679843902588, + "step": 260 + }, + { + "drift/chosen_abs": 0.14223968982696533, + "drift/rejected_abs": 0.07098479568958282, + "epoch": 0.25319479278633705, + "grad_norm": 624.0, + "learning_rate": 2.225992533345824e-07, + "logits/chosen": -2.369666337966919, + "logits/rejected": -2.405613422393799, + "logps/chosen": -0.28374093770980835, + "logps/rejected": -0.2866232693195343, + "loss": 4434.9873046875, + "loss/core": 4434.9853515625, + "loss/preference_sft": 0.28374093770980835, + "loss/reference_anchor": 0.07014183700084686, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.4223629236221313, + "rewards/margins": 0.7129005193710327, + "rewards/rejected": 0.7094623446464539, + "step": 265 + }, + { + "drift/chosen_abs": 0.18329541385173798, + "drift/rejected_abs": 0.06777513027191162, + "epoch": 0.2579720530275887, + "grad_norm": 1004.0, + "learning_rate": 2.210664666191579e-07, + "logits/chosen": -2.1638669967651367, + "logits/rejected": -2.2433056831359863, + "logps/chosen": -0.3162863850593567, + "logps/rejected": -0.2930854558944702, + "loss": 4429.1513671875, + "loss/core": 4429.1494140625, + "loss/preference_sft": 0.3162863850593567, + "loss/reference_anchor": 0.09944357722997665, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8323523998260498, + "rewards/margins": 1.1556055545806885, + "rewards/rejected": 0.6767469644546509, + "step": 270 + }, + { + "drift/chosen_abs": 0.24756188690662384, + "drift/rejected_abs": 0.1202814131975174, + "epoch": 0.2627493132688403, + "grad_norm": 1320.0, + "learning_rate": 2.1949755323446848e-07, + "logits/chosen": -2.2548816204071045, + "logits/rejected": -2.2354800701141357, + "logps/chosen": -0.2556923031806946, + "logps/rejected": -0.2606434226036072, + "loss": 4427.748046875, + "loss/core": 4427.7431640625, + "loss/preference_sft": 0.2556923031806946, + "loss/reference_anchor": 0.2529018521308899, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.473615884780884, + "rewards/margins": 1.273666501045227, + "rewards/rejected": 1.1999495029449463, + "step": 275 + }, + { + "drift/chosen_abs": 0.2426796853542328, + "drift/rejected_abs": 0.13973277807235718, + "epoch": 0.26752657351009196, + "grad_norm": 2256.0, + "learning_rate": 2.1789310318467426e-07, + "logits/chosen": -1.8090507984161377, + "logits/rejected": -1.8918311595916748, + "logps/chosen": -0.3173496425151825, + "logps/rejected": -0.32060688734054565, + "loss": 4430.7841796875, + "loss/core": 4430.7802734375, + "loss/preference_sft": 0.3173496425151825, + "loss/reference_anchor": 0.15054082870483398, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.4233853816986084, + "rewards/margins": 1.029157280921936, + "rewards/rejected": 1.3942279815673828, + "step": 280 + }, + { + "drift/chosen_abs": 0.28747934103012085, + "drift/rejected_abs": 0.16268548369407654, + "epoch": 0.2723038337513436, + "grad_norm": 2704.0, + "learning_rate": 2.1625371983782182e-07, + "logits/chosen": -2.0867972373962402, + "logits/rejected": -2.0485246181488037, + "logps/chosen": -0.2663497030735016, + "logps/rejected": -0.27806463837623596, + "loss": 4427.39501953125, + "loss/core": 4427.3876953125, + "loss/preference_sft": 0.2663497030735016, + "loss/reference_anchor": 0.32496556639671326, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.874793291091919, + "rewards/margins": 1.3126866817474365, + "rewards/rejected": 1.562106728553772, + "step": 285 + }, + { + "drift/chosen_abs": 0.2037903517484665, + "drift/rejected_abs": 0.09013260900974274, + "epoch": 0.27708109399259523, + "grad_norm": 1144.0, + "learning_rate": 2.14580019698942e-07, + "logits/chosen": -2.1753249168395996, + "logits/rejected": -2.2431302070617676, + "logps/chosen": -0.294014573097229, + "logps/rejected": -0.292147696018219, + "loss": 4429.5263671875, + "loss/core": 4429.5224609375, + "loss/preference_sft": 0.294014573097229, + "loss/reference_anchor": 0.18461060523986816, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.0374693870544434, + "rewards/margins": 1.1413319110870361, + "rewards/rejected": 0.8961375951766968, + "step": 290 + }, + { + "drift/chosen_abs": 0.2267070710659027, + "drift/rejected_abs": 0.16824612021446228, + "epoch": 0.28185835423384686, + "grad_norm": 14912.0, + "learning_rate": 2.1287263217820773e-07, + "logits/chosen": -2.1485249996185303, + "logits/rejected": -2.1281566619873047, + "logps/chosen": -0.27404850721359253, + "logps/rejected": -0.2835575342178345, + "loss": 4437.10986328125, + "loss/core": 4437.1025390625, + "loss/preference_sft": 0.27404850721359253, + "loss/reference_anchor": 0.32569584250450134, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.265812397003174, + "rewards/margins": 0.5857616662979126, + "rewards/rejected": 1.6800508499145508, + "step": 295 + }, + { + "drift/chosen_abs": 0.27692022919654846, + "drift/rejected_abs": 0.1342446506023407, + "epoch": 0.28663561447509855, + "grad_norm": 1496.0, + "learning_rate": 2.111321993542385e-07, + "logits/chosen": -1.9965057373046875, + "logits/rejected": -1.9842058420181274, + "logps/chosen": -0.2693387567996979, + "logps/rejected": -0.28916555643081665, + "loss": 4425.4814453125, + "loss/core": 4425.4775390625, + "loss/preference_sft": 0.2693387567996979, + "loss/reference_anchor": 0.1915903389453888, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.7670462131500244, + "rewards/margins": 1.4375035762786865, + "rewards/rejected": 1.3295425176620483, + "step": 300 + }, + { + "drift/chosen_abs": 0.18260391056537628, + "drift/rejected_abs": 0.13512252271175385, + "epoch": 0.2914128747163502, + "grad_norm": 4416.0, + "learning_rate": 2.0935937573264096e-07, + "logits/chosen": -2.097672700881958, + "logits/rejected": -2.110541820526123, + "logps/chosen": -0.28904464840888977, + "logps/rejected": -0.29786160588264465, + "loss": 4438.23974609375, + "loss/core": 4438.23681640625, + "loss/preference_sft": 0.28904464840888977, + "loss/reference_anchor": 0.13433417677879333, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.8258289098739624, + "rewards/margins": 0.4759580194950104, + "rewards/rejected": 1.3498706817626953, + "step": 305 + }, + { + "drift/chosen_abs": 0.20982956886291504, + "drift/rejected_abs": 0.137477308511734, + "epoch": 0.2961901349576018, + "grad_norm": 3392.0, + "learning_rate": 2.0755482799987596e-07, + "logits/chosen": -2.025709629058838, + "logits/rejected": -1.9913642406463623, + "logps/chosen": -0.2986164689064026, + "logps/rejected": -0.3017011284828186, + "loss": 4434.96875, + "loss/core": 4434.96337890625, + "loss/preference_sft": 0.2986164689064026, + "loss/reference_anchor": 0.26511436700820923, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.0975375175476074, + "rewards/margins": 0.7258044481277466, + "rewards/rejected": 1.3717331886291504, + "step": 310 + }, + { + "drift/chosen_abs": 0.15806427597999573, + "drift/rejected_abs": 0.09864726662635803, + "epoch": 0.30096739519885346, + "grad_norm": 640.0, + "learning_rate": 2.0571923477254526e-07, + "logits/chosen": -2.2739768028259277, + "logits/rejected": -2.2487998008728027, + "logps/chosen": -0.2909921109676361, + "logps/rejected": -0.2905241847038269, + "loss": 4436.4794921875, + "loss/core": 4436.4765625, + "loss/preference_sft": 0.2909921109676361, + "loss/reference_anchor": 0.10613216459751129, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.5796961784362793, + "rewards/margins": 0.600797176361084, + "rewards/rejected": 0.9788990020751953, + "step": 315 + }, + { + "drift/chosen_abs": 0.28660792112350464, + "drift/rejected_abs": 0.19245363771915436, + "epoch": 0.3057446554401051, + "grad_norm": 668.0, + "learning_rate": 2.038532863421914e-07, + "logits/chosen": -2.1396350860595703, + "logits/rejected": -2.0602240562438965, + "logps/chosen": -0.28930574655532837, + "logps/rejected": -0.2744773030281067, + "loss": 4432.1318359375, + "loss/core": 4432.1240234375, + "loss/preference_sft": 0.28930574655532837, + "loss/reference_anchor": 0.3356773257255554, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.8660788536071777, + "rewards/margins": 0.9429539442062378, + "rewards/rejected": 1.9231246709823608, + "step": 320 + }, + { + "drift/chosen_abs": 0.15086838603019714, + "drift/rejected_abs": 0.1520775854587555, + "epoch": 0.31052191568135673, + "grad_norm": 856.0, + "learning_rate": 2.0195768441570726e-07, + "logits/chosen": -1.9964319467544556, + "logits/rejected": -2.05317759513855, + "logps/chosen": -0.2916659116744995, + "logps/rejected": -0.2865615487098694, + "loss": 4445.12451171875, + "loss/core": 4445.11767578125, + "loss/preference_sft": 0.2916659116744995, + "loss/reference_anchor": 0.3259647488594055, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.5083897113800049, + "rewards/margins": -0.008684945292770863, + "rewards/rejected": 1.5170745849609375, + "step": 325 + }, + { + "drift/chosen_abs": 0.259127140045166, + "drift/rejected_abs": 0.11504222452640533, + "epoch": 0.31529917592260837, + "grad_norm": 1272.0, + "learning_rate": 2.0003314185145307e-07, + "logits/chosen": -2.14955735206604, + "logits/rejected": -2.0943520069122314, + "logps/chosen": -0.28039321303367615, + "logps/rejected": -0.283574640750885, + "loss": 4425.5029296875, + "loss/core": 4425.498046875, + "loss/preference_sft": 0.28039321303367615, + "loss/reference_anchor": 0.229940265417099, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.5912716388702393, + "rewards/margins": 1.4409186840057373, + "rewards/rejected": 1.150353193283081, + "step": 330 + }, + { + "drift/chosen_abs": 0.27030646800994873, + "drift/rejected_abs": 0.14940232038497925, + "epoch": 0.32007643616386, + "grad_norm": 5376.0, + "learning_rate": 1.9808038239117913e-07, + "logits/chosen": -1.9976972341537476, + "logits/rejected": -1.965847373008728, + "logps/chosen": -0.28784728050231934, + "logps/rejected": -0.2873719334602356, + "loss": 4427.10205078125, + "loss/core": 4427.0966796875, + "loss/preference_sft": 0.28784728050231934, + "loss/reference_anchor": 0.2397988736629486, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.7028987407684326, + "rewards/margins": 1.3128286600112915, + "rewards/rejected": 1.3900703191757202, + "step": 335 + }, + { + "drift/chosen_abs": 0.16661614179611206, + "drift/rejected_abs": 0.08331886678934097, + "epoch": 0.3248536964051117, + "grad_norm": 1120.0, + "learning_rate": 1.9610014038785646e-07, + "logits/chosen": -2.2249255180358887, + "logits/rejected": -2.211129665374756, + "logps/chosen": -0.2958478629589081, + "logps/rejected": -0.3064306080341339, + "loss": 4433.41796875, + "loss/core": 4433.41552734375, + "loss/preference_sft": 0.2958478629589081, + "loss/reference_anchor": 0.08543743938207626, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.664698600769043, + "rewards/margins": 0.83290034532547, + "rewards/rejected": 0.8317981958389282, + "step": 340 + }, + { + "drift/chosen_abs": 0.28869912028312683, + "drift/rejected_abs": 0.23126927018165588, + "epoch": 0.3296309566463633, + "grad_norm": 1304.0, + "learning_rate": 1.9409316052951657e-07, + "logits/chosen": -1.9891431331634521, + "logits/rejected": -2.0764071941375732, + "logps/chosen": -0.2763379216194153, + "logps/rejected": -0.2923387885093689, + "loss": 4436.98974609375, + "loss/core": 4436.9755859375, + "loss/preference_sft": 0.2763379216194153, + "loss/reference_anchor": 0.6921347975730896, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.886991500854492, + "rewards/margins": 0.5755015015602112, + "rewards/rejected": 2.311490058898926, + "step": 345 + }, + { + "drift/chosen_abs": 0.37554502487182617, + "drift/rejected_abs": 0.12323877960443497, + "epoch": 0.33440821688761496, + "grad_norm": 2496.0, + "learning_rate": 1.920601975592047e-07, + "logits/chosen": -2.176884174346924, + "logits/rejected": -2.1506075859069824, + "logps/chosen": -0.2848382294178009, + "logps/rejected": -0.29111620783805847, + "loss": 4411.89306640625, + "loss/core": 4411.876953125, + "loss/preference_sft": 0.2848382294178009, + "loss/reference_anchor": 0.7674547433853149, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.7542431354522705, + "rewards/margins": 2.534789562225342, + "rewards/rejected": 1.2194538116455078, + "step": 350 + }, + { + "drift/chosen_abs": 0.1966264843940735, + "drift/rejected_abs": 0.07787960767745972, + "epoch": 0.3391854771288666, + "grad_norm": 470.0, + "learning_rate": 1.900020159911519e-07, + "logits/chosen": -2.094768524169922, + "logits/rejected": -2.162729263305664, + "logps/chosen": -0.3112557828426361, + "logps/rejected": -0.31276172399520874, + "loss": 4428.56884765625, + "loss/core": 4428.56494140625, + "loss/preference_sft": 0.3112557828426361, + "loss/reference_anchor": 0.14194537699222565, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.9648243188858032, + "rewards/margins": 1.2004649639129639, + "rewards/rejected": 0.7643591165542603, + "step": 355 + }, + { + "drift/chosen_abs": 0.17585548758506775, + "drift/rejected_abs": 0.1803542524576187, + "epoch": 0.34396273737011823, + "grad_norm": 2096.0, + "learning_rate": 1.879193898232725e-07, + "logits/chosen": -2.1437807083129883, + "logits/rejected": -2.1195027828216553, + "logps/chosen": -0.31370845437049866, + "logps/rejected": -0.3198932111263275, + "loss": 4445.20849609375, + "loss/core": 4445.2041015625, + "loss/preference_sft": 0.31370845437049866, + "loss/reference_anchor": 0.19741208851337433, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.757554292678833, + "rewards/margins": -0.0393296480178833, + "rewards/rejected": 1.7968838214874268, + "step": 360 + }, + { + "drift/chosen_abs": 0.3701009154319763, + "drift/rejected_abs": 0.08817116916179657, + "epoch": 0.34873999761136987, + "grad_norm": 2512.0, + "learning_rate": 1.8581310224609496e-07, + "logits/chosen": -2.1306817531585693, + "logits/rejected": -2.202624797821045, + "logps/chosen": -0.2909499704837799, + "logps/rejected": -0.2839373052120209, + "loss": 4408.3310546875, + "loss/core": 4408.3134765625, + "loss/preference_sft": 0.2909499704837799, + "loss/reference_anchor": 0.8197543025016785, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.6982064247131348, + "rewards/margins": 2.817063808441162, + "rewards/rejected": 0.8811424970626831, + "step": 365 + }, + { + "drift/chosen_abs": 0.2949160039424896, + "drift/rejected_abs": 0.12354662269353867, + "epoch": 0.3535172578526215, + "grad_norm": 1032.0, + "learning_rate": 1.8368394534823635e-07, + "logits/chosen": -2.03263258934021, + "logits/rejected": -2.0422098636627197, + "logps/chosen": -0.268283873796463, + "logps/rejected": -0.2804756760597229, + "loss": 4421.8681640625, + "loss/core": 4421.8623046875, + "loss/preference_sft": 0.268283873796463, + "loss/reference_anchor": 0.2536497116088867, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.947744846343994, + "rewards/margins": 1.7133744955062866, + "rewards/rejected": 1.234370231628418, + "step": 370 + }, + { + "drift/chosen_abs": 0.49125900864601135, + "drift/rejected_abs": 0.23263196647167206, + "epoch": 0.35829451809387314, + "grad_norm": 1624.0, + "learning_rate": 1.8153271981852968e-07, + "logits/chosen": -2.010446310043335, + "logits/rejected": -1.984309196472168, + "logps/chosen": -0.30579593777656555, + "logps/rejected": -0.29049938917160034, + "loss": 4411.03564453125, + "loss/core": 4411.00390625, + "loss/preference_sft": 0.30579593777656555, + "loss/reference_anchor": 1.5279619693756104, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 4.912415504455566, + "rewards/margins": 2.5893051624298096, + "rewards/rejected": 2.323111057281494, + "step": 375 + }, + { + "drift/chosen_abs": 0.2286815345287323, + "drift/rejected_abs": 0.1312122792005539, + "epoch": 0.36307177833512483, + "grad_norm": 3968.0, + "learning_rate": 1.7936023464491812e-07, + "logits/chosen": -2.0856316089630127, + "logits/rejected": -2.1967413425445557, + "logps/chosen": -0.27951109409332275, + "logps/rejected": -0.2807064950466156, + "loss": 4431.7783203125, + "loss/core": 4431.7705078125, + "loss/preference_sft": 0.27951109409332275, + "loss/reference_anchor": 0.35531511902809143, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.286098003387451, + "rewards/margins": 0.9744806289672852, + "rewards/rejected": 1.311617136001587, + "step": 380 + }, + { + "drift/chosen_abs": 0.1671498715877533, + "drift/rejected_abs": 0.1056649461388588, + "epoch": 0.36784903857637646, + "grad_norm": 1776.0, + "learning_rate": 1.7716730681022723e-07, + "logits/chosen": -2.121617078781128, + "logits/rejected": -2.081577777862549, + "logps/chosen": -0.31088462471961975, + "logps/rejected": -0.28644299507141113, + "loss": 4436.3212890625, + "loss/core": 4436.31787109375, + "loss/preference_sft": 0.31088462471961975, + "loss/reference_anchor": 0.11547583341598511, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.6688594818115234, + "rewards/margins": 0.6149738430976868, + "rewards/rejected": 1.0538856983184814, + "step": 385 + }, + { + "drift/chosen_abs": 0.22545960545539856, + "drift/rejected_abs": 0.12498001754283905, + "epoch": 0.3726262988176281, + "grad_norm": 580.0, + "learning_rate": 1.7495476098493152e-07, + "logits/chosen": -2.199582099914551, + "logits/rejected": -2.216892957687378, + "logps/chosen": -0.27879446744918823, + "logps/rejected": -0.28513437509536743, + "loss": 4430.99560546875, + "loss/core": 4430.9912109375, + "loss/preference_sft": 0.27879446744918823, + "loss/reference_anchor": 0.1869107335805893, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.2526803016662598, + "rewards/margins": 1.0196499824523926, + "rewards/rejected": 1.2330303192138672, + "step": 390 + }, + { + "drift/chosen_abs": 0.3004913330078125, + "drift/rejected_abs": 0.16865012049674988, + "epoch": 0.37740355905887973, + "grad_norm": 704.0, + "learning_rate": 1.7272342921702937e-07, + "logits/chosen": -2.0919270515441895, + "logits/rejected": -2.11561918258667, + "logps/chosen": -0.30641791224479675, + "logps/rejected": -0.303880900144577, + "loss": 4427.22021484375, + "loss/core": 4427.20849609375, + "loss/preference_sft": 0.30641791224479675, + "loss/reference_anchor": 0.5793026089668274, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.0033681392669678, + "rewards/margins": 1.31758713722229, + "rewards/rejected": 1.6857811212539673, + "step": 395 + }, + { + "drift/chosen_abs": 0.2755824029445648, + "drift/rejected_abs": 0.2149803191423416, + "epoch": 0.38218081930013137, + "grad_norm": 432.0, + "learning_rate": 1.7047415061914393e-07, + "logits/chosen": -2.0630311965942383, + "logits/rejected": -2.2178916931152344, + "logps/chosen": -0.27443453669548035, + "logps/rejected": -0.2907470166683197, + "loss": 4437.09130859375, + "loss/core": 4437.0791015625, + "loss/preference_sft": 0.27443453669548035, + "loss/reference_anchor": 0.5900745987892151, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.7551357746124268, + "rewards/margins": 0.606482982635498, + "rewards/rejected": 2.1486525535583496, + "step": 400 + }, + { + "drift/chosen_abs": 0.26228779554367065, + "drift/rejected_abs": 0.12948082387447357, + "epoch": 0.386958079541383, + "grad_norm": 4704.0, + "learning_rate": 1.6820777105296707e-07, + "logits/chosen": -2.1135382652282715, + "logits/rejected": -2.212303876876831, + "logps/chosen": -0.3208262026309967, + "logps/rejected": -0.2868503928184509, + "loss": 4427.3076171875, + "loss/core": 4427.296875, + "loss/preference_sft": 0.3208262026309967, + "loss/reference_anchor": 0.47984394431114197, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.6186184883117676, + "rewards/margins": 1.3284586668014526, + "rewards/rejected": 1.2901595830917358, + "step": 405 + }, + { + "drift/chosen_abs": 0.2780519723892212, + "drift/rejected_abs": 0.13061827421188354, + "epoch": 0.39173533978263464, + "grad_norm": 540.0, + "learning_rate": 1.6592514281116553e-07, + "logits/chosen": -2.0848355293273926, + "logits/rejected": -2.101752519607544, + "logps/chosen": -0.26733091473579407, + "logps/rejected": -0.2704174518585205, + "loss": 4425.0966796875, + "loss/core": 4425.09033203125, + "loss/preference_sft": 0.26733091473579407, + "loss/reference_anchor": 0.29540133476257324, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 2.780134439468384, + "rewards/margins": 1.4746670722961426, + "rewards/rejected": 1.3054672479629517, + "step": 410 + }, + { + "drift/chosen_abs": 0.23144730925559998, + "drift/rejected_abs": 0.07132745534181595, + "epoch": 0.3965126000238863, + "grad_norm": 7296.0, + "learning_rate": 1.636271242968684e-07, + "logits/chosen": -2.039712429046631, + "logits/rejected": -2.0561647415161133, + "logps/chosen": -0.2998116612434387, + "logps/rejected": -0.28820380568504333, + "loss": 4422.5302734375, + "loss/core": 4422.5244140625, + "loss/preference_sft": 0.2998116612434387, + "loss/reference_anchor": 0.21635302901268005, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.3144731521606445, + "rewards/margins": 1.6700124740600586, + "rewards/rejected": 0.644460916519165, + "step": 415 + }, + { + "drift/chosen_abs": 0.32540231943130493, + "drift/rejected_abs": 0.18714219331741333, + "epoch": 0.40128986026513797, + "grad_norm": 900.0, + "learning_rate": 1.6131457970085684e-07, + "logits/chosen": -1.9318962097167969, + "logits/rejected": -1.9521112442016602, + "logps/chosen": -0.284771591424942, + "logps/rejected": -0.2870418131351471, + "loss": 4426.5439453125, + "loss/core": 4426.5341796875, + "loss/preference_sft": 0.284771591424942, + "loss/reference_anchor": 0.46427035331726074, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.2529983520507812, + "rewards/margins": 1.3839852809906006, + "rewards/rejected": 1.8690131902694702, + "step": 420 + }, + { + "drift/chosen_abs": 0.2929674983024597, + "drift/rejected_abs": 0.1980661153793335, + "epoch": 0.4060671205063896, + "grad_norm": 1904.0, + "learning_rate": 1.5898837867657727e-07, + "logits/chosen": -1.9516912698745728, + "logits/rejected": -1.9994312524795532, + "logps/chosen": -0.28913792967796326, + "logps/rejected": -0.27645057439804077, + "loss": 4432.4873046875, + "loss/core": 4432.4775390625, + "loss/preference_sft": 0.28913792967796326, + "loss/reference_anchor": 0.4840005040168762, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.9288175106048584, + "rewards/margins": 0.9494360685348511, + "rewards/rejected": 1.9793813228607178, + "step": 425 + }, + { + "drift/chosen_abs": 0.25099247694015503, + "drift/rejected_abs": 0.0831591859459877, + "epoch": 0.41084438074764124, + "grad_norm": 668.0, + "learning_rate": 1.5664939601310023e-07, + "logits/chosen": -2.182654857635498, + "logits/rejected": -2.2524609565734863, + "logps/chosen": -0.3082042336463928, + "logps/rejected": -0.30552542209625244, + "loss": 4422.7587890625, + "loss/core": 4422.75, + "loss/preference_sft": 0.3082042336463928, + "loss/reference_anchor": 0.41477394104003906, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.5072789192199707, + "rewards/margins": 1.676422357559204, + "rewards/rejected": 0.830856442451477, + "step": 430 + }, + { + "drift/chosen_abs": 0.24665072560310364, + "drift/rejected_abs": 0.12369408458471298, + "epoch": 0.41562164098889287, + "grad_norm": 1112.0, + "learning_rate": 1.5429851130614807e-07, + "logits/chosen": -2.1079397201538086, + "logits/rejected": -2.0382118225097656, + "logps/chosen": -0.28440576791763306, + "logps/rejected": -0.26352208852767944, + "loss": 4428.2412109375, + "loss/core": 4428.23681640625, + "loss/preference_sft": 0.28440576791763306, + "loss/reference_anchor": 0.22342030704021454, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.4649205207824707, + "rewards/margins": 1.229503870010376, + "rewards/rejected": 1.2354166507720947, + "step": 435 + }, + { + "drift/chosen_abs": 0.22085194289684296, + "drift/rejected_abs": 0.08876828849315643, + "epoch": 0.4203989012301445, + "grad_norm": 4608.0, + "learning_rate": 1.51936608627315e-07, + "logits/chosen": -2.0002598762512207, + "logits/rejected": -2.0259335041046143, + "logps/chosen": -0.3116574287414551, + "logps/rejected": -0.31580719351768494, + "loss": 4426.97900390625, + "loss/core": 4426.974609375, + "loss/preference_sft": 0.3116574287414551, + "loss/reference_anchor": 0.183195561170578, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.205994129180908, + "rewards/margins": 1.3205196857452393, + "rewards/rejected": 0.8854740858078003, + "step": 440 + }, + { + "drift/chosen_abs": 0.37103694677352905, + "drift/rejected_abs": 0.18114325404167175, + "epoch": 0.42517616147139614, + "grad_norm": 7104.0, + "learning_rate": 1.4956457619160375e-07, + "logits/chosen": -2.050297975540161, + "logits/rejected": -2.0345191955566406, + "logps/chosen": -0.27982237935066223, + "logps/rejected": -0.25793343782424927, + "loss": 4420.06103515625, + "loss/core": 4420.046875, + "loss/preference_sft": 0.27982237935066223, + "loss/reference_anchor": 0.6739336252212524, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.709260940551758, + "rewards/margins": 1.9022051095962524, + "rewards/rejected": 1.8070557117462158, + "step": 445 + }, + { + "drift/chosen_abs": 0.19236549735069275, + "drift/rejected_abs": 0.08619394898414612, + "epoch": 0.4299534217126478, + "grad_norm": 768.0, + "learning_rate": 1.471833060234044e-07, + "logits/chosen": -2.10903263092041, + "logits/rejected": -2.172616481781006, + "logps/chosen": -0.30059614777565, + "logps/rejected": -0.2986820936203003, + "loss": 4430.38623046875, + "loss/core": 4430.3837890625, + "loss/preference_sft": 0.30059614777565, + "loss/reference_anchor": 0.12340258061885834, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9236551523208618, + "rewards/margins": 1.0623431205749512, + "rewards/rejected": 0.8613119125366211, + "step": 450 + }, + { + "drift/chosen_abs": 0.21780920028686523, + "drift/rejected_abs": 0.13248507678508759, + "epoch": 0.4347306819538994, + "grad_norm": 1728.0, + "learning_rate": 1.4479369362104037e-07, + "logits/chosen": -2.1391212940216064, + "logits/rejected": -2.252551555633545, + "logps/chosen": -0.26534712314605713, + "logps/rejected": -0.27983564138412476, + "loss": 4433.15234375, + "loss/core": 4433.1484375, + "loss/preference_sft": 0.26534712314605713, + "loss/reference_anchor": 0.1820131540298462, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.1771018505096436, + "rewards/margins": 0.8640018701553345, + "rewards/rejected": 1.313099980354309, + "step": 455 + }, + { + "drift/chosen_abs": 0.20761752128601074, + "drift/rejected_abs": 0.09282220900058746, + "epoch": 0.4395079421951511, + "grad_norm": 652.0, + "learning_rate": 1.4239663762000817e-07, + "logits/chosen": -2.139101505279541, + "logits/rejected": -2.070023536682129, + "logps/chosen": -0.2922465205192566, + "logps/rejected": -0.29120540618896484, + "loss": 4429.29296875, + "loss/core": 4429.28857421875, + "loss/preference_sft": 0.2922465205192566, + "loss/reference_anchor": 0.17264321446418762, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.0735580921173096, + "rewards/margins": 1.1463443040847778, + "rewards/rejected": 0.9272136688232422, + "step": 460 + }, + { + "drift/chosen_abs": 0.2610882520675659, + "drift/rejected_abs": 0.143789604306221, + "epoch": 0.44428520243640274, + "grad_norm": 10240.0, + "learning_rate": 1.3999303945503747e-07, + "logits/chosen": -2.1653521060943604, + "logits/rejected": -2.247696876525879, + "logps/chosen": -0.31048905849456787, + "logps/rejected": -0.31169334053993225, + "loss": 4429.36865234375, + "loss/core": 4429.359375, + "loss/preference_sft": 0.31048905849456787, + "loss/reference_anchor": 0.44403213262557983, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.610751152038574, + "rewards/margins": 1.1738497018814087, + "rewards/rejected": 1.4369014501571655, + "step": 465 + }, + { + "drift/chosen_abs": 0.15377414226531982, + "drift/rejected_abs": 0.10233066231012344, + "epoch": 0.4490624626776544, + "grad_norm": 884.0, + "learning_rate": 1.3758380302109808e-07, + "logits/chosen": -2.145686626434326, + "logits/rejected": -2.1528048515319824, + "logps/chosen": -0.2958281636238098, + "logps/rejected": -0.26482534408569336, + "loss": 4437.5927734375, + "loss/core": 4437.5908203125, + "loss/preference_sft": 0.2958281636238098, + "loss/reference_anchor": 0.08905575424432755, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.5374128818511963, + "rewards/margins": 0.5175694227218628, + "rewards/rejected": 1.019843578338623, + "step": 470 + }, + { + "drift/chosen_abs": 0.19338826835155487, + "drift/rejected_abs": 0.05476611852645874, + "epoch": 0.453839722918906, + "grad_norm": 520.0, + "learning_rate": 1.351698343334823e-07, + "logits/chosen": -2.10408091545105, + "logits/rejected": -2.1622872352600098, + "logps/chosen": -0.3007555603981018, + "logps/rejected": -0.29687556624412537, + "loss": 4426.18212890625, + "loss/core": 4426.17724609375, + "loss/preference_sft": 0.3007555603981018, + "loss/reference_anchor": 0.22863420844078064, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.9303607940673828, + "rewards/margins": 1.399163842201233, + "rewards/rejected": 0.5311970114707947, + "step": 475 + }, + { + "drift/chosen_abs": 0.24555771052837372, + "drift/rejected_abs": 0.133039191365242, + "epoch": 0.45861698316015764, + "grad_norm": 5504.0, + "learning_rate": 1.3275204118708942e-07, + "logits/chosen": -2.0921359062194824, + "logits/rejected": -1.9713423252105713, + "logps/chosen": -0.301283597946167, + "logps/rejected": -0.31479784846305847, + "loss": 4430.07861328125, + "loss/core": 4430.0654296875, + "loss/preference_sft": 0.301283597946167, + "loss/reference_anchor": 0.6136071681976318, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.4555182456970215, + "rewards/margins": 1.125307559967041, + "rewards/rejected": 1.33021080493927, + "step": 480 + }, + { + "drift/chosen_abs": 0.20899991691112518, + "drift/rejected_abs": 0.10918338596820831, + "epoch": 0.4633942434014093, + "grad_norm": 2944.0, + "learning_rate": 1.3033133281504132e-07, + "logits/chosen": -1.9529765844345093, + "logits/rejected": -1.9704357385635376, + "logps/chosen": -0.29765480756759644, + "logps/rejected": -0.296121209859848, + "loss": 4431.419921875, + "loss/core": 4431.4150390625, + "loss/preference_sft": 0.29765480756759644, + "loss/reference_anchor": 0.23028752207756042, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.089999198913574, + "rewards/margins": 1.000409483909607, + "rewards/rejected": 1.0895897150039673, + "step": 485 + }, + { + "drift/chosen_abs": 0.23929134011268616, + "drift/rejected_abs": 0.11025607585906982, + "epoch": 0.4681715036426609, + "grad_norm": 980.0, + "learning_rate": 1.2790861954675702e-07, + "logits/chosen": -2.0607128143310547, + "logits/rejected": -2.0974857807159424, + "logps/chosen": -0.3084251582622528, + "logps/rejected": -0.30655646324157715, + "loss": 4427.3017578125, + "loss/core": 4427.2978515625, + "loss/preference_sft": 0.3084251582622528, + "loss/reference_anchor": 0.14670784771442413, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.3920226097106934, + "rewards/margins": 1.2922639846801758, + "rewards/rejected": 1.0997586250305176, + "step": 490 + }, + { + "drift/chosen_abs": 0.19296832382678986, + "drift/rejected_abs": 0.09996967762708664, + "epoch": 0.47294876388391255, + "grad_norm": 1216.0, + "learning_rate": 1.2548481246561504e-07, + "logits/chosen": -2.0633440017700195, + "logits/rejected": -2.0872926712036133, + "logps/chosen": -0.2833230495452881, + "logps/rejected": -0.28482872247695923, + "loss": 4432.07861328125, + "loss/core": 4432.0751953125, + "loss/preference_sft": 0.2833230495452881, + "loss/reference_anchor": 0.14006902277469635, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.929682970046997, + "rewards/margins": 0.9321301579475403, + "rewards/rejected": 0.9975531697273254, + "step": 495 + }, + { + "drift/chosen_abs": 0.36558204889297485, + "drift/rejected_abs": 0.1663735955953598, + "epoch": 0.47772602412516424, + "grad_norm": 5088.0, + "learning_rate": 1.230608230663321e-07, + "logits/chosen": -2.070899486541748, + "logits/rejected": -2.0423977375030518, + "logps/chosen": -0.2811706066131592, + "logps/rejected": -0.28334346413612366, + "loss": 4418.2373046875, + "loss/core": 4418.22265625, + "loss/preference_sft": 0.2811706066131592, + "loss/reference_anchor": 0.7044563889503479, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.652653455734253, + "rewards/margins": 1.9970003366470337, + "rewards/rejected": 1.6556532382965088, + "step": 500 + }, + { + "drift/chosen_abs": 0.34318313002586365, + "drift/rejected_abs": 0.13852709531784058, + "epoch": 0.4825032843664159, + "grad_norm": 816.0, + "learning_rate": 1.206375629121874e-07, + "logits/chosen": -2.0183844566345215, + "logits/rejected": -2.008913278579712, + "logps/chosen": -0.3097713589668274, + "logps/rejected": -0.31553733348846436, + "loss": 4417.1552734375, + "loss/core": 4417.146484375, + "loss/preference_sft": 0.3097713589668274, + "loss/reference_anchor": 0.4379523694515228, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.4314303398132324, + "rewards/margins": 2.0832791328430176, + "rewards/rejected": 1.348151683807373, + "step": 505 + }, + { + "drift/chosen_abs": 0.2345653474330902, + "drift/rejected_abs": 0.12095403671264648, + "epoch": 0.4872805446076675, + "grad_norm": 1072.0, + "learning_rate": 1.1821594329222079e-07, + "logits/chosen": -2.2127575874328613, + "logits/rejected": -2.1563878059387207, + "logps/chosen": -0.2902802526950836, + "logps/rejected": -0.28386133909225464, + "loss": 4429.57666015625, + "loss/core": 4429.5654296875, + "loss/preference_sft": 0.2902802526950836, + "loss/reference_anchor": 0.5237426161766052, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.3436882495880127, + "rewards/margins": 1.1352647542953491, + "rewards/rejected": 1.208423376083374, + "step": 510 + }, + { + "drift/chosen_abs": 0.18333016335964203, + "drift/rejected_abs": 0.1809997260570526, + "epoch": 0.49205780484891914, + "grad_norm": 2576.0, + "learning_rate": 1.157968748785344e-07, + "logits/chosen": -1.914223074913025, + "logits/rejected": -1.981766939163208, + "logps/chosen": -0.2812362015247345, + "logps/rejected": -0.27950748801231384, + "loss": 4444.3720703125, + "loss/core": 4444.36767578125, + "loss/preference_sft": 0.2812362015247345, + "loss/reference_anchor": 0.19075706601142883, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.8306372165679932, + "rewards/margins": 0.022197848185896873, + "rewards/rejected": 1.8084392547607422, + "step": 515 + }, + { + "drift/chosen_abs": 0.3202487826347351, + "drift/rejected_abs": 0.13263753056526184, + "epoch": 0.4968350650901708, + "grad_norm": 2064.0, + "learning_rate": 1.1338126738382597e-07, + "logits/chosen": -1.9274823665618896, + "logits/rejected": -1.931096076965332, + "logps/chosen": -0.2886351943016052, + "logps/rejected": -0.2917082607746124, + "loss": 4419.962890625, + "loss/core": 4419.9501953125, + "loss/preference_sft": 0.2886351943016052, + "loss/reference_anchor": 0.6030197143554688, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.202364444732666, + "rewards/margins": 1.8788830041885376, + "rewards/rejected": 1.3234814405441284, + "step": 520 + }, + { + "drift/chosen_abs": 0.21438315510749817, + "drift/rejected_abs": 0.07398609071969986, + "epoch": 0.5016123253314224, + "grad_norm": 604.0, + "learning_rate": 1.1097002921928316e-07, + "logits/chosen": -2.086273670196533, + "logits/rejected": -2.169018268585205, + "logps/chosen": -0.2757102847099304, + "logps/rejected": -0.27384692430496216, + "loss": 4425.99462890625, + "loss/core": 4425.9892578125, + "loss/preference_sft": 0.2757102847099304, + "loss/reference_anchor": 0.19271975755691528, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.142575740814209, + "rewards/margins": 1.4045182466506958, + "rewards/rejected": 0.7380577921867371, + "step": 525 + }, + { + "drift/chosen_abs": 0.16398575901985168, + "drift/rejected_abs": 0.08183001726865768, + "epoch": 0.5063895855726741, + "grad_norm": 556.0, + "learning_rate": 1.0856406715296717e-07, + "logits/chosen": -2.1869404315948486, + "logits/rejected": -2.284536361694336, + "logps/chosen": -0.2831934988498688, + "logps/rejected": -0.2927997410297394, + "loss": 4433.50634765625, + "loss/core": 4433.50341796875, + "loss/preference_sft": 0.2831934988498688, + "loss/reference_anchor": 0.07691037654876709, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.6397440433502197, + "rewards/margins": 0.822593092918396, + "rewards/rejected": 0.817150890827179, + "step": 530 + }, + { + "drift/chosen_abs": 0.3339373767375946, + "drift/rejected_abs": 0.12080500274896622, + "epoch": 0.5111668458139257, + "grad_norm": 564.0, + "learning_rate": 1.061642859688146e-07, + "logits/chosen": -2.11379075050354, + "logits/rejected": -2.042389392852783, + "logps/chosen": -0.2667238712310791, + "logps/rejected": -0.27604439854621887, + "loss": 4416.22265625, + "loss/core": 4416.216796875, + "loss/preference_sft": 0.2667238712310791, + "loss/reference_anchor": 0.2798369526863098, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.339317798614502, + "rewards/margins": 2.1484532356262207, + "rewards/rejected": 1.1908643245697021, + "step": 535 + }, + { + "drift/chosen_abs": 0.2929610311985016, + "drift/rejected_abs": 0.08645624667406082, + "epoch": 0.5159441060551774, + "grad_norm": 2688.0, + "learning_rate": 1.0377158812638491e-07, + "logits/chosen": -2.1425912380218506, + "logits/rejected": -2.234184980392456, + "logps/chosen": -0.29007601737976074, + "logps/rejected": -0.280531108379364, + "loss": 4416.90625, + "loss/core": 4416.8984375, + "loss/preference_sft": 0.29007601737976074, + "loss/reference_anchor": 0.3554505705833435, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.9293251037597656, + "rewards/margins": 2.1080482006073, + "rewards/rejected": 0.821276843547821, + "step": 540 + }, + { + "drift/chosen_abs": 0.2901972234249115, + "drift/rejected_abs": 0.14684346318244934, + "epoch": 0.520721366296429, + "grad_norm": 3792.0, + "learning_rate": 1.0138687342148249e-07, + "logits/chosen": -2.062620162963867, + "logits/rejected": -2.1082162857055664, + "logps/chosen": -0.30190232396125793, + "logps/rejected": -0.32236361503601074, + "loss": 4426.2060546875, + "loss/core": 4426.193359375, + "loss/preference_sft": 0.30190232396125793, + "loss/reference_anchor": 0.6151673197746277, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.90141224861145, + "rewards/margins": 1.4344381093978882, + "rewards/rejected": 1.4669743776321411, + "step": 545 + }, + { + "drift/chosen_abs": 0.24899080395698547, + "drift/rejected_abs": 0.20708198845386505, + "epoch": 0.5254986265376806, + "grad_norm": 592.0, + "learning_rate": 9.90110386477801e-08, + "logits/chosen": -2.1318652629852295, + "logits/rejected": -2.2062549591064453, + "logps/chosen": -0.25858795642852783, + "logps/rejected": -0.253892183303833, + "loss": 4438.9169921875, + "loss/core": 4438.9072265625, + "loss/preference_sft": 0.25858795642852783, + "loss/reference_anchor": 0.47126907110214233, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.489907741546631, + "rewards/margins": 0.4215244650840759, + "rewards/rejected": 2.0683834552764893, + "step": 550 + }, + { + "drift/chosen_abs": 0.21308758854866028, + "drift/rejected_abs": 0.15445470809936523, + "epoch": 0.5302758867789323, + "grad_norm": 1928.0, + "learning_rate": 9.664497725957164e-08, + "logits/chosen": -2.1934444904327393, + "logits/rejected": -2.2147269248962402, + "logps/chosen": -0.2912050783634186, + "logps/rejected": -0.2784259617328644, + "loss": 4436.732421875, + "loss/core": 4436.72607421875, + "loss/preference_sft": 0.2912050783634186, + "loss/reference_anchor": 0.23212206363677979, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.127945899963379, + "rewards/margins": 0.5856080651283264, + "rewards/rejected": 1.5423381328582764, + "step": 555 + }, + { + "drift/chosen_abs": 0.2658155560493469, + "drift/rejected_abs": 0.19533106684684753, + "epoch": 0.5350531470201839, + "grad_norm": 7616.0, + "learning_rate": 9.428957903578045e-08, + "logits/chosen": -1.9657647609710693, + "logits/rejected": -1.980133295059204, + "logps/chosen": -0.2851273715496063, + "logps/rejected": -0.28451281785964966, + "loss": 4435.39208984375, + "loss/core": 4435.38037109375, + "loss/preference_sft": 0.2851273715496063, + "loss/reference_anchor": 0.557953417301178, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.6540966033935547, + "rewards/margins": 0.7040950655937195, + "rewards/rejected": 1.9500014781951904, + "step": 560 + }, + { + "drift/chosen_abs": 0.34854352474212646, + "drift/rejected_abs": 0.19613364338874817, + "epoch": 0.5398304072614356, + "grad_norm": 696.0, + "learning_rate": 9.194572974534976e-08, + "logits/chosen": -2.024336338043213, + "logits/rejected": -2.093052864074707, + "logps/chosen": -0.27176398038864136, + "logps/rejected": -0.26332786679267883, + "loss": 4424.57861328125, + "loss/core": 4424.56201171875, + "loss/preference_sft": 0.27176398038864136, + "loss/reference_anchor": 0.8150361776351929, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.4846279621124268, + "rewards/margins": 1.5311681032180786, + "rewards/rejected": 1.9534600973129272, + "step": 565 + }, + { + "drift/chosen_abs": 0.227237731218338, + "drift/rejected_abs": 0.16712255775928497, + "epoch": 0.5446076675026872, + "grad_norm": 4416.0, + "learning_rate": 8.96143108141412e-08, + "logits/chosen": -1.9618107080459595, + "logits/rejected": -1.954664945602417, + "logps/chosen": -0.2768324613571167, + "logps/rejected": -0.27417004108428955, + "loss": 4436.580078125, + "loss/core": 4436.5751953125, + "loss/preference_sft": 0.2768324613571167, + "loss/reference_anchor": 0.21785183250904083, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.271784543991089, + "rewards/margins": 0.6015557050704956, + "rewards/rejected": 1.670229196548462, + "step": 570 + }, + { + "drift/chosen_abs": 0.22515693306922913, + "drift/rejected_abs": 0.08775409311056137, + "epoch": 0.5493849277439389, + "grad_norm": 360.0, + "learning_rate": 8.72961989934668e-08, + "logits/chosen": -2.231755495071411, + "logits/rejected": -2.245164632797241, + "logps/chosen": -0.2810257077217102, + "logps/rejected": -0.28133219480514526, + "loss": 4426.43994140625, + "loss/core": 4426.4345703125, + "loss/preference_sft": 0.2810257077217102, + "loss/reference_anchor": 0.2272057831287384, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.2512524127960205, + "rewards/margins": 1.3741371631622314, + "rewards/rejected": 0.8771151304244995, + "step": 575 + }, + { + "drift/chosen_abs": 0.14085161685943604, + "drift/rejected_abs": 0.1031990498304367, + "epoch": 0.5541621879851905, + "grad_norm": 720.0, + "learning_rate": 8.499226603037854e-08, + "logits/chosen": -2.2430803775787354, + "logits/rejected": -2.2786197662353516, + "logps/chosen": -0.2992137372493744, + "logps/rejected": -0.29944515228271484, + "loss": 4439.31689453125, + "loss/core": 4439.31396484375, + "loss/preference_sft": 0.2992137372493744, + "loss/reference_anchor": 0.10587255656719208, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.407078742980957, + "rewards/margins": 0.39435142278671265, + "rewards/rejected": 1.0127273797988892, + "step": 580 + }, + { + "drift/chosen_abs": 0.22692131996154785, + "drift/rejected_abs": 0.10220160335302353, + "epoch": 0.5589394482264421, + "grad_norm": 608.0, + "learning_rate": 8.270337833983987e-08, + "logits/chosen": -2.0355045795440674, + "logits/rejected": -2.1195898056030273, + "logps/chosen": -0.30346351861953735, + "logps/rejected": -0.2966625392436981, + "loss": 4427.97900390625, + "loss/core": 4427.97216796875, + "loss/preference_sft": 0.30346351861953735, + "loss/reference_anchor": 0.29685312509536743, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.2688138484954834, + "rewards/margins": 1.247382402420044, + "rewards/rejected": 1.0214314460754395, + "step": 585 + }, + { + "drift/chosen_abs": 0.21154217422008514, + "drift/rejected_abs": 0.13458245992660522, + "epoch": 0.5637167084676937, + "grad_norm": 1672.0, + "learning_rate": 8.04303966789025e-08, + "logits/chosen": -2.1316933631896973, + "logits/rejected": -2.150923252105713, + "logps/chosen": -0.27440136671066284, + "logps/rejected": -0.27802544832229614, + "loss": 4434.2373046875, + "loss/core": 4434.2314453125, + "loss/preference_sft": 0.27440136671066284, + "loss/reference_anchor": 0.25449132919311523, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.114384889602661, + "rewards/margins": 0.7690680623054504, + "rewards/rejected": 1.3453166484832764, + "step": 590 + }, + { + "drift/chosen_abs": 0.21570686995983124, + "drift/rejected_abs": 0.11859677731990814, + "epoch": 0.5684939687089454, + "grad_norm": 1072.0, + "learning_rate": 7.817417582301098e-08, + "logits/chosen": -1.8743267059326172, + "logits/rejected": -1.8942207098007202, + "logps/chosen": -0.284064382314682, + "logps/rejected": -0.28866657614707947, + "loss": 4431.5322265625, + "loss/core": 4431.529296875, + "loss/preference_sft": 0.284064382314682, + "loss/reference_anchor": 0.1300676316022873, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.1570687294006348, + "rewards/margins": 0.9721587896347046, + "rewards/rejected": 1.1849100589752197, + "step": 595 + }, + { + "drift/chosen_abs": 0.19431225955486298, + "drift/rejected_abs": 0.1243596076965332, + "epoch": 0.5732712289501971, + "grad_norm": 1800.0, + "learning_rate": 7.59355642445566e-08, + "logits/chosen": -2.2408175468444824, + "logits/rejected": -2.15144419670105, + "logps/chosen": -0.26184308528900146, + "logps/rejected": -0.2716737985610962, + "loss": 4434.80078125, + "loss/core": 4434.79736328125, + "loss/preference_sft": 0.26184308528900146, + "loss/reference_anchor": 0.16086336970329285, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9430396556854248, + "rewards/margins": 0.7340011596679688, + "rewards/rejected": 1.2090386152267456, + "step": 600 + }, + { + "drift/chosen_abs": 0.21506313979625702, + "drift/rejected_abs": 0.09374111890792847, + "epoch": 0.5780484891914487, + "grad_norm": 7328.0, + "learning_rate": 7.37154037938015e-08, + "logits/chosen": -2.073078155517578, + "logits/rejected": -2.174865245819092, + "logps/chosen": -0.3253878951072693, + "logps/rejected": -0.33154183626174927, + "loss": 4428.4287109375, + "loss/core": 4428.4248046875, + "loss/preference_sft": 0.3253878951072693, + "loss/reference_anchor": 0.17495615780353546, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.148521900177002, + "rewards/margins": 1.2160072326660156, + "rewards/rejected": 0.9325143694877625, + "step": 605 + }, + { + "drift/chosen_abs": 0.21458843350410461, + "drift/rejected_abs": 0.11676479876041412, + "epoch": 0.5828257494327004, + "grad_norm": 1976.0, + "learning_rate": 7.151452938229325e-08, + "logits/chosen": -2.0549139976501465, + "logits/rejected": -2.0558762550354004, + "logps/chosen": -0.2830585837364197, + "logps/rejected": -0.29241445660591125, + "loss": 4431.64208984375, + "loss/core": 4431.63623046875, + "loss/preference_sft": 0.2830585837364197, + "loss/reference_anchor": 0.25421541929244995, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.145282745361328, + "rewards/margins": 0.9789865612983704, + "rewards/rejected": 1.166296362876892, + "step": 610 + }, + { + "drift/chosen_abs": 0.3049691319465637, + "drift/rejected_abs": 0.16369685530662537, + "epoch": 0.587603009673952, + "grad_norm": 11968.0, + "learning_rate": 6.933376866888883e-08, + "logits/chosen": -2.038628339767456, + "logits/rejected": -2.1392931938171387, + "logps/chosen": -0.2770799994468689, + "logps/rejected": -0.3140313923358917, + "loss": 4426.2294921875, + "loss/core": 4426.2177734375, + "loss/preference_sft": 0.2770799994468689, + "loss/reference_anchor": 0.5493339896202087, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.0482850074768066, + "rewards/margins": 1.4278751611709595, + "rewards/rejected": 1.6204099655151367, + "step": 615 + }, + { + "drift/chosen_abs": 0.26089203357696533, + "drift/rejected_abs": 0.1435510367155075, + "epoch": 0.5923802699152036, + "grad_norm": 15232.0, + "learning_rate": 6.717394174850605e-08, + "logits/chosen": -2.150646924972534, + "logits/rejected": -2.166919708251953, + "logps/chosen": -0.2822549045085907, + "logps/rejected": -0.29912179708480835, + "loss": 4429.2724609375, + "loss/core": 4429.2646484375, + "loss/preference_sft": 0.2822549045085907, + "loss/reference_anchor": 0.33313748240470886, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.6065964698791504, + "rewards/margins": 1.1729382276535034, + "rewards/rejected": 1.433658480644226, + "step": 620 + }, + { + "drift/chosen_abs": 0.19429583847522736, + "drift/rejected_abs": 0.1452254056930542, + "epoch": 0.5971575301564552, + "grad_norm": 1168.0, + "learning_rate": 6.503586084371926e-08, + "logits/chosen": -2.172908306121826, + "logits/rejected": -2.1675751209259033, + "logps/chosen": -0.2829391360282898, + "logps/rejected": -0.2997308373451233, + "loss": 4438.06396484375, + "loss/core": 4438.0595703125, + "loss/preference_sft": 0.2829391360282898, + "loss/reference_anchor": 0.17451608180999756, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9426425695419312, + "rewards/margins": 0.4928255081176758, + "rewards/rejected": 1.4498172998428345, + "step": 625 + }, + { + "drift/chosen_abs": 0.2902236878871918, + "drift/rejected_abs": 0.13140039145946503, + "epoch": 0.6019347903977069, + "grad_norm": 2480.0, + "learning_rate": 6.29203299993155e-08, + "logits/chosen": -2.1176817417144775, + "logits/rejected": -2.0981597900390625, + "logps/chosen": -0.30078643560409546, + "logps/rejected": -0.3083663880825043, + "loss": 4419.7548828125, + "loss/core": 4419.74609375, + "loss/preference_sft": 0.30078643560409546, + "loss/reference_anchor": 0.43593829870224, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.901242256164551, + "rewards/margins": 1.8964433670043945, + "rewards/rejected": 1.0047987699508667, + "step": 630 + }, + { + "drift/chosen_abs": 0.17258086800575256, + "drift/rejected_abs": 0.1321617066860199, + "epoch": 0.6067120506389586, + "grad_norm": 3680.0, + "learning_rate": 6.082814477992656e-08, + "logits/chosen": -2.1916544437408447, + "logits/rejected": -2.11995530128479, + "logps/chosen": -0.30291134119033813, + "logps/rejected": -0.306274950504303, + "loss": 4439.31982421875, + "loss/core": 4439.31494140625, + "loss/preference_sft": 0.30291134119033813, + "loss/reference_anchor": 0.23412831127643585, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.723780632019043, + "rewards/margins": 0.4066292643547058, + "rewards/rejected": 1.317151427268982, + "step": 635 + }, + { + "drift/chosen_abs": 0.1660202294588089, + "drift/rejected_abs": 0.06376107037067413, + "epoch": 0.6114893108802102, + "grad_norm": 1200.0, + "learning_rate": 5.87600919708494e-08, + "logits/chosen": -2.0012378692626953, + "logits/rejected": -2.0457763671875, + "logps/chosen": -0.28748735785484314, + "logps/rejected": -0.2896059453487396, + "loss": 4430.99169921875, + "loss/core": 4430.98876953125, + "loss/preference_sft": 0.28748735785484314, + "loss/reference_anchor": 0.1316143423318863, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.6601464748382568, + "rewards/margins": 1.0247886180877686, + "rewards/rejected": 0.6353579163551331, + "step": 640 + }, + { + "drift/chosen_abs": 0.203060582280159, + "drift/rejected_abs": 0.18572744727134705, + "epoch": 0.6162665711214619, + "grad_norm": 5792.0, + "learning_rate": 5.671694928216829e-08, + "logits/chosen": -2.225250244140625, + "logits/rejected": -2.2663300037384033, + "logps/chosen": -0.28407204151153564, + "logps/rejected": -0.27350106835365295, + "loss": 4442.513671875, + "loss/core": 4442.50830078125, + "loss/preference_sft": 0.28407204151153564, + "loss/reference_anchor": 0.25616979598999023, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.0303478240966797, + "rewards/margins": 0.17582125961780548, + "rewards/rejected": 1.8545265197753906, + "step": 645 + }, + { + "drift/chosen_abs": 0.3154805302619934, + "drift/rejected_abs": 0.2100723534822464, + "epoch": 0.6210438313627135, + "grad_norm": 6400.0, + "learning_rate": 5.469948505629e-08, + "logits/chosen": -2.017364263534546, + "logits/rejected": -2.079219341278076, + "logps/chosen": -0.28993043303489685, + "logps/rejected": -0.28738412261009216, + "loss": 4430.4951171875, + "loss/core": 4430.47998046875, + "loss/preference_sft": 0.28993043303489685, + "loss/reference_anchor": 0.7488113641738892, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.1542510986328125, + "rewards/margins": 1.0541503429412842, + "rewards/rejected": 2.1001007556915283, + "step": 650 + }, + { + "drift/chosen_abs": 0.1768004447221756, + "drift/rejected_abs": 0.12505300343036652, + "epoch": 0.6258210916039652, + "grad_norm": 8960.0, + "learning_rate": 5.270845797900168e-08, + "logits/chosen": -2.223820209503174, + "logits/rejected": -2.153284788131714, + "logps/chosen": -0.28208497166633606, + "logps/rejected": -0.29163843393325806, + "loss": 4437.68603515625, + "loss/core": 4437.68017578125, + "loss/preference_sft": 0.28208497166633606, + "loss/reference_anchor": 0.26417121291160583, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.765069603919983, + "rewards/margins": 0.5174070596694946, + "rewards/rejected": 1.2476627826690674, + "step": 655 + }, + { + "drift/chosen_abs": 0.18167448043823242, + "drift/rejected_abs": 0.14043021202087402, + "epoch": 0.6305983518452167, + "grad_norm": 524.0, + "learning_rate": 5.0744616794160104e-08, + "logits/chosen": -2.0674781799316406, + "logits/rejected": -2.1492257118225098, + "logps/chosen": -0.27626532316207886, + "logps/rejected": -0.28123727440834045, + "loss": 4439.0361328125, + "loss/core": 4439.03369140625, + "loss/preference_sft": 0.27626532316207886, + "loss/reference_anchor": 0.11728350073099136, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.8160436153411865, + "rewards/margins": 0.41351088881492615, + "rewards/rejected": 1.402532935142517, + "step": 660 + }, + { + "drift/chosen_abs": 0.22600972652435303, + "drift/rejected_abs": 0.1264456957578659, + "epoch": 0.6353756120864684, + "grad_norm": 796.0, + "learning_rate": 4.880870002211963e-08, + "logits/chosen": -2.2491302490234375, + "logits/rejected": -2.25697660446167, + "logps/chosen": -0.27793681621551514, + "logps/rejected": -0.25179094076156616, + "loss": 4431.15283203125, + "loss/core": 4431.146484375, + "loss/preference_sft": 0.27793681621551514, + "loss/reference_anchor": 0.2595617473125458, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.259474754333496, + "rewards/margins": 1.0019110441207886, + "rewards/rejected": 1.2575637102127075, + "step": 665 + }, + { + "drift/chosen_abs": 0.13364611566066742, + "drift/rejected_abs": 0.09645704180002213, + "epoch": 0.64015287232772, + "grad_norm": 540.0, + "learning_rate": 4.6901435682004996e-08, + "logits/chosen": -2.2417209148406982, + "logits/rejected": -2.2763659954071045, + "logps/chosen": -0.29147717356681824, + "logps/rejected": -0.2934878468513489, + "loss": 4439.51708984375, + "loss/core": 4439.515625, + "loss/preference_sft": 0.29147717356681824, + "loss/reference_anchor": 0.06112232059240341, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.3360573053359985, + "rewards/margins": 0.3732479214668274, + "rewards/rejected": 0.9628093838691711, + "step": 670 + }, + { + "drift/chosen_abs": 0.1628817468881607, + "drift/rejected_abs": 0.13016179203987122, + "epoch": 0.6449301325689717, + "grad_norm": 2032.0, + "learning_rate": 4.502354101793314e-08, + "logits/chosen": -2.3644509315490723, + "logits/rejected": -2.2733235359191895, + "logps/chosen": -0.27977296710014343, + "logps/rejected": -0.29269108176231384, + "loss": 4440.22900390625, + "loss/core": 4440.2255859375, + "loss/preference_sft": 0.27977296710014343, + "loss/reference_anchor": 0.17271290719509125, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6281238794326782, + "rewards/margins": 0.33540040254592896, + "rewards/rejected": 1.2927234172821045, + "step": 675 + }, + { + "drift/chosen_abs": 0.3136964738368988, + "drift/rejected_abs": 0.12950031459331512, + "epoch": 0.6497073928102234, + "grad_norm": 1472.0, + "learning_rate": 4.3175722229287034e-08, + "logits/chosen": -2.1214194297790527, + "logits/rejected": -2.114104747772217, + "logps/chosen": -0.27199044823646545, + "logps/rejected": -0.27481597661972046, + "loss": 4419.1142578125, + "loss/core": 4419.107421875, + "loss/preference_sft": 0.27199044823646545, + "loss/reference_anchor": 0.31056779623031616, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.136345624923706, + "rewards/margins": 1.9250768423080444, + "rewards/rejected": 1.2112689018249512, + "step": 680 + }, + { + "drift/chosen_abs": 0.23225578665733337, + "drift/rejected_abs": 0.23389002680778503, + "epoch": 0.654484653051475, + "grad_norm": 2960.0, + "learning_rate": 4.135867420514276e-08, + "logits/chosen": -2.2221508026123047, + "logits/rejected": -2.2248616218566895, + "logps/chosen": -0.2901459038257599, + "logps/rejected": -0.26247456669807434, + "loss": 4445.1943359375, + "loss/core": 4445.1845703125, + "loss/preference_sft": 0.2901459038257599, + "loss/reference_anchor": 0.46210724115371704, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.3225579261779785, + "rewards/margins": -0.016050493344664574, + "rewards/rejected": 2.3386082649230957, + "step": 685 + }, + { + "drift/chosen_abs": 0.2951485216617584, + "drift/rejected_abs": 0.2140798568725586, + "epoch": 0.6592619132927267, + "grad_norm": 9984.0, + "learning_rate": 3.957308026295035e-08, + "logits/chosen": -2.093303680419922, + "logits/rejected": -2.1656064987182617, + "logps/chosen": -0.31711024045944214, + "logps/rejected": -0.32263869047164917, + "loss": 4435.220703125, + "loss/core": 4435.2021484375, + "loss/preference_sft": 0.31711024045944214, + "loss/reference_anchor": 0.8856789469718933, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.9496922492980957, + "rewards/margins": 0.8108631372451782, + "rewards/rejected": 2.138829469680786, + "step": 690 + }, + { + "drift/chosen_abs": 0.23619715869426727, + "drift/rejected_abs": 0.11902375519275665, + "epoch": 0.6640391735339782, + "grad_norm": 1752.0, + "learning_rate": 3.7819611891566084e-08, + "logits/chosen": -1.9933494329452515, + "logits/rejected": -1.9954357147216797, + "logps/chosen": -0.268182635307312, + "logps/rejected": -0.28130608797073364, + "loss": 4428.77734375, + "loss/core": 4428.77392578125, + "loss/preference_sft": 0.268182635307312, + "loss/reference_anchor": 0.13460581004619598, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.3605830669403076, + "rewards/margins": 1.1809606552124023, + "rewards/rejected": 1.1796222925186157, + "step": 695 + }, + { + "drift/chosen_abs": 0.3273255228996277, + "drift/rejected_abs": 0.16635169088840485, + "epoch": 0.6688164337752299, + "grad_norm": 3360.0, + "learning_rate": 3.609892849873286e-08, + "logits/chosen": -2.2435264587402344, + "logits/rejected": -2.1759636402130127, + "logps/chosen": -0.2879250645637512, + "logps/rejected": -0.2684784531593323, + "loss": 4424.0283203125, + "loss/core": 4424.0146484375, + "loss/preference_sft": 0.2879250645637512, + "loss/reference_anchor": 0.6359601616859436, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.2727675437927246, + "rewards/margins": 1.61063551902771, + "rewards/rejected": 1.6621326208114624, + "step": 700 + }, + { + "drift/chosen_abs": 0.13324177265167236, + "drift/rejected_abs": 0.052085865288972855, + "epoch": 0.6735936940164815, + "grad_norm": 450.0, + "learning_rate": 3.4411677163103894e-08, + "logits/chosen": -2.1316683292388916, + "logits/rejected": -2.1392605304718018, + "logps/chosen": -0.3257368206977844, + "logps/rejected": -0.31321820616722107, + "loss": 4433.72265625, + "loss/core": 4433.72119140625, + "loss/preference_sft": 0.3257368206977844, + "loss/reference_anchor": 0.06571613252162933, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.3298277854919434, + "rewards/margins": 0.8104680180549622, + "rewards/rejected": 0.5193597078323364, + "step": 705 + }, + { + "drift/chosen_abs": 0.24692943692207336, + "drift/rejected_abs": 0.08578932285308838, + "epoch": 0.6783709542577332, + "grad_norm": 1712.0, + "learning_rate": 3.2758492390902945e-08, + "logits/chosen": -1.9040149450302124, + "logits/rejected": -1.964320421218872, + "logps/chosen": -0.278387188911438, + "logps/rejected": -0.28676044940948486, + "loss": 4422.9931640625, + "loss/core": 4422.98876953125, + "loss/preference_sft": 0.278387188911438, + "loss/reference_anchor": 0.17939075827598572, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.4657251834869385, + "rewards/margins": 1.624982237815857, + "rewards/rejected": 0.8407428860664368, + "step": 710 + }, + { + "drift/chosen_abs": 0.2357044667005539, + "drift/rejected_abs": 0.09313996881246567, + "epoch": 0.6831482144989849, + "grad_norm": 884.0, + "learning_rate": 3.11399958773126e-08, + "logits/chosen": -2.0695366859436035, + "logits/rejected": -2.1179842948913574, + "logps/chosen": -0.28330761194229126, + "logps/rejected": -0.2819029688835144, + "loss": 4425.52294921875, + "loss/core": 4425.52001953125, + "loss/preference_sft": 0.28330761194229126, + "loss/reference_anchor": 0.11744773387908936, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.3560948371887207, + "rewards/margins": 1.4276307821273804, + "rewards/rejected": 0.9284639358520508, + "step": 715 + }, + { + "drift/chosen_abs": 0.2549819350242615, + "drift/rejected_abs": 0.11347774416208267, + "epoch": 0.6879254747402365, + "grad_norm": 704.0, + "learning_rate": 2.9556796272679972e-08, + "logits/chosen": -2.1076555252075195, + "logits/rejected": -2.143085479736328, + "logps/chosen": -0.28872913122177124, + "logps/rejected": -0.3133319020271301, + "loss": 4425.2392578125, + "loss/core": 4425.23291015625, + "loss/preference_sft": 0.28872913122177124, + "loss/reference_anchor": 0.2932053208351135, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.549466133117676, + "rewards/margins": 1.4717979431152344, + "rewards/rejected": 1.0776679515838623, + "step": 720 + }, + { + "drift/chosen_abs": 0.11858288198709488, + "drift/rejected_abs": 0.1146119013428688, + "epoch": 0.6927027349814882, + "grad_norm": 396.0, + "learning_rate": 2.8009488953628215e-08, + "logits/chosen": -2.0915627479553223, + "logits/rejected": -2.0317933559417725, + "logps/chosen": -0.2838587760925293, + "logps/rejected": -0.2749176621437073, + "loss": 4444.0947265625, + "loss/core": 4444.091796875, + "loss/preference_sft": 0.2838587760925293, + "loss/reference_anchor": 0.11706352233886719, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.1828110218048096, + "rewards/margins": 0.038224995136260986, + "rewards/rejected": 1.1445858478546143, + "step": 725 + }, + { + "drift/chosen_abs": 0.28038161993026733, + "drift/rejected_abs": 0.10303550958633423, + "epoch": 0.6974799952227397, + "grad_norm": 2032.0, + "learning_rate": 2.649865579915976e-08, + "logits/chosen": -2.084571123123169, + "logits/rejected": -2.0591697692871094, + "logps/chosen": -0.3031243681907654, + "logps/rejected": -0.3205743432044983, + "loss": 4420.45458984375, + "loss/core": 4420.4453125, + "loss/preference_sft": 0.3031243681907654, + "loss/reference_anchor": 0.4167904257774353, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.803816556930542, + "rewards/margins": 1.8526108264923096, + "rewards/rejected": 0.9512056112289429, + "step": 730 + }, + { + "drift/chosen_abs": 0.28037887811660767, + "drift/rejected_abs": 0.11086313426494598, + "epoch": 0.7022572554639914, + "grad_norm": 7264.0, + "learning_rate": 2.5024864971835507e-08, + "logits/chosen": -2.197540760040283, + "logits/rejected": -2.1911709308624268, + "logps/chosen": -0.2654857039451599, + "logps/rejected": -0.27312350273132324, + "loss": 4422.1875, + "loss/core": 4422.1806640625, + "loss/preference_sft": 0.2654857039451599, + "loss/reference_anchor": 0.29703524708747864, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.802237033843994, + "rewards/margins": 1.697741150856018, + "rewards/rejected": 1.1044957637786865, + "step": 735 + }, + { + "drift/chosen_abs": 0.21863920986652374, + "drift/rejected_abs": 0.165724977850914, + "epoch": 0.707034515705243, + "grad_norm": 3120.0, + "learning_rate": 2.3588670704111997e-08, + "logits/chosen": -1.931133508682251, + "logits/rejected": -1.8537521362304688, + "logps/chosen": -0.2914375364780426, + "logps/rejected": -0.3034089207649231, + "loss": 4437.83544921875, + "loss/core": 4437.828125, + "loss/preference_sft": 0.2914375364780426, + "loss/reference_anchor": 0.32963576912879944, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.18485689163208, + "rewards/margins": 0.5304257869720459, + "rewards/rejected": 1.6544309854507446, + "step": 740 + }, + { + "drift/chosen_abs": 0.22953541576862335, + "drift/rejected_abs": 0.12510935962200165, + "epoch": 0.7118117759464947, + "grad_norm": 636.0, + "learning_rate": 2.219061308991721e-08, + "logits/chosen": -2.2116262912750244, + "logits/rejected": -2.234771251678467, + "logps/chosen": -0.2957614064216614, + "logps/rejected": -0.28059297800064087, + "loss": 4430.69677734375, + "loss/core": 4430.68994140625, + "loss/preference_sft": 0.2957614064216614, + "loss/reference_anchor": 0.3111433684825897, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.295354127883911, + "rewards/margins": 1.0461337566375732, + "rewards/rejected": 1.2492201328277588, + "step": 745 + }, + { + "drift/chosen_abs": 0.22382231056690216, + "drift/rejected_abs": 0.1400088369846344, + "epoch": 0.7165890361877463, + "grad_norm": 7872.0, + "learning_rate": 2.0831217881543557e-08, + "logits/chosen": -2.2578673362731934, + "logits/rejected": -2.194847583770752, + "logps/chosen": -0.27832141518592834, + "logps/rejected": -0.30063265562057495, + "loss": 4433.3857421875, + "loss/core": 4433.3818359375, + "loss/preference_sft": 0.27832141518592834, + "loss/reference_anchor": 0.1458483338356018, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.2344486713409424, + "rewards/margins": 0.8346776962280273, + "rewards/rejected": 1.3997710943222046, + "step": 750 + }, + { + "drift/chosen_abs": 0.3602936863899231, + "drift/rejected_abs": 0.19181835651397705, + "epoch": 0.721366296428998, + "grad_norm": 560.0, + "learning_rate": 1.951099629193366e-08, + "logits/chosen": -2.169501781463623, + "logits/rejected": -2.14741587638855, + "logps/chosen": -0.2755396068096161, + "logps/rejected": -0.2857043743133545, + "loss": 4422.30126953125, + "loss/core": 4422.2890625, + "loss/preference_sft": 0.2755396068096161, + "loss/reference_anchor": 0.6152486801147461, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.602374315261841, + "rewards/margins": 1.6893268823623657, + "rewards/rejected": 1.9130470752716064, + "step": 755 + }, + { + "drift/chosen_abs": 0.3550851047039032, + "drift/rejected_abs": 0.20769810676574707, + "epoch": 0.7261435566702497, + "grad_norm": 8768.0, + "learning_rate": 1.823044480243431e-08, + "logits/chosen": -2.0105481147766113, + "logits/rejected": -2.006004810333252, + "logps/chosen": -0.3070371150970459, + "logps/rejected": -0.3485947549343109, + "loss": 4425.2255859375, + "loss/core": 4425.2138671875, + "loss/preference_sft": 0.3070371150970459, + "loss/reference_anchor": 0.5617225170135498, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 3.5491435527801514, + "rewards/margins": 1.4741640090942383, + "rewards/rejected": 2.074979305267334, + "step": 760 + }, + { + "drift/chosen_abs": 0.16385112702846527, + "drift/rejected_abs": 0.08402819186449051, + "epoch": 0.7309208169115012, + "grad_norm": 844.0, + "learning_rate": 1.699004497608994e-08, + "logits/chosen": -2.3256075382232666, + "logits/rejected": -2.332451105117798, + "logps/chosen": -0.27758467197418213, + "logps/rejected": -0.26881900429725647, + "loss": 4433.8310546875, + "loss/core": 4433.8291015625, + "loss/preference_sft": 0.27758467197418213, + "loss/reference_anchor": 0.07551290094852448, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.637289047241211, + "rewards/margins": 0.7998763918876648, + "rewards/rejected": 0.8374127149581909, + "step": 765 + }, + { + "drift/chosen_abs": 0.2287888526916504, + "drift/rejected_abs": 0.07531949877738953, + "epoch": 0.7356980771527529, + "grad_norm": 11264.0, + "learning_rate": 1.5790263276546658e-08, + "logits/chosen": -2.220702648162842, + "logits/rejected": -2.295063018798828, + "logps/chosen": -0.286711722612381, + "logps/rejected": -0.2794603705406189, + "loss": 4424.37353515625, + "loss/core": 4424.3681640625, + "loss/preference_sft": 0.286711722612381, + "loss/reference_anchor": 0.25186899304389954, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.287888526916504, + "rewards/margins": 1.5352530479431152, + "rewards/rejected": 0.7526354789733887, + "step": 770 + }, + { + "drift/chosen_abs": 0.18583203852176666, + "drift/rejected_abs": 0.1396988481283188, + "epoch": 0.7404753373940045, + "grad_norm": 528.0, + "learning_rate": 1.4631550892634126e-08, + "logits/chosen": -2.1028919219970703, + "logits/rejected": -1.9989187717437744, + "logps/chosen": -0.2728358209133148, + "logps/rejected": -0.2678207755088806, + "loss": 4438.3876953125, + "loss/core": 4438.38330078125, + "loss/preference_sft": 0.2728358209133148, + "loss/reference_anchor": 0.1961844116449356, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.8562612533569336, + "rewards/margins": 0.4593871533870697, + "rewards/rejected": 1.3968740701675415, + "step": 775 + }, + { + "drift/chosen_abs": 0.2970121502876282, + "drift/rejected_abs": 0.19716815650463104, + "epoch": 0.7452525976352562, + "grad_norm": 29184.0, + "learning_rate": 1.3514343568692132e-08, + "logits/chosen": -2.24552845954895, + "logits/rejected": -2.210707426071167, + "logps/chosen": -0.29537197947502136, + "logps/rejected": -0.2902478277683258, + "loss": 4431.3583984375, + "loss/core": 4431.33984375, + "loss/preference_sft": 0.29537197947502136, + "loss/reference_anchor": 0.8959586024284363, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.9684689044952393, + "rewards/margins": 0.9983173608779907, + "rewards/rejected": 1.9701515436172485, + "step": 780 + }, + { + "drift/chosen_abs": 0.19076788425445557, + "drift/rejected_abs": 0.07851524651050568, + "epoch": 0.7500298578765078, + "grad_norm": 944.0, + "learning_rate": 1.2439061440704724e-08, + "logits/chosen": -2.2142930030822754, + "logits/rejected": -2.226604700088501, + "logps/chosen": -0.2813388407230377, + "logps/rejected": -0.281820148229599, + "loss": 4430.03515625, + "loss/core": 4430.0283203125, + "loss/preference_sft": 0.2813388407230377, + "loss/reference_anchor": 0.3390761911869049, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.9072763919830322, + "rewards/margins": 1.1225610971450806, + "rewards/rejected": 0.7847151756286621, + "step": 785 + }, + { + "drift/chosen_abs": 0.33144018054008484, + "drift/rejected_abs": 0.13096961379051208, + "epoch": 0.7548071181177595, + "grad_norm": 3488.0, + "learning_rate": 1.1406108878304468e-08, + "logits/chosen": -2.039895534515381, + "logits/rejected": -2.0326807498931885, + "logps/chosen": -0.2768157422542572, + "logps/rejected": -0.27031582593917847, + "loss": 4417.72802734375, + "loss/core": 4417.72021484375, + "loss/preference_sft": 0.2768157422542572, + "loss/reference_anchor": 0.3354681134223938, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 3.3139610290527344, + "rewards/margins": 2.033888339996338, + "rewards/rejected": 1.2800720930099487, + "step": 790 + }, + { + "drift/chosen_abs": 0.3740524649620056, + "drift/rejected_abs": 0.09586691856384277, + "epoch": 0.7595843783590112, + "grad_norm": 8256.0, + "learning_rate": 1.0415874332705381e-08, + "logits/chosen": -2.1573445796966553, + "logits/rejected": -2.1065096855163574, + "logps/chosen": -0.2745102345943451, + "logps/rejected": -0.2804674804210663, + "loss": 4408.5126953125, + "loss/core": 4408.4970703125, + "loss/preference_sft": 0.2745102345943451, + "loss/reference_anchor": 0.7442028522491455, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.7369015216827393, + "rewards/margins": 2.7862377166748047, + "rewards/rejected": 0.9506635665893555, + "step": 795 + }, + { + "drift/chosen_abs": 0.2516661584377289, + "drift/rejected_abs": 0.14492499828338623, + "epoch": 0.7643616386002627, + "grad_norm": 1168.0, + "learning_rate": 9.468730190622484e-09, + "logits/chosen": -2.473392963409424, + "logits/rejected": -2.458265781402588, + "logps/chosen": -0.2965652346611023, + "logps/rejected": -0.3087556064128876, + "loss": 4430.37841796875, + "loss/core": 4430.365234375, + "loss/preference_sft": 0.2965652346611023, + "loss/reference_anchor": 0.6160001158714294, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.5161986351013184, + "rewards/margins": 1.0671333074569702, + "rewards/rejected": 1.4490649700164795, + "step": 800 + }, + { + "drift/chosen_abs": 0.09642498195171356, + "drift/rejected_abs": 0.06358600407838821, + "epoch": 0.7691388988415144, + "grad_norm": 772.0, + "learning_rate": 8.565032634232194e-09, + "logits/chosen": -2.1253273487091064, + "logits/rejected": -2.0759212970733643, + "logps/chosen": -0.3012627065181732, + "logps/rejected": -0.3006402850151062, + "loss": 4440.068359375, + "loss/core": 4440.0673828125, + "loss/preference_sft": 0.3012627065181732, + "loss/reference_anchor": 0.021201241761446, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 0.962838351726532, + "rewards/margins": 0.3286899924278259, + "rewards/rejected": 0.634148359298706, + "step": 805 + }, + { + "drift/chosen_abs": 0.1828320473432541, + "drift/rejected_abs": 0.1643926501274109, + "epoch": 0.773916159082766, + "grad_norm": 3008.0, + "learning_rate": 7.70512150722702e-09, + "logits/chosen": -2.1040894985198975, + "logits/rejected": -2.042335033416748, + "logps/chosen": -0.2662808299064636, + "logps/rejected": -0.26375871896743774, + "loss": 4442.34619140625, + "loss/core": 4442.3408203125, + "loss/preference_sft": 0.2662808299064636, + "loss/reference_anchor": 0.245298832654953, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.8268150091171265, + "rewards/margins": 0.18445713818073273, + "rewards/rejected": 1.6423580646514893, + "step": 810 + }, + { + "drift/chosen_abs": 0.24483975768089294, + "drift/rejected_abs": 0.10785166919231415, + "epoch": 0.7786934193240177, + "grad_norm": 588.0, + "learning_rate": 6.8893201870139915e-09, + "logits/chosen": -2.097686290740967, + "logits/rejected": -2.2861804962158203, + "logps/chosen": -0.31777235865592957, + "logps/rejected": -0.3054574131965637, + "loss": 4426.5830078125, + "loss/core": 4426.57666015625, + "loss/preference_sft": 0.31777235865592957, + "loss/reference_anchor": 0.284587562084198, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.445316791534424, + "rewards/margins": 1.3685089349746704, + "rewards/rejected": 1.0768078565597534, + "step": 815 + }, + { + "drift/chosen_abs": 0.26204735040664673, + "drift/rejected_abs": 0.20677991211414337, + "epoch": 0.7834706795652693, + "grad_norm": 2288.0, + "learning_rate": 6.117935463105808e-09, + "logits/chosen": -1.823381781578064, + "logits/rejected": -1.8026411533355713, + "logps/chosen": -0.2800712585449219, + "logps/rejected": -0.3262674808502197, + "loss": 4437.53466796875, + "loss/core": 4437.5244140625, + "loss/preference_sft": 0.2800712585449219, + "loss/reference_anchor": 0.490020215511322, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.620026111602783, + "rewards/margins": 0.566874086856842, + "rewards/rejected": 2.053152084350586, + "step": 820 + }, + { + "drift/chosen_abs": 0.35397663712501526, + "drift/rejected_abs": 0.16934973001480103, + "epoch": 0.788247939806521, + "grad_norm": 1288.0, + "learning_rate": 5.391257421749826e-09, + "logits/chosen": -2.1389248371124268, + "logits/rejected": -2.1224210262298584, + "logps/chosen": -0.29372864961624146, + "logps/rejected": -0.2752332091331482, + "loss": 4420.16455078125, + "loss/core": 4420.1484375, + "loss/preference_sft": 0.29372864961624146, + "loss/reference_anchor": 0.7682027816772461, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.5389609336853027, + "rewards/margins": 1.84674072265625, + "rewards/rejected": 1.6922200918197632, + "step": 825 + }, + { + "drift/chosen_abs": 0.28098636865615845, + "drift/rejected_abs": 0.12216347455978394, + "epoch": 0.7930252000477725, + "grad_norm": 7328.0, + "learning_rate": 4.709559336838462e-09, + "logits/chosen": -1.9067840576171875, + "logits/rejected": -1.9582593441009521, + "logps/chosen": -0.3030111491680145, + "logps/rejected": -0.3043147027492523, + "loss": 4423.7431640625, + "loss/core": 4423.7333984375, + "loss/preference_sft": 0.3030111491680145, + "loss/reference_anchor": 0.462859570980072, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.809863567352295, + "rewards/margins": 1.590742588043213, + "rewards/rejected": 1.2191212177276611, + "step": 830 + }, + { + "drift/chosen_abs": 0.16951501369476318, + "drift/rejected_abs": 0.10462959110736847, + "epoch": 0.7978024602890242, + "grad_norm": 596.0, + "learning_rate": 4.073097567142025e-09, + "logits/chosen": -2.2416045665740967, + "logits/rejected": -2.323671817779541, + "logps/chosen": -0.29937443137168884, + "logps/rejected": -0.28850868344306946, + "loss": 4435.8212890625, + "loss/core": 4435.818359375, + "loss/preference_sft": 0.29937443137168884, + "loss/reference_anchor": 0.1115211695432663, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.6946293115615845, + "rewards/margins": 0.649151086807251, + "rewards/rejected": 1.0454782247543335, + "step": 835 + }, + { + "drift/chosen_abs": 0.20362646877765656, + "drift/rejected_abs": 0.15938568115234375, + "epoch": 0.8025797205302759, + "grad_norm": 1040.0, + "learning_rate": 3.482111459902695e-09, + "logits/chosen": -2.0392162799835205, + "logits/rejected": -2.133927822113037, + "logps/chosen": -0.2889602780342102, + "logps/rejected": -0.3033998906612396, + "loss": 4439.00927734375, + "loss/core": 4439.0029296875, + "loss/preference_sft": 0.2889602780342102, + "loss/reference_anchor": 0.33250418305397034, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.036264657974243, + "rewards/margins": 0.44385433197021484, + "rewards/rejected": 1.5924100875854492, + "step": 840 + }, + { + "drift/chosen_abs": 0.46066784858703613, + "drift/rejected_abs": 0.22359898686408997, + "epoch": 0.8073569807715275, + "grad_norm": 1544.0, + "learning_rate": 2.9368232608258797e-09, + "logits/chosen": -2.173532485961914, + "logits/rejected": -2.1856465339660645, + "logps/chosen": -0.2729622721672058, + "logps/rejected": -0.27251607179641724, + "loss": 4414.25390625, + "loss/core": 4414.23193359375, + "loss/preference_sft": 0.2729622721672058, + "loss/reference_anchor": 1.0718512535095215, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.606678009033203, + "rewards/margins": 2.375056028366089, + "rewards/rejected": 2.2316224575042725, + "step": 845 + }, + { + "drift/chosen_abs": 0.17466862499713898, + "drift/rejected_abs": 0.06862124055624008, + "epoch": 0.8121342410127792, + "grad_norm": 424.0, + "learning_rate": 2.4374380305025866e-09, + "logits/chosen": -2.2522521018981934, + "logits/rejected": -2.309323787689209, + "logps/chosen": -0.2542715072631836, + "logps/rejected": -0.2512281537055969, + "loss": 4430.23974609375, + "loss/core": 4430.2373046875, + "loss/preference_sft": 0.2542715072631836, + "loss/reference_anchor": 0.06204066798090935, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.7456880807876587, + "rewards/margins": 1.0702584981918335, + "rewards/rejected": 0.6754294633865356, + "step": 850 + }, + { + "drift/chosen_abs": 0.23169496655464172, + "drift/rejected_abs": 0.19141283631324768, + "epoch": 0.8169115012540308, + "grad_norm": 2560.0, + "learning_rate": 1.984143567294594e-09, + "logits/chosen": -2.089970350265503, + "logits/rejected": -2.082474946975708, + "logps/chosen": -0.2673383355140686, + "logps/rejected": -0.2954670786857605, + "loss": 4439.4287109375, + "loss/core": 4439.421875, + "loss/preference_sft": 0.2673383355140686, + "loss/reference_anchor": 0.32883888483047485, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.314150333404541, + "rewards/margins": 0.4091102182865143, + "rewards/rejected": 1.9050400257110596, + "step": 855 + }, + { + "drift/chosen_abs": 0.19237197935581207, + "drift/rejected_abs": 0.10806970298290253, + "epoch": 0.8216887614952825, + "grad_norm": 544.0, + "learning_rate": 1.577110336711096e-09, + "logits/chosen": -2.185058116912842, + "logits/rejected": -2.2582504749298096, + "logps/chosen": -0.29970070719718933, + "logps/rejected": -0.30701369047164917, + "loss": 4433.2958984375, + "loss/core": 4433.291015625, + "loss/preference_sft": 0.29970070719718933, + "loss/reference_anchor": 0.2221534699201584, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.921847939491272, + "rewards/margins": 0.8417887687683105, + "rewards/rejected": 1.0800589323043823, + "step": 860 + }, + { + "drift/chosen_abs": 0.26305049657821655, + "drift/rejected_abs": 0.12108287960290909, + "epoch": 0.826466021736534, + "grad_norm": 736.0, + "learning_rate": 1.2164914073037048e-09, + "logits/chosen": -2.147150754928589, + "logits/rejected": -2.2282862663269043, + "logps/chosen": -0.29672080278396606, + "logps/rejected": -0.28275054693222046, + "loss": 4426.12451171875, + "loss/core": 4426.11181640625, + "loss/preference_sft": 0.29672080278396606, + "loss/reference_anchor": 0.6080988049507141, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.629103183746338, + "rewards/margins": 1.4199514389038086, + "rewards/rejected": 1.2091517448425293, + "step": 865 + }, + { + "drift/chosen_abs": 0.4451891779899597, + "drift/rejected_abs": 0.21213407814502716, + "epoch": 0.8312432819777857, + "grad_norm": 2024.0, + "learning_rate": 9.024223931035357e-10, + "logits/chosen": -1.9941089153289795, + "logits/rejected": -1.9594297409057617, + "logps/chosen": -0.2970310151576996, + "logps/rejected": -0.2791312336921692, + "loss": 4414.5986328125, + "loss/core": 4414.580078125, + "loss/preference_sft": 0.2970310151576996, + "loss/reference_anchor": 0.8715806007385254, + "rewards/accuracies": 0.875, + "rewards/chosen": 4.448572158813477, + "rewards/margins": 2.3311219215393066, + "rewards/rejected": 2.1174495220184326, + "step": 870 + }, + { + "drift/chosen_abs": 0.19540363550186157, + "drift/rejected_abs": 0.13800457119941711, + "epoch": 0.8360205422190374, + "grad_norm": 440.0, + "learning_rate": 6.350214026223516e-10, + "logits/chosen": -2.0932259559631348, + "logits/rejected": -2.0954394340515137, + "logps/chosen": -0.30296754837036133, + "logps/rejected": -0.3049096465110779, + "loss": 4436.95068359375, + "loss/core": 4436.94580078125, + "loss/preference_sft": 0.30296754837036133, + "loss/reference_anchor": 0.24102571606636047, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.953322172164917, + "rewards/margins": 0.5735519528388977, + "rewards/rejected": 1.379770278930664, + "step": 875 + }, + { + "drift/chosen_abs": 0.37028616666793823, + "drift/rejected_abs": 0.15220406651496887, + "epoch": 0.840797802460289, + "grad_norm": 360.0, + "learning_rate": 4.143889944367429e-10, + "logits/chosen": -2.00821590423584, + "logits/rejected": -1.9785652160644531, + "logps/chosen": -0.28579357266426086, + "logps/rejected": -0.30298155546188354, + "loss": 4416.18603515625, + "loss/core": 4416.17138671875, + "loss/preference_sft": 0.28579357266426086, + "loss/reference_anchor": 0.7266721725463867, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.702655076980591, + "rewards/margins": 2.1888763904571533, + "rewards/rejected": 1.5137783288955688, + "step": 880 + }, + { + "drift/chosen_abs": 0.15521343052387238, + "drift/rejected_abs": 0.07931456714868546, + "epoch": 0.8455750627015407, + "grad_norm": 2080.0, + "learning_rate": 2.406081393722531e-10, + "logits/chosen": -2.103982448577881, + "logits/rejected": -2.132598876953125, + "logps/chosen": -0.29826053977012634, + "logps/rejected": -0.3063696026802063, + "loss": 4434.3544921875, + "loss/core": 4434.3525390625, + "loss/preference_sft": 0.29826053977012634, + "loss/reference_anchor": 0.05555032566189766, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.5512726306915283, + "rewards/margins": 0.7596304416656494, + "rewards/rejected": 0.7916421890258789, + "step": 885 + }, + { + "drift/chosen_abs": 0.18553657829761505, + "drift/rejected_abs": 0.1623484194278717, + "epoch": 0.8503523229427923, + "grad_norm": 1152.0, + "learning_rate": 1.1374418930135133e-10, + "logits/chosen": -2.1390254497528076, + "logits/rejected": -2.191925525665283, + "logps/chosen": -0.26982808113098145, + "logps/rejected": -0.28849366307258606, + "loss": 4441.68017578125, + "loss/core": 4441.67431640625, + "loss/preference_sft": 0.26982808113098145, + "loss/reference_anchor": 0.2661302983760834, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8522002696990967, + "rewards/margins": 0.22871632874011993, + "rewards/rejected": 1.6234838962554932, + "step": 890 + }, + { + "drift/chosen_abs": 0.21298833191394806, + "drift/rejected_abs": 0.09839890152215958, + "epoch": 0.855129583184044, + "grad_norm": 5824.0, + "learning_rate": 3.3844852567285756e-11, + "logits/chosen": -2.102201223373413, + "logits/rejected": -2.060621738433838, + "logps/chosen": -0.29058605432510376, + "logps/rejected": -0.2914868891239166, + "loss": 4429.2578125, + "loss/core": 4429.2548828125, + "loss/preference_sft": 0.29058605432510376, + "loss/reference_anchor": 0.13424192368984222, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.129883289337158, + "rewards/margins": 1.1480109691619873, + "rewards/rejected": 0.9818723797798157, + "step": 895 + }, + { + "drift/chosen_abs": 0.29029321670532227, + "drift/rejected_abs": 0.14265631139278412, + "epoch": 0.8599068434252956, + "grad_norm": 7520.0, + "learning_rate": 9.401760429905703e-13, + "logits/chosen": -2.05942964553833, + "logits/rejected": -2.0243592262268066, + "logps/chosen": -0.2692040205001831, + "logps/rejected": -0.27301639318466187, + "loss": 4425.076171875, + "loss/core": 4425.06787109375, + "loss/preference_sft": 0.2692040205001831, + "loss/reference_anchor": 0.38637253642082214, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.9029033184051514, + "rewards/margins": 1.476595163345337, + "rewards/rejected": 1.4263079166412354, + "step": 900 + }, + { + "epoch": 0.8599068434252956, + "step": 900, + "total_flos": 0.0, + "train_loss": 4429.236100260417, + "train_runtime": 7053.367, + "train_samples_per_second": 2.042, + "train_steps_per_second": 0.128 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..41f3c37 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83ff0b25a219a85d50b74252407e1f50951913b55f5de2827f469ceb004a2663 +size 6993