From 6acfd2f71abbecadd9557b7acff2663c2cd5dff1 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Tue, 21 Jul 2026 19:32:10 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: promotion/qwen3-8b-aaai27-flagship-dpo-s42 Source: Original Platform --- .gitattributes | 36 + README.md | 24 + all_results.json | 9 + chat_template.jinja | 89 ++ config.json | 71 + config.yaml | 58 + generation_config.json | 12 + job_status.json | 23 + model.safetensors | 3 + objective_protocol.json | 69 + pair_manifest.json | 34 + provenance.json | 12 + run_status.json | 13 + split_manifest.json | 23 + stability_gate.json | 38 + tokenizer.json | 3 + tokenizer_config.json | 30 + train_results.json | 9 + trainer_state.json | 2923 +++++++++++++++++++++++++++++++++++++++ training_args.bin | 3 + 20 files changed, 3482 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 config.yaml create mode 100644 generation_config.json create mode 100644 job_status.json create mode 100644 model.safetensors create mode 100644 objective_protocol.json create mode 100644 pair_manifest.json create mode 100644 provenance.json create mode 100644 run_status.json create mode 100644 split_manifest.json create mode 100644 stability_gate.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..7962380 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +--- +library_name: transformers +base_model: Qwen/Qwen3-8B +tags: +- ronpo +- mnpo +- preference-optimization +--- + +# qwen3-8b-aaai27-flagship-dpo-s42 + +Research checkpoint for the RONPO AAAI revision experiments. + +- Method: dpo +- Base model: `Qwen/Qwen3-8B` +- Seed: 42 +- Local source at upload time: `/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/dpo/seed42/attempt1` +- Uploaded at UTC: 2026-07-12T17:21:45Z +- Run status metadata: `{"attempt": 1, "effective_batch_size": 16, "method": "dpo", "objective_protocol": "objective_protocol.json", "optimizer_steps": 900, "seed": 42, "split_manifest": "split_manifest.json", "stability_gate": "stability_gate.json", "status": "completed", "wandb_run_id": "f11bcc5bf0a6", "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/f11bcc5bf0a6"}` + +AAAI-27 P1 matched budget: 900 optimizer steps, effective batch 16; passed non-thinking and collapse stability gates; sealed-test metrics are not used for checkpoint selection. + +Intended use: reproducibility and evaluation for the RONPO paper. This +checkpoint is not intended as a production assistant. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..ebf6347 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 0.690198503865136, + "train_runtime": 8363.1914, + "train_samples": 16746, + "train_samples_per_second": 1.722, + "train_steps_per_second": 0.108 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..a82dcca --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +model_name_or_path: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218 +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/avg: 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: dpo +eta: 0.0075 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.0 +preference_sft_weight: 0.0 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 5.0e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 42 +gradient_accumulation_steps: 8 +gradient_checkpointing: false +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 2 +per_device_eval_batch_size: 2 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/dpo/seed42/attempt1 +run_name: aaai27-flagship-full-dpo-s42-a1 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/job_status.json b/job_status.json new file mode 100644 index 0000000..67fc62f --- /dev/null +++ b/job_status.json @@ -0,0 +1,23 @@ +{ + "status": "completed", + "stage": "full", + "method": "dpo", + "seed": 42, + "attempt": 1, + "gpu": 6, + "gpus": [ + 6 + ], + "returncode": 0, + "finite_metrics": true, + "oom": false, + "traceback": false, + "reason": "passed", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "f11bcc5bf0a6", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/f11bcc5bf0a6", + "output_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/dpo/seed42/attempt1", + "log": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/logs/full_dpo_s42_a1.log", + "completed_at": "2026-07-12T17:12:24Z" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..bb2b6bd --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:333ee8a1d8bb147056dcf547fd91aeba0f4d6a1c51a487d8b8edecb1083ca470 +size 16381517208 diff --git a/objective_protocol.json b/objective_protocol.json new file mode 100644 index 0000000..35b3b44 --- /dev/null +++ b/objective_protocol.json @@ -0,0 +1,69 @@ +{ + "schema_version": 1, + "frozen_at_kst": "2026-07-12T21:40:00+09:00", + "base_model": "Qwen/Qwen3-8B", + "primary_triple": { + "helpfulness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "ultrafeedback-helpfulness", + "transform": "identity" + }, + "safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "conciseness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "helpsteer-verbosity", + "transform": "negate" + } + }, + "gate_thresholds": { + "raw_score_std_strictly_greater_than": 0.01, + "aggregate_prompt_pair_spearman_median_strictly_less_than": 0.0, + "aggregate_pairwise_top1_mismatch_at_least": 0.5 + }, + "gate_pool": "source-train Qwen3-8B response pool split into deterministic train/validation; sealed source-test excluded", + "training_seeds": [42, 43, 44], + "optimizer_steps": 900, + "effective_batch_size": 16, + "decode": { + "enable_thinking": false, + "temperature": 0.7, + "top_p": 0.9, + "max_new_tokens": 2048, + "decode_seed": 42 + }, + "methods": [ + "ronpo_full_expect", + "ronpo_k_only", + "dpo", + "ipo", + "simpo", + "kto", + "sppo_avg", + "inpo_avg", + "ht_mnpo_helpfulness", + "ht_mnpo_safety", + "ht_mnpo_conciseness" + ], + "selection_policy": "No sealed-test selection. One preregistered configuration per method; failures use only the declared stability retry recipe, at most three attempts.", + "stability_gate": { + "validation_prompts": 128, + "min_candidate_base_mean_word_ratio": 0.33, + "max_candidate_base_mean_word_ratio": 2.0, + "max_consecutive_identical_word_run": 20, + "think_tag_count": 0, + "empty_response_count": 0, + "retry_recipe": "attempt 2 halves learning rate, enables gradient checkpointing, and adds reference=0.02/SFT=0.002 anchoring to unanchored unified losses; attempt 3 quarters learning rate and uses reference=0.05/SFT=0.005; RONPO uses reference=0.05/SFT=0.005 from attempt 1" + }, + "primary_metric": "mean prompt-level minimum of per-prompt min-max-normalized helpfulness, safety, conciseness", + "bootstrap": { + "unit": "prompt", + "paired": true, + "resamples": 2000, + "interval": "percentile_95", + "seed": 42 + } +} diff --git a/pair_manifest.json b/pair_manifest.json new file mode 100644 index 0000000..be034b0 --- /dev/null +++ b/pair_manifest.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "objectives": [ + "helpfulness", + "safety", + "conciseness" + ], + "normalization": "per_prompt_minmax", + "adversary": { + "steps": 25, + "alpha": 1.0, + "kappa": 0.05, + "preference_scale": 8.0 + }, + "ronpo_estimator": "Rao-Blackwellized full sigma expectation with three common response pairs per prompt", + "counts": { + "train": { + "avg": 16746, + "ronpo_full_expect": 50340, + "ronpo_k_only": 50340, + "ht_mnpo_helpfulness": 16743, + "ht_mnpo_safety": 16764, + "ht_mnpo_conciseness": 16755 + }, + "validation": { + "avg": 1859, + "ronpo_full_expect": 5592, + "ronpo_k_only": 5592, + "ht_mnpo_helpfulness": 1864, + "ht_mnpo_safety": 1862, + "ht_mnpo_conciseness": 1858 + } + } +} diff --git a/provenance.json b/provenance.json new file mode 100644 index 0000000..f395d7f --- /dev/null +++ b/provenance.json @@ -0,0 +1,12 @@ +{ + "method": "dpo", + "seed": 42, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "f11bcc5bf0a6", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/f11bcc5bf0a6", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "stability_gate": "stability_gate.json" +} diff --git a/run_status.json b/run_status.json new file mode 100644 index 0000000..963c4ce --- /dev/null +++ b/run_status.json @@ -0,0 +1,13 @@ +{ + "method": "dpo", + "seed": 42, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "f11bcc5bf0a6", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/f11bcc5bf0a6", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "stability_gate": "stability_gate.json", + "status": "completed" +} diff --git a/split_manifest.json b/split_manifest.json new file mode 100644 index 0000000..4fb96bd --- /dev/null +++ b/split_manifest.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "split_rule": "sort source-train prompts by sha256('flagship-v1||'+prompt_id); first round(10%) validation", + "validation_fraction": 0.1, + "source_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/avg_oracle", + "base_model": "Qwen/Qwen3-8B", + "counts": { + "train": 16780, + "validation": 1864, + "sealed_test": 604 + }, + "prompt_id_sha256": { + "train": "fcf6f2ea5a0b7bcd41179e6f5288f4d850b2d04bdb729da91907bd18bf733a69", + "validation": "a87ba6f519192164daa487b109a6547dd8cbde46a27a373d6da201ca2d061b8f", + "sealed_test": "b398ced6af6cf73d731e1a5f32a0631d8060323d51eda1e375695566bf5f1867" + }, + "overlaps": { + "train_validation": 0, + "train_sealed_test": 0, + "validation_sealed_test": 0 + }, + "sealed_test_policy": "prompts only until S4a; no objective scores or model generations used in S0-S3" +} diff --git a/stability_gate.json b/stability_gate.json new file mode 100644 index 0000000..65a022b --- /dev/null +++ b/stability_gate.json @@ -0,0 +1,38 @@ +{ + "status": "passed", + "passed": true, + "base": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 388.1484375, + "max_words": 1379, + "max_repeat_run": 4 + }, + "candidate": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 406.4921875, + "max_words": 1334, + "max_repeat_run": 4 + }, + "candidate_base_mean_word_ratio": 1.0472596260290241, + "thresholds": { + "min_length_ratio": 0.33, + "max_length_ratio": 2.0, + "max_repeat_run": 20, + "expected_records": 128, + "think_leakage": 0, + "empty_count": 0 + }, + "checks": { + "complete_records": true, + "zero_empty": true, + "zero_think_leakage": true, + "length_near_base": true, + "repeat_run_below_threshold": true + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ffd19d2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..ebf6347 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 0.690198503865136, + "train_runtime": 8363.1914, + "train_samples": 16746, + "train_samples_per_second": 1.722, + "train_steps_per_second": 0.108 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..14bbcc7 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,2923 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8599068434252956, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.004777260241251642, + "grad_norm": 0.396484375, + "learning_rate": 2.222222222222222e-08, + "logits/chosen": -1.7133970260620117, + "logits/rejected": -1.6876779794692993, + "logps/chosen": -0.2796992361545563, + "logps/rejected": -0.28187569975852966, + "loss": 0.6895627379417419, + "loss/core": 0.6895627379417419, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.4170753955841064, + "rewards/margins": 1.4557363986968994, + "rewards/rejected": 0.9613393545150757, + "step": 5 + }, + { + "epoch": 0.009554520482503284, + "grad_norm": 2.28125, + "learning_rate": 5e-08, + "logits/chosen": -1.9638487100601196, + "logits/rejected": -2.050144910812378, + "logps/chosen": -0.28905805945396423, + "logps/rejected": -0.2884846329689026, + "loss": 0.6929782032966614, + "loss/core": 0.6929782032966614, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.6930923461914062, + "rewards/margins": 0.08007440716028214, + "rewards/rejected": 1.6130180358886719, + "step": 10 + }, + { + "epoch": 0.014331780723754926, + "grad_norm": 0.44140625, + "learning_rate": 7.777777777777778e-08, + "logits/chosen": -1.8020557165145874, + "logits/rejected": -1.7314844131469727, + "logps/chosen": -0.27712005376815796, + "logps/rejected": -0.2847710847854614, + "loss": 0.6906865239143372, + "loss/core": 0.6906865239143372, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.1371042728424072, + "rewards/margins": 1.0041325092315674, + "rewards/rejected": 1.1329715251922607, + "step": 15 + }, + { + "epoch": 0.01910904096500657, + "grad_norm": 0.3046875, + "learning_rate": 1.0555555555555555e-07, + "logits/chosen": -1.9990392923355103, + "logits/rejected": -1.998739242553711, + "logps/chosen": -0.3013564348220825, + "logps/rejected": -0.28647691011428833, + "loss": 0.6906139254570007, + "loss/core": 0.6906139254570007, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9157869815826416, + "rewards/margins": 1.0352976322174072, + "rewards/rejected": 0.8804894685745239, + "step": 20 + }, + { + "epoch": 0.02388630120625821, + "grad_norm": 0.2392578125, + "learning_rate": 1.3333333333333334e-07, + "logits/chosen": -1.9096256494522095, + "logits/rejected": -1.9282000064849854, + "logps/chosen": -0.2734399437904358, + "logps/rejected": -0.2599286735057831, + "loss": 0.6880686283111572, + "loss/core": 0.6880686283111572, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 3.8207664489746094, + "rewards/margins": 2.1955325603485107, + "rewards/rejected": 1.6252334117889404, + "step": 25 + }, + { + "epoch": 0.028663561447509853, + "grad_norm": 0.59765625, + "learning_rate": 1.611111111111111e-07, + "logits/chosen": -2.0870721340179443, + "logits/rejected": -2.0627994537353516, + "logps/chosen": -0.30142563581466675, + "logps/rejected": -0.32440415024757385, + "loss": 0.6901697516441345, + "loss/core": 0.6901697516441345, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.26503849029541, + "rewards/margins": 1.239709496498108, + "rewards/rejected": 1.0253286361694336, + "step": 30 + }, + { + "epoch": 0.033440821688761495, + "grad_norm": 0.12890625, + "learning_rate": 1.8888888888888888e-07, + "logits/chosen": -1.8673341274261475, + "logits/rejected": -1.8807346820831299, + "logps/chosen": -0.2671809196472168, + "logps/rejected": -0.26550590991973877, + "loss": 0.6895102262496948, + "loss/core": 0.6895102262496948, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.6639485359191895, + "rewards/margins": 1.506712555885315, + "rewards/rejected": 1.157235860824585, + "step": 35 + }, + { + "epoch": 0.03821808193001314, + "grad_norm": 0.421875, + "learning_rate": 2.1666666666666667e-07, + "logits/chosen": -1.6795200109481812, + "logits/rejected": -1.7307487726211548, + "logps/chosen": -0.2936388850212097, + "logps/rejected": -0.2830830514431, + "loss": 0.6864294409751892, + "loss/core": 0.6864294409751892, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.467957019805908, + "rewards/margins": 2.8773446083068848, + "rewards/rejected": 1.5906131267547607, + "step": 40 + }, + { + "epoch": 0.04299534217126478, + "grad_norm": 0.1943359375, + "learning_rate": 2.4444444444444445e-07, + "logits/chosen": -1.867241621017456, + "logits/rejected": -1.9943357706069946, + "logps/chosen": -0.28906434774398804, + "logps/rejected": -0.28838273882865906, + "loss": 0.690056562423706, + "loss/core": 0.690056562423706, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.092913866043091, + "rewards/margins": 1.250601053237915, + "rewards/rejected": 0.8423126935958862, + "step": 45 + }, + { + "epoch": 0.04777260241251642, + "grad_norm": 0.94140625, + "learning_rate": 2.7222222222222216e-07, + "logits/chosen": -1.9381431341171265, + "logits/rejected": -1.944374442100525, + "logps/chosen": -0.2815219759941101, + "logps/rejected": -0.28127673268318176, + "loss": 0.6898090839385986, + "loss/core": 0.6898090839385986, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.8812808990478516, + "rewards/margins": 1.3943486213684082, + "rewards/rejected": 1.4869322776794434, + "step": 50 + }, + { + "epoch": 0.05254986265376806, + "grad_norm": 0.07958984375, + "learning_rate": 3e-07, + "logits/chosen": -1.9054334163665771, + "logits/rejected": -1.8701159954071045, + "logps/chosen": -0.28710708022117615, + "logps/rejected": -0.28994759917259216, + "loss": 0.6927647590637207, + "loss/core": 0.6927647590637207, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.277327060699463, + "rewards/margins": 0.2503672242164612, + "rewards/rejected": 2.0269598960876465, + "step": 55 + }, + { + "epoch": 0.057327122895019705, + "grad_norm": 3.265625, + "learning_rate": 3.2777777777777776e-07, + "logits/chosen": -1.8265936374664307, + "logits/rejected": -1.817793846130371, + "logps/chosen": -0.27443718910217285, + "logps/rejected": -0.2840142548084259, + "loss": 0.6916871666908264, + "loss/core": 0.6916871666908264, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.1564693450927734, + "rewards/margins": 0.6028232574462891, + "rewards/rejected": 1.553646206855774, + "step": 60 + }, + { + "epoch": 0.06210438313627135, + "grad_norm": 0.1669921875, + "learning_rate": 3.5555555555555553e-07, + "logits/chosen": -1.6590009927749634, + "logits/rejected": -1.720873236656189, + "logps/chosen": -0.3111419975757599, + "logps/rejected": -0.304778516292572, + "loss": 0.688835859298706, + "loss/core": 0.688835859298706, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.9504354000091553, + "rewards/margins": 1.7633116245269775, + "rewards/rejected": 1.1871236562728882, + "step": 65 + }, + { + "epoch": 0.06688164337752299, + "grad_norm": 0.462890625, + "learning_rate": 3.8333333333333335e-07, + "logits/chosen": -1.652000069618225, + "logits/rejected": -1.681039571762085, + "logps/chosen": -0.37990865111351013, + "logps/rejected": -0.29802781343460083, + "loss": 0.6898948550224304, + "loss/core": 0.6898948550224304, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.6380057334899902, + "rewards/margins": 1.4650633335113525, + "rewards/rejected": 2.1729423999786377, + "step": 70 + }, + { + "epoch": 0.07165890361877464, + "grad_norm": 0.12890625, + "learning_rate": 4.1111111111111107e-07, + "logits/chosen": -1.710818886756897, + "logits/rejected": -1.736729383468628, + "logps/chosen": -0.31785252690315247, + "logps/rejected": -0.2882820963859558, + "loss": 0.6908040642738342, + "loss/core": 0.6908040642738342, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.5783510208129883, + "rewards/margins": 0.9765921831130981, + "rewards/rejected": 1.6017587184906006, + "step": 75 + }, + { + "epoch": 0.07643616386002627, + "grad_norm": 0.1689453125, + "learning_rate": 4.3888888888888884e-07, + "logits/chosen": -1.7501447200775146, + "logits/rejected": -1.8643512725830078, + "logps/chosen": -0.2600446045398712, + "logps/rejected": -0.27591589093208313, + "loss": 0.6883189082145691, + "loss/core": 0.6883189082145691, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.2465500831604004, + "rewards/margins": 2.013552188873291, + "rewards/rejected": 1.2329976558685303, + "step": 80 + }, + { + "epoch": 0.08121342410127792, + "grad_norm": 0.2001953125, + "learning_rate": 4.6666666666666666e-07, + "logits/chosen": -1.8498824834823608, + "logits/rejected": -1.8502506017684937, + "logps/chosen": -0.30674856901168823, + "logps/rejected": -0.27847328782081604, + "loss": 0.6879407167434692, + "loss/core": 0.6879407167434692, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.505821943283081, + "rewards/margins": 2.1826915740966797, + "rewards/rejected": 1.323130488395691, + "step": 85 + }, + { + "epoch": 0.08599068434252956, + "grad_norm": 0.0810546875, + "learning_rate": 4.944444444444445e-07, + "logits/chosen": -2.03493595123291, + "logits/rejected": -2.0456414222717285, + "logps/chosen": -0.28171059489250183, + "logps/rejected": -0.29523080587387085, + "loss": 0.6909321546554565, + "loss/core": 0.6909321546554565, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.7502819299697876, + "rewards/margins": 0.8973544836044312, + "rewards/rejected": 0.8529273867607117, + "step": 90 + }, + { + "epoch": 0.09076794458378121, + "grad_norm": 0.37890625, + "learning_rate": 4.999699149323369e-07, + "logits/chosen": -1.6194324493408203, + "logits/rejected": -1.6468042135238647, + "logps/chosen": -0.2668497860431671, + "logps/rejected": -0.27971798181533813, + "loss": 0.6904693841934204, + "loss/core": 0.6904693841934204, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.2152295112609863, + "rewards/margins": 1.1187059879302979, + "rewards/rejected": 2.0965230464935303, + "step": 95 + }, + { + "epoch": 0.09554520482503284, + "grad_norm": 0.19921875, + "learning_rate": 4.998477067547739e-07, + "logits/chosen": -1.8947890996932983, + "logits/rejected": -1.9110848903656006, + "logps/chosen": -0.2584729790687561, + "logps/rejected": -0.24682433903217316, + "loss": 0.6902803182601929, + "loss/core": 0.6902803182601929, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.4498555660247803, + "rewards/margins": 1.1651194095611572, + "rewards/rejected": 1.2847360372543335, + "step": 100 + }, + { + "epoch": 0.10032246506628449, + "grad_norm": 0.67578125, + "learning_rate": 4.996315410727229e-07, + "logits/chosen": -1.93181574344635, + "logits/rejected": -1.9506447315216064, + "logps/chosen": -0.283364474773407, + "logps/rejected": -0.27992352843284607, + "loss": 0.6902110576629639, + "loss/core": 0.6902110576629639, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.005598306655884, + "rewards/margins": 1.2088587284088135, + "rewards/rejected": 0.7967394590377808, + "step": 105 + }, + { + "epoch": 0.10509972530753613, + "grad_norm": 0.1845703125, + "learning_rate": 4.993214991772562e-07, + "logits/chosen": -1.7346656322479248, + "logits/rejected": -1.6285288333892822, + "logps/chosen": -0.33683377504348755, + "logps/rejected": -0.29718539118766785, + "loss": 0.6869697570800781, + "loss/core": 0.6869697570800781, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 4.486939907073975, + "rewards/margins": 2.5928866863250732, + "rewards/rejected": 1.8940532207489014, + "step": 110 + }, + { + "epoch": 0.10987698554878778, + "grad_norm": 0.859375, + "learning_rate": 4.989176976624511e-07, + "logits/chosen": -1.63823664188385, + "logits/rejected": -1.5655968189239502, + "logps/chosen": -0.30025607347488403, + "logps/rejected": -0.3001025915145874, + "loss": 0.6923608183860779, + "loss/core": 0.6923608183860779, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7768100500106812, + "rewards/margins": 0.34278303384780884, + "rewards/rejected": 1.434027075767517, + "step": 115 + }, + { + "epoch": 0.11465424579003941, + "grad_norm": 1.65625, + "learning_rate": 4.984202883815428e-07, + "logits/chosen": -1.682744026184082, + "logits/rejected": -1.6865253448486328, + "logps/chosen": -0.30211514234542847, + "logps/rejected": -0.29606205224990845, + "loss": 0.6914452910423279, + "loss/core": 0.6914452910423279, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8103902339935303, + "rewards/margins": 0.6875762343406677, + "rewards/rejected": 1.1228139400482178, + "step": 120 + }, + { + "epoch": 0.11943150603129106, + "grad_norm": 0.130859375, + "learning_rate": 4.978294583898195e-07, + "logits/chosen": -1.8059618473052979, + "logits/rejected": -1.8209705352783203, + "logps/chosen": -0.2989288568496704, + "logps/rejected": -0.3068915009498596, + "loss": 0.6905549168586731, + "loss/core": 0.6905549168586731, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.0661938190460205, + "rewards/margins": 1.0517460107803345, + "rewards/rejected": 1.0144474506378174, + "step": 125 + }, + { + "epoch": 0.1242087662725427, + "grad_norm": 0.27734375, + "learning_rate": 4.971454298742779e-07, + "logits/chosen": -1.6160223484039307, + "logits/rejected": -1.7299797534942627, + "logps/chosen": -0.27980923652648926, + "logps/rejected": -0.2786286771297455, + "loss": 0.6908574104309082, + "loss/core": 0.6908574104309082, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.8091075420379639, + "rewards/margins": 0.9205910563468933, + "rewards/rejected": 0.8885166049003601, + "step": 130 + }, + { + "epoch": 0.12898602651379434, + "grad_norm": 0.408203125, + "learning_rate": 4.963684600700678e-07, + "logits/chosen": -1.771058440208435, + "logits/rejected": -1.842339277267456, + "logps/chosen": -0.3017735481262207, + "logps/rejected": -0.29897257685661316, + "loss": 0.6880813837051392, + "loss/core": 0.6880813837051392, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.3773961067199707, + "rewards/margins": 2.0708842277526855, + "rewards/rejected": 1.3065119981765747, + "step": 135 + }, + { + "epoch": 0.13376328675504598, + "grad_norm": 0.1669921875, + "learning_rate": 4.954988411637571e-07, + "logits/chosen": -2.0918688774108887, + "logits/rejected": -2.117227077484131, + "logps/chosen": -0.28260543942451477, + "logps/rejected": -0.28928446769714355, + "loss": 0.6908842325210571, + "loss/core": 0.6908842325210571, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6272077560424805, + "rewards/margins": 0.9105762243270874, + "rewards/rejected": 0.7166314721107483, + "step": 140 + }, + { + "epoch": 0.13854054699629761, + "grad_norm": 0.1630859375, + "learning_rate": 4.945369001834514e-07, + "logits/chosen": -1.5617491006851196, + "logits/rejected": -1.5613640546798706, + "logps/chosen": -0.3025224804878235, + "logps/rejected": -0.3009711802005768, + "loss": 0.6887967586517334, + "loss/core": 0.6887967586517334, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.592343807220459, + "rewards/margins": 1.7993513345718384, + "rewards/rejected": 1.7929922342300415, + "step": 145 + }, + { + "epoch": 0.14331780723754928, + "grad_norm": 0.404296875, + "learning_rate": 4.93482998875813e-07, + "logits/chosen": -1.913665771484375, + "logits/rejected": -1.9741178750991821, + "logps/chosen": -0.3164519667625427, + "logps/rejected": -0.30851152539253235, + "loss": 0.6878332495689392, + "loss/core": 0.6878332495689392, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.6481528282165527, + "rewards/margins": 2.1936802864074707, + "rewards/rejected": 1.4544728994369507, + "step": 150 + }, + { + "epoch": 0.1480950674788009, + "grad_norm": 0.2041015625, + "learning_rate": 4.923375335700223e-07, + "logits/chosen": -1.8092594146728516, + "logits/rejected": -1.8692359924316406, + "logps/chosen": -0.284790575504303, + "logps/rejected": -0.2954999506473541, + "loss": 0.6881645917892456, + "loss/core": 0.6881645917892456, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.8475961685180664, + "rewards/margins": 2.052863597869873, + "rewards/rejected": 0.7947325110435486, + "step": 155 + }, + { + "epoch": 0.15287232772005255, + "grad_norm": 0.91015625, + "learning_rate": 4.911009350287347e-07, + "logits/chosen": -1.7557824850082397, + "logits/rejected": -1.7192802429199219, + "logps/chosen": -0.3111857771873474, + "logps/rejected": -0.3161669075489044, + "loss": 0.6890115737915039, + "loss/core": 0.6890115737915039, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.1435818672180176, + "rewards/margins": 1.7508246898651123, + "rewards/rejected": 1.3927572965621948, + "step": 160 + }, + { + "epoch": 0.15764958796130418, + "grad_norm": 0.2294921875, + "learning_rate": 4.897736682860885e-07, + "logits/chosen": -1.7782700061798096, + "logits/rejected": -1.7287534475326538, + "logps/chosen": -0.31163567304611206, + "logps/rejected": -0.3064088225364685, + "loss": 0.69112628698349, + "loss/core": 0.69112628698349, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.5969486236572266, + "rewards/margins": 0.8150261044502258, + "rewards/rejected": 0.7819225788116455, + "step": 165 + }, + { + "epoch": 0.16242684820255585, + "grad_norm": 0.8828125, + "learning_rate": 4.883562324728241e-07, + "logits/chosen": -1.8792940378189087, + "logits/rejected": -1.8148422241210938, + "logps/chosen": -0.27398645877838135, + "logps/rejected": -0.2596638798713684, + "loss": 0.6938985586166382, + "loss/core": 0.6938985586166382, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.6043877601623535, + "rewards/margins": -0.16653324663639069, + "rewards/rejected": 2.770921230316162, + "step": 170 + }, + { + "epoch": 0.16720410844380748, + "grad_norm": 1.359375, + "learning_rate": 4.868491606285823e-07, + "logits/chosen": -1.7575905323028564, + "logits/rejected": -1.772924780845642, + "logps/chosen": -0.28738316893577576, + "logps/rejected": -0.3002680242061615, + "loss": 0.6899343729019165, + "loss/core": 0.6899343729019165, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 2.5785281658172607, + "rewards/margins": 1.3044723272323608, + "rewards/rejected": 1.274056077003479, + "step": 175 + }, + { + "epoch": 0.17198136868505912, + "grad_norm": 1.453125, + "learning_rate": 4.852530195014489e-07, + "logits/chosen": -1.8603105545043945, + "logits/rejected": -1.8089433908462524, + "logps/chosen": -0.2715868353843689, + "logps/rejected": -0.26199498772621155, + "loss": 0.6889883279800415, + "loss/core": 0.6889883279800415, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.23846697807312, + "rewards/margins": 1.7489553689956665, + "rewards/rejected": 1.4895117282867432, + "step": 180 + }, + { + "epoch": 0.17675862892631075, + "grad_norm": 0.16015625, + "learning_rate": 4.835684093348244e-07, + "logits/chosen": -1.9552524089813232, + "logits/rejected": -1.9065921306610107, + "logps/chosen": -0.2885257303714752, + "logps/rejected": -0.2921094298362732, + "loss": 0.6921087503433228, + "loss/core": 0.6921087503433228, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.0444104671478271, + "rewards/margins": 0.41770505905151367, + "rewards/rejected": 0.6267052888870239, + "step": 185 + }, + { + "epoch": 0.18153588916756241, + "grad_norm": 0.341796875, + "learning_rate": 4.817959636416969e-07, + "logits/chosen": -1.9302583932876587, + "logits/rejected": -1.9251024723052979, + "logps/chosen": -0.2820015251636505, + "logps/rejected": -0.2878292202949524, + "loss": 0.6891991496086121, + "loss/core": 0.6891991496086121, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.8541626930236816, + "rewards/margins": 1.6507861614227295, + "rewards/rejected": 1.2033761739730835, + "step": 190 + }, + { + "epoch": 0.18631314940881405, + "grad_norm": 0.2275390625, + "learning_rate": 4.799363489664039e-07, + "logits/chosen": -1.654236078262329, + "logits/rejected": -1.6461032629013062, + "logps/chosen": -0.2908519208431244, + "logps/rejected": -0.2898189425468445, + "loss": 0.6891365051269531, + "loss/core": 0.6891365051269531, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.66994047164917, + "rewards/margins": 1.7045999765396118, + "rewards/rejected": 0.9653403162956238, + "step": 195 + }, + { + "epoch": 0.19109040965006568, + "grad_norm": 0.205078125, + "learning_rate": 4.779902646339721e-07, + "logits/chosen": -1.722882628440857, + "logits/rejected": -1.7433340549468994, + "logps/chosen": -0.3230392038822174, + "logps/rejected": -0.31719517707824707, + "loss": 0.689689040184021, + "loss/core": 0.689689040184021, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.6197898387908936, + "rewards/margins": 1.4049842357635498, + "rewards/rejected": 1.2148058414459229, + "step": 200 + }, + { + "epoch": 0.19586766989131732, + "grad_norm": 0.138671875, + "learning_rate": 4.759584424871301e-07, + "logits/chosen": -1.8010181188583374, + "logits/rejected": -1.880504846572876, + "logps/chosen": -0.25066837668418884, + "logps/rejected": -0.2679479718208313, + "loss": 0.6904597282409668, + "loss/core": 0.6904597282409668, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.283839702606201, + "rewards/margins": 1.1656420230865479, + "rewards/rejected": 2.1181979179382324, + "step": 205 + }, + { + "epoch": 0.20064493013256898, + "grad_norm": 0.353515625, + "learning_rate": 4.738416466110917e-07, + "logits/chosen": -1.8253402709960938, + "logits/rejected": -1.8085130453109741, + "logps/chosen": -0.3120880722999573, + "logps/rejected": -0.3225416839122772, + "loss": 0.6898230314254761, + "loss/core": 0.6898230314254761, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.835217237472534, + "rewards/margins": 1.358270525932312, + "rewards/rejected": 1.4769468307495117, + "step": 210 + }, + { + "epoch": 0.20542219037382062, + "grad_norm": 0.345703125, + "learning_rate": 4.716406730462153e-07, + "logits/chosen": -1.854882836341858, + "logits/rejected": -1.812995195388794, + "logps/chosen": -0.2745797038078308, + "logps/rejected": -0.2867583930492401, + "loss": 0.689670205116272, + "loss/core": 0.689670205116272, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.2003917694091797, + "rewards/margins": 1.5229412317276, + "rewards/rejected": 1.6774505376815796, + "step": 215 + }, + { + "epoch": 0.21019945061507225, + "grad_norm": 0.353515625, + "learning_rate": 4.693563494886454e-07, + "logits/chosen": -1.7740695476531982, + "logits/rejected": -1.7250359058380127, + "logps/chosen": -0.30255165696144104, + "logps/rejected": -0.30088987946510315, + "loss": 0.6911439895629883, + "loss/core": 0.6911439895629883, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.885136365890503, + "rewards/margins": 0.8066938519477844, + "rewards/rejected": 1.0784424543380737, + "step": 220 + }, + { + "epoch": 0.2149767108563239, + "grad_norm": 0.193359375, + "learning_rate": 4.6698953497905016e-07, + "logits/chosen": -1.8212635517120361, + "logits/rejected": -1.884863257408142, + "logps/chosen": -0.2905574440956116, + "logps/rejected": -0.2746399939060211, + "loss": 0.6877695918083191, + "loss/core": 0.6877695918083191, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.6224703788757324, + "rewards/margins": 2.204864501953125, + "rewards/rejected": 1.417605996131897, + "step": 225 + }, + { + "epoch": 0.21975397109757555, + "grad_norm": 1.875, + "learning_rate": 4.645411195795709e-07, + "logits/chosen": -1.955877661705017, + "logits/rejected": -1.9922053813934326, + "logps/chosen": -0.28303641080856323, + "logps/rejected": -0.305550754070282, + "loss": 0.6905184984207153, + "loss/core": 0.6905184984207153, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8412774801254272, + "rewards/margins": 1.0649278163909912, + "rewards/rejected": 0.776349663734436, + "step": 230 + }, + { + "epoch": 0.2245312313388272, + "grad_norm": 0.2451171875, + "learning_rate": 4.6201202403910643e-07, + "logits/chosen": -1.695408821105957, + "logits/rejected": -1.7123172283172607, + "logps/chosen": -0.2921196520328522, + "logps/rejected": -0.2863583266735077, + "loss": 0.691733717918396, + "loss/core": 0.691733717918396, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9099562168121338, + "rewards/margins": 0.5792465806007385, + "rewards/rejected": 1.33070969581604, + "step": 235 + }, + { + "epoch": 0.22930849158007882, + "grad_norm": 0.2353515625, + "learning_rate": 4.594031994470573e-07, + "logits/chosen": -1.7617086172103882, + "logits/rejected": -1.7821518182754517, + "logps/chosen": -0.2942032217979431, + "logps/rejected": -0.2943900525569916, + "loss": 0.6913124918937683, + "loss/core": 0.6913124918937683, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.7612661123275757, + "rewards/margins": 0.7373276948928833, + "rewards/rejected": 1.023938536643982, + "step": 240 + }, + { + "epoch": 0.23408575182133046, + "grad_norm": 1.8515625, + "learning_rate": 4.567156268756593e-07, + "logits/chosen": -1.919356346130371, + "logits/rejected": -1.896200180053711, + "logps/chosen": -0.2767832577228546, + "logps/rejected": -0.29151099920272827, + "loss": 0.6873155832290649, + "loss/core": 0.6873155832290649, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.6117758750915527, + "rewards/margins": 2.4639251232147217, + "rewards/rejected": 1.1478503942489624, + "step": 245 + }, + { + "epoch": 0.23886301206258212, + "grad_norm": 0.29296875, + "learning_rate": 4.5395031701104303e-07, + "logits/chosen": -1.6390726566314697, + "logits/rejected": -1.6990959644317627, + "logps/chosen": -0.3004065454006195, + "logps/rejected": -0.2949826121330261, + "loss": 0.6900212168693542, + "loss/core": 0.6900212168693542, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.3332924842834473, + "rewards/margins": 1.2689025402069092, + "rewards/rejected": 1.0643898248672485, + "step": 250 + }, + { + "epoch": 0.24364027230383375, + "grad_norm": 0.1708984375, + "learning_rate": 4.511083097731555e-07, + "logits/chosen": -1.8358352184295654, + "logits/rejected": -1.8455946445465088, + "logps/chosen": -0.28558123111724854, + "logps/rejected": -0.28805023431777954, + "loss": 0.6910611391067505, + "loss/core": 0.6910611391067505, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.444948673248291, + "rewards/margins": 0.8477497100830078, + "rewards/rejected": 1.597198724746704, + "step": 255 + }, + { + "epoch": 0.2484175325450854, + "grad_norm": 0.08935546875, + "learning_rate": 4.481906739246894e-07, + "logits/chosen": -1.9092012643814087, + "logits/rejected": -1.9937626123428345, + "logps/chosen": -0.28719645738601685, + "logps/rejected": -0.2705669701099396, + "loss": 0.691189169883728, + "loss/core": 0.691189169883728, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.4050984382629395, + "rewards/margins": 0.8234094381332397, + "rewards/rejected": 1.5816890001296997, + "step": 260 + }, + { + "epoch": 0.25319479278633705, + "grad_norm": 0.08447265625, + "learning_rate": 4.451985066691648e-07, + "logits/chosen": -1.7978445291519165, + "logits/rejected": -1.8153514862060547, + "logps/chosen": -0.29918935894966125, + "logps/rejected": -0.30210354924201965, + "loss": 0.6931018233299255, + "loss/core": 0.6931018233299255, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.2333295345306396, + "rewards/margins": 0.06469568610191345, + "rewards/rejected": 1.1686336994171143, + "step": 265 + }, + { + "epoch": 0.2579720530275887, + "grad_norm": 0.16015625, + "learning_rate": 4.421329332383158e-07, + "logits/chosen": -1.6397037506103516, + "logits/rejected": -1.7484862804412842, + "logps/chosen": -0.2880600392818451, + "logps/rejected": -0.2899312376976013, + "loss": 0.6904688477516174, + "loss/core": 0.6904688477516174, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.2856483459472656, + "rewards/margins": 1.1514227390289307, + "rewards/rejected": 2.134225606918335, + "step": 270 + }, + { + "epoch": 0.2627493132688403, + "grad_norm": 1.3125, + "learning_rate": 4.3899510646893696e-07, + "logits/chosen": -1.868186593055725, + "logits/rejected": -1.8120629787445068, + "logps/chosen": -0.29679086804389954, + "logps/rejected": -0.29938483238220215, + "loss": 0.6903032064437866, + "loss/core": 0.6903032064437866, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.321898937225342, + "rewards/margins": 1.151911735534668, + "rewards/rejected": 1.1699872016906738, + "step": 275 + }, + { + "epoch": 0.26752657351009196, + "grad_norm": 0.1435546875, + "learning_rate": 4.357862063693485e-07, + "logits/chosen": -1.7666244506835938, + "logits/rejected": -1.7176767587661743, + "logps/chosen": -0.28811368346214294, + "logps/rejected": -0.3051670491695404, + "loss": 0.6900285482406616, + "loss/core": 0.6900285482406616, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.0461502075195312, + "rewards/margins": 1.2626762390136719, + "rewards/rejected": 0.7834742069244385, + "step": 280 + }, + { + "epoch": 0.2723038337513436, + "grad_norm": 0.8359375, + "learning_rate": 4.3250743967564364e-07, + "logits/chosen": -1.8882230520248413, + "logits/rejected": -1.8435192108154297, + "logps/chosen": -0.2955285310745239, + "logps/rejected": -0.2927626669406891, + "loss": 0.6907385587692261, + "loss/core": 0.6907385587692261, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.62160325050354, + "rewards/margins": 0.9927070736885071, + "rewards/rejected": 1.6288961172103882, + "step": 285 + }, + { + "epoch": 0.27708109399259523, + "grad_norm": 1.984375, + "learning_rate": 4.29160039397884e-07, + "logits/chosen": -1.8738224506378174, + "logits/rejected": -1.9035943746566772, + "logps/chosen": -0.2865127623081207, + "logps/rejected": -0.28757035732269287, + "loss": 0.6931725740432739, + "loss/core": 0.6931725740432739, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.4819949865341187, + "rewards/margins": 0.02711881324648857, + "rewards/rejected": 1.454876184463501, + "step": 290 + }, + { + "epoch": 0.28185835423384686, + "grad_norm": 0.171875, + "learning_rate": 4.2574526435641546e-07, + "logits/chosen": -2.1733994483947754, + "logits/rejected": -2.1729981899261475, + "logps/chosen": -0.24496681988239288, + "logps/rejected": -0.25612831115722656, + "loss": 0.6915844082832336, + "loss/core": 0.6915844082832336, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 1.3056122064590454, + "rewards/margins": 0.6268185973167419, + "rewards/rejected": 0.6787935495376587, + "step": 295 + }, + { + "epoch": 0.28663561447509855, + "grad_norm": 2.359375, + "learning_rate": 4.22264398708477e-07, + "logits/chosen": -1.5950920581817627, + "logits/rejected": -1.6194194555282593, + "logps/chosen": -0.31406429409980774, + "logps/rejected": -0.3146823048591614, + "loss": 0.6867777109146118, + "loss/core": 0.6867777109146118, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.7072041034698486, + "rewards/margins": 2.6247377395629883, + "rewards/rejected": 1.08246648311615, + "step": 300 + }, + { + "epoch": 0.2914128747163502, + "grad_norm": 0.2119140625, + "learning_rate": 4.187187514652819e-07, + "logits/chosen": -1.7839601039886475, + "logits/rejected": -1.8283946514129639, + "logps/chosen": -0.27006229758262634, + "logps/rejected": -0.25947099924087524, + "loss": 0.6900721788406372, + "loss/core": 0.6900721788406372, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.1529765129089355, + "rewards/margins": 1.242806077003479, + "rewards/rejected": 0.9101703763008118, + "step": 305 + }, + { + "epoch": 0.2961901349576018, + "grad_norm": 2.5, + "learning_rate": 4.151096559997519e-07, + "logits/chosen": -1.8296064138412476, + "logits/rejected": -1.7307389974594116, + "logps/chosen": -0.2725442945957184, + "logps/rejected": -0.27926790714263916, + "loss": 0.6918596625328064, + "loss/core": 0.6918596625328064, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.6172053813934326, + "rewards/margins": 0.5880039930343628, + "rewards/rejected": 2.029201030731201, + "step": 310 + }, + { + "epoch": 0.30096739519885346, + "grad_norm": 0.158203125, + "learning_rate": 4.114384695450905e-07, + "logits/chosen": -1.722001314163208, + "logits/rejected": -1.8182132244110107, + "logps/chosen": -0.26871687173843384, + "logps/rejected": -0.2755733132362366, + "loss": 0.6892502903938293, + "loss/core": 0.6892502903938293, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.917477607727051, + "rewards/margins": 1.570624828338623, + "rewards/rejected": 1.3468530178070068, + "step": 315 + }, + { + "epoch": 0.3057446554401051, + "grad_norm": 0.134765625, + "learning_rate": 4.077065726843828e-07, + "logits/chosen": -1.7197027206420898, + "logits/rejected": -1.6908174753189087, + "logps/chosen": -0.2902878522872925, + "logps/rejected": -0.29251793026924133, + "loss": 0.6922987103462219, + "loss/core": 0.6922987103462219, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.5793731212615967, + "rewards/margins": 0.34973055124282837, + "rewards/rejected": 1.229642629623413, + "step": 320 + }, + { + "epoch": 0.31052191568135673, + "grad_norm": 1.2578125, + "learning_rate": 4.039153688314145e-07, + "logits/chosen": -1.9648669958114624, + "logits/rejected": -1.9164518117904663, + "logps/chosen": -0.2875637114048004, + "logps/rejected": -0.28430742025375366, + "loss": 0.688163697719574, + "loss/core": 0.688163697719574, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.6862456798553467, + "rewards/margins": 2.1581714153289795, + "rewards/rejected": 1.5280741453170776, + "step": 325 + }, + { + "epoch": 0.31529917592260837, + "grad_norm": 0.103515625, + "learning_rate": 4.0006628370290613e-07, + "logits/chosen": -1.8999824523925781, + "logits/rejected": -1.8628829717636108, + "logps/chosen": -0.2762024998664856, + "logps/rejected": -0.2705720365047455, + "loss": 0.6908760070800781, + "loss/core": 0.6908760070800781, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.054731845855713, + "rewards/margins": 0.9240147471427917, + "rewards/rejected": 1.130717158317566, + "step": 330 + }, + { + "epoch": 0.32007643616386, + "grad_norm": 3.0, + "learning_rate": 3.9616076478235826e-07, + "logits/chosen": -1.8090893030166626, + "logits/rejected": -1.8052091598510742, + "logps/chosen": -0.30249348282814026, + "logps/rejected": -0.31619948148727417, + "loss": 0.6897532343864441, + "loss/core": 0.6897532343864441, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.117969036102295, + "rewards/margins": 1.4059538841247559, + "rewards/rejected": 0.7120150327682495, + "step": 335 + }, + { + "epoch": 0.3248536964051117, + "grad_norm": 1.0390625, + "learning_rate": 3.922002807757129e-07, + "logits/chosen": -1.8413950204849243, + "logits/rejected": -1.9030017852783203, + "logps/chosen": -0.2811221480369568, + "logps/rejected": -0.28223514556884766, + "loss": 0.6902806162834167, + "loss/core": 0.6902806162834167, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.029637575149536, + "rewards/margins": 1.1650125980377197, + "rewards/rejected": 0.8646249771118164, + "step": 340 + }, + { + "epoch": 0.3296309566463633, + "grad_norm": 0.259765625, + "learning_rate": 3.8818632105903315e-07, + "logits/chosen": -1.8142598867416382, + "logits/rejected": -1.8903926610946655, + "logps/chosen": -0.28178513050079346, + "logps/rejected": -0.29352661967277527, + "loss": 0.6911460757255554, + "loss/core": 0.6911460757255554, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.6476151943206787, + "rewards/margins": 0.8132193684577942, + "rewards/rejected": 0.8343957662582397, + "step": 345 + }, + { + "epoch": 0.33440821688761496, + "grad_norm": 0.2060546875, + "learning_rate": 3.841203951184094e-07, + "logits/chosen": -1.8314239978790283, + "logits/rejected": -1.8243396282196045, + "logps/chosen": -0.2603296935558319, + "logps/rejected": -0.2569552958011627, + "loss": 0.6905307769775391, + "loss/core": 0.6905307769775391, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9986594915390015, + "rewards/margins": 1.056462287902832, + "rewards/rejected": 0.9421975016593933, + "step": 350 + }, + { + "epoch": 0.3391854771288666, + "grad_norm": 0.201171875, + "learning_rate": 3.800040319823038e-07, + "logits/chosen": -1.7914011478424072, + "logits/rejected": -1.8717437982559204, + "logps/chosen": -0.2806769907474518, + "logps/rejected": -0.2930828928947449, + "loss": 0.691979169845581, + "loss/core": 0.691979169845581, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.851120948791504, + "rewards/margins": 0.5048354864120483, + "rewards/rejected": 1.346285343170166, + "step": 355 + }, + { + "epoch": 0.34396273737011823, + "grad_norm": 1.40625, + "learning_rate": 3.75838779646545e-07, + "logits/chosen": -1.856149673461914, + "logits/rejected": -1.932403326034546, + "logps/chosen": -0.2797439694404602, + "logps/rejected": -0.2799036204814911, + "loss": 0.6898868680000305, + "loss/core": 0.6898868680000305, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.6374688148498535, + "rewards/margins": 1.3426095247268677, + "rewards/rejected": 1.2948594093322754, + "step": 360 + }, + { + "epoch": 0.34873999761136987, + "grad_norm": 1.0390625, + "learning_rate": 3.7162620449218993e-07, + "logits/chosen": -1.8249025344848633, + "logits/rejected": -1.8712995052337646, + "logps/chosen": -0.28412720561027527, + "logps/rejected": -0.28386378288269043, + "loss": 0.6882467865943909, + "loss/core": 0.6882467865943909, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.882859945297241, + "rewards/margins": 2.0732054710388184, + "rewards/rejected": 0.8096548318862915, + "step": 365 + }, + { + "epoch": 0.3535172578526215, + "grad_norm": 0.384765625, + "learning_rate": 3.673678906964727e-07, + "logits/chosen": -1.9151166677474976, + "logits/rejected": -2.025606155395508, + "logps/chosen": -0.28493157029151917, + "logps/rejected": -0.28482019901275635, + "loss": 0.6917159557342529, + "loss/core": 0.6917159557342529, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6433870792388916, + "rewards/margins": 0.576850175857544, + "rewards/rejected": 1.0665369033813477, + "step": 370 + }, + { + "epoch": 0.35829451809387314, + "grad_norm": 0.1875, + "learning_rate": 3.6306543963705936e-07, + "logits/chosen": -1.7109639644622803, + "logits/rejected": -1.7306935787200928, + "logps/chosen": -0.25769031047821045, + "logps/rejected": -0.25766560435295105, + "loss": 0.6912951469421387, + "loss/core": 0.6912951469421387, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.024385929107666, + "rewards/margins": 0.7996337413787842, + "rewards/rejected": 2.22475266456604, + "step": 375 + }, + { + "epoch": 0.36307177833512483, + "grad_norm": 0.26171875, + "learning_rate": 3.5872046928983623e-07, + "logits/chosen": -1.8375533819198608, + "logits/rejected": -1.8777393102645874, + "logps/chosen": -0.2792344093322754, + "logps/rejected": -0.2905234694480896, + "loss": 0.6921393871307373, + "loss/core": 0.6921393871307373, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.058870315551758, + "rewards/margins": 0.4689110219478607, + "rewards/rejected": 1.5899592638015747, + "step": 380 + }, + { + "epoch": 0.36784903857637646, + "grad_norm": 0.361328125, + "learning_rate": 3.5433461362045447e-07, + "logits/chosen": -1.8220224380493164, + "logits/rejected": -1.8235307931900024, + "logps/chosen": -0.2701142430305481, + "logps/rejected": -0.27504315972328186, + "loss": 0.6915521621704102, + "loss/core": 0.6915521621704102, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.1791160106658936, + "rewards/margins": 0.6585676670074463, + "rewards/rejected": 1.5205484628677368, + "step": 385 + }, + { + "epoch": 0.3726262988176281, + "grad_norm": 0.1103515625, + "learning_rate": 3.4990952196986305e-07, + "logits/chosen": -1.6957165002822876, + "logits/rejected": -1.7675812244415283, + "logps/chosen": -0.30696994066238403, + "logps/rejected": -0.3029830753803253, + "loss": 0.6909955739974976, + "loss/core": 0.6909955739974976, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.7640354633331299, + "rewards/margins": 0.8730796575546265, + "rewards/rejected": 0.8909558057785034, + "step": 390 + }, + { + "epoch": 0.37740355905887973, + "grad_norm": 0.3359375, + "learning_rate": 3.4544685843405875e-07, + "logits/chosen": -1.924168348312378, + "logits/rejected": -1.979353666305542, + "logps/chosen": -0.2945513129234314, + "logps/rejected": -0.2900546193122864, + "loss": 0.6909527778625488, + "loss/core": 0.6909527778625488, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 1.686589241027832, + "rewards/margins": 0.8840621113777161, + "rewards/rejected": 0.8025272488594055, + "step": 395 + }, + { + "epoch": 0.38218081930013137, + "grad_norm": 0.2373046875, + "learning_rate": 3.4094830123828786e-07, + "logits/chosen": -1.6329810619354248, + "logits/rejected": -1.6912469863891602, + "logps/chosen": -0.28971177339553833, + "logps/rejected": -0.2938389778137207, + "loss": 0.6880810260772705, + "loss/core": 0.6880810260772705, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.956228256225586, + "rewards/margins": 2.1219091415405273, + "rewards/rejected": 0.8343192934989929, + "step": 400 + }, + { + "epoch": 0.386958079541383, + "grad_norm": 0.1748046875, + "learning_rate": 3.3641554210593414e-07, + "logits/chosen": -1.8328759670257568, + "logits/rejected": -1.867147445678711, + "logps/chosen": -0.31374454498291016, + "logps/rejected": -0.3037855923175812, + "loss": 0.6904991269111633, + "loss/core": 0.6904991269111633, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.327601432800293, + "rewards/margins": 1.0923205614089966, + "rewards/rejected": 1.2352811098098755, + "step": 405 + }, + { + "epoch": 0.39173533978263464, + "grad_norm": 0.2216796875, + "learning_rate": 3.3185028562233107e-07, + "logits/chosen": -2.08441424369812, + "logits/rejected": -2.0821681022644043, + "logps/chosen": -0.27686241269111633, + "logps/rejected": -0.288194864988327, + "loss": 0.6909042596817017, + "loss/core": 0.6909042596817017, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.0894763469696045, + "rewards/margins": 0.9023572206497192, + "rewards/rejected": 1.1871192455291748, + "step": 410 + }, + { + "epoch": 0.3965126000238863, + "grad_norm": 0.22265625, + "learning_rate": 3.272542485937368e-07, + "logits/chosen": -1.667150855064392, + "logits/rejected": -1.7496572732925415, + "logps/chosen": -0.32961294054985046, + "logps/rejected": -0.32087013125419617, + "loss": 0.6915945410728455, + "loss/core": 0.6915945410728455, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.0019575357437134, + "rewards/margins": 0.6251779794692993, + "rewards/rejected": 0.37677961587905884, + "step": 415 + }, + { + "epoch": 0.40128986026513797, + "grad_norm": 0.609375, + "learning_rate": 3.226291594017137e-07, + "logits/chosen": -2.015800952911377, + "logits/rejected": -2.012826919555664, + "logps/chosen": -0.28343337774276733, + "logps/rejected": -0.29794415831565857, + "loss": 0.6899334192276001, + "loss/core": 0.6899334192276001, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.0445852279663086, + "rewards/margins": 1.314929723739624, + "rewards/rejected": 0.7296555638313293, + "step": 420 + }, + { + "epoch": 0.4060671205063896, + "grad_norm": 0.8984375, + "learning_rate": 3.1797675735315454e-07, + "logits/chosen": -1.7338758707046509, + "logits/rejected": -1.8056646585464478, + "logps/chosen": -0.291433185338974, + "logps/rejected": -0.2666085660457611, + "loss": 0.691834032535553, + "loss/core": 0.691834032535553, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.2369046211242676, + "rewards/margins": 0.6058995127677917, + "rewards/rejected": 1.6310052871704102, + "step": 425 + }, + { + "epoch": 0.41084438074764124, + "grad_norm": 1.0, + "learning_rate": 3.1329879202620047e-07, + "logits/chosen": -1.8133704662322998, + "logits/rejected": -1.8503717184066772, + "logps/chosen": -0.2787948548793793, + "logps/rejected": -0.2662429213523865, + "loss": 0.6860483884811401, + "loss/core": 0.6860483884811401, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.410374641418457, + "rewards/margins": 2.9887146949768066, + "rewards/rejected": 1.4216595888137817, + "step": 430 + }, + { + "epoch": 0.41562164098889287, + "grad_norm": 0.2431640625, + "learning_rate": 3.0859702261229613e-07, + "logits/chosen": -2.042301893234253, + "logits/rejected": -1.9976842403411865, + "logps/chosen": -0.27745509147644043, + "logps/rejected": -0.29362621903419495, + "loss": 0.6915836334228516, + "loss/core": 0.6915836334228516, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.6335694789886475, + "rewards/margins": 0.6322575807571411, + "rewards/rejected": 1.0013117790222168, + "step": 435 + }, + { + "epoch": 0.4203989012301445, + "grad_norm": 0.97265625, + "learning_rate": 3.0387321725463e-07, + "logits/chosen": -1.752429723739624, + "logits/rejected": -1.7554237842559814, + "logps/chosen": -0.2960103452205658, + "logps/rejected": -0.3004111647605896, + "loss": 0.6891411542892456, + "loss/core": 0.6891411542892456, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.0792078971862793, + "rewards/margins": 1.6967817544937134, + "rewards/rejected": 1.3824257850646973, + "step": 440 + }, + { + "epoch": 0.42517616147139614, + "grad_norm": 0.369140625, + "learning_rate": 2.991291523832075e-07, + "logits/chosen": -1.6757290363311768, + "logits/rejected": -1.6205558776855469, + "logps/chosen": -0.2944514751434326, + "logps/rejected": -0.29790419340133667, + "loss": 0.689667284488678, + "loss/core": 0.689667284488678, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.9721293449401855, + "rewards/margins": 1.4168652296066284, + "rewards/rejected": 1.5552639961242676, + "step": 445 + }, + { + "epoch": 0.4299534217126478, + "grad_norm": 1.6953125, + "learning_rate": 2.943666120468088e-07, + "logits/chosen": -1.6464523077011108, + "logits/rejected": -1.6663997173309326, + "logps/chosen": -0.29781681299209595, + "logps/rejected": -0.29050326347351074, + "loss": 0.6886329054832458, + "loss/core": 0.6886329054832458, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.4628093242645264, + "rewards/margins": 1.8670060634613037, + "rewards/rejected": 1.5958033800125122, + "step": 450 + }, + { + "epoch": 0.4347306819538994, + "grad_norm": 0.58203125, + "learning_rate": 2.8958738724208073e-07, + "logits/chosen": -1.641239881515503, + "logits/rejected": -1.832718849182129, + "logps/chosen": -0.324314147233963, + "logps/rejected": -0.31268811225891113, + "loss": 0.688445508480072, + "loss/core": 0.688445508480072, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.4496636390686035, + "rewards/margins": 1.958392858505249, + "rewards/rejected": 1.4912705421447754, + "step": 455 + }, + { + "epoch": 0.4395079421951511, + "grad_norm": 0.419921875, + "learning_rate": 2.8479327524001633e-07, + "logits/chosen": -1.6839364767074585, + "logits/rejected": -1.7105252742767334, + "logps/chosen": -0.2922411262989044, + "logps/rejected": -0.3082396388053894, + "loss": 0.688636839389801, + "loss/core": 0.688636839389801, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.7023568153381348, + "rewards/margins": 1.8497339487075806, + "rewards/rejected": 1.852622389793396, + "step": 460 + }, + { + "epoch": 0.44428520243640274, + "grad_norm": 0.166015625, + "learning_rate": 2.7998607891007493e-07, + "logits/chosen": -1.5535634756088257, + "logits/rejected": -1.6348403692245483, + "logps/chosen": -0.29626718163490295, + "logps/rejected": -0.29167088866233826, + "loss": 0.6897278428077698, + "loss/core": 0.6897278428077698, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.309946298599243, + "rewards/margins": 1.3803914785385132, + "rewards/rejected": 0.9295549392700195, + "step": 465 + }, + { + "epoch": 0.4490624626776544, + "grad_norm": 1.859375, + "learning_rate": 2.7516760604219616e-07, + "logits/chosen": -1.8788540363311768, + "logits/rejected": -1.9592180252075195, + "logps/chosen": -0.27540072798728943, + "logps/rejected": -0.2596161961555481, + "loss": 0.6886305809020996, + "loss/core": 0.6886305809020996, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.0465798377990723, + "rewards/margins": 1.8731689453125, + "rewards/rejected": 1.1734110116958618, + "step": 470 + }, + { + "epoch": 0.453839722918906, + "grad_norm": 0.6640625, + "learning_rate": 2.703396686669646e-07, + "logits/chosen": -1.8047220706939697, + "logits/rejected": -1.782589316368103, + "logps/chosen": -0.2949548363685608, + "logps/rejected": -0.31417742371559143, + "loss": 0.6925175786018372, + "loss/core": 0.6925175786018372, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8359830379486084, + "rewards/margins": 0.3028287887573242, + "rewards/rejected": 1.5331543684005737, + "step": 475 + }, + { + "epoch": 0.45861698316015764, + "grad_norm": 0.19140625, + "learning_rate": 2.6550408237417884e-07, + "logits/chosen": -1.9774296283721924, + "logits/rejected": -2.0623116493225098, + "logps/chosen": -0.27460455894470215, + "logps/rejected": -0.27350351214408875, + "loss": 0.6898782849311829, + "loss/core": 0.6898782849311829, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.366447687149048, + "rewards/margins": 1.3521076440811157, + "rewards/rejected": 1.0143399238586426, + "step": 480 + }, + { + "epoch": 0.4633942434014093, + "grad_norm": 0.173828125, + "learning_rate": 2.6066266563008265e-07, + "logits/chosen": -2.0387158393859863, + "logits/rejected": -2.014871120452881, + "logps/chosen": -0.2912241816520691, + "logps/rejected": -0.2957185208797455, + "loss": 0.6910130381584167, + "loss/core": 0.6910130381584167, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7464618682861328, + "rewards/margins": 0.8589704632759094, + "rewards/rejected": 0.8874912261962891, + "step": 485 + }, + { + "epoch": 0.4681715036426609, + "grad_norm": 0.33203125, + "learning_rate": 2.5581723909351404e-07, + "logits/chosen": -1.8752286434173584, + "logits/rejected": -1.8981329202651978, + "logps/chosen": -0.3062916398048401, + "logps/rejected": -0.2947053015232086, + "loss": 0.6905814409255981, + "loss/core": 0.6905814409255981, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.0861191749572754, + "rewards/margins": 1.0360615253448486, + "rewards/rejected": 1.0500577688217163, + "step": 490 + }, + { + "epoch": 0.47294876388391255, + "grad_norm": 0.353515625, + "learning_rate": 2.509696249312301e-07, + "logits/chosen": -1.8822190761566162, + "logits/rejected": -1.9906330108642578, + "logps/chosen": -0.29457131028175354, + "logps/rejected": -0.2900959551334381, + "loss": 0.6889396905899048, + "loss/core": 0.6889396905899048, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.6438632011413574, + "rewards/margins": 1.7446101903915405, + "rewards/rejected": 0.8992528915405273, + "step": 495 + }, + { + "epoch": 0.47772602412516424, + "grad_norm": 0.447265625, + "learning_rate": 2.461216461326642e-07, + "logits/chosen": -1.7823041677474976, + "logits/rejected": -1.7042522430419922, + "logps/chosen": -0.276335746049881, + "logps/rejected": -0.30160245299339294, + "loss": 0.6881250143051147, + "loss/core": 0.6881250143051147, + "rewards/accuracies": 0.875, + "rewards/chosen": 4.211169242858887, + "rewards/margins": 2.174818277359009, + "rewards/rejected": 2.036350727081299, + "step": 500 + }, + { + "epoch": 0.4825032843664159, + "grad_norm": 1.28125, + "learning_rate": 2.412751258243748e-07, + "logits/chosen": -1.5979036092758179, + "logits/rejected": -1.695265531539917, + "logps/chosen": -0.28008055686950684, + "logps/rejected": -0.27932852506637573, + "loss": 0.6890323162078857, + "loss/core": 0.6890323162078857, + "rewards/accuracies": 0.8125, + "rewards/chosen": 3.665738344192505, + "rewards/margins": 1.8280740976333618, + "rewards/rejected": 1.8376646041870117, + "step": 505 + }, + { + "epoch": 0.4872805446076675, + "grad_norm": 0.060302734375, + "learning_rate": 2.3643188658444158e-07, + "logits/chosen": -1.8131134510040283, + "logits/rejected": -1.9090598821640015, + "logps/chosen": -0.31393712759017944, + "logps/rejected": -0.29055261611938477, + "loss": 0.6901880502700806, + "loss/core": 0.6901880502700806, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.030540704727173, + "rewards/margins": 1.2225555181503296, + "rewards/rejected": 0.8079849481582642, + "step": 510 + }, + { + "epoch": 0.49205780484891914, + "grad_norm": 0.369140625, + "learning_rate": 2.315937497570688e-07, + "logits/chosen": -1.6081165075302124, + "logits/rejected": -1.6191253662109375, + "logps/chosen": -0.28171616792678833, + "logps/rejected": -0.30241820216178894, + "loss": 0.6910374164581299, + "loss/core": 0.6910374164581299, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.5543110370635986, + "rewards/margins": 0.8602564930915833, + "rewards/rejected": 1.6940546035766602, + "step": 515 + }, + { + "epoch": 0.4968350650901708, + "grad_norm": 0.50390625, + "learning_rate": 2.2676253476765194e-07, + "logits/chosen": -1.8473793268203735, + "logits/rejected": -1.940637230873108, + "logps/chosen": -0.3185504972934723, + "logps/rejected": -0.3340589106082916, + "loss": 0.6879608035087585, + "loss/core": 0.6879608035087585, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.108973503112793, + "rewards/margins": 2.145254373550415, + "rewards/rejected": 0.9637192487716675, + "step": 520 + }, + { + "epoch": 0.5016123253314224, + "grad_norm": 1.5703125, + "learning_rate": 2.2194005843856633e-07, + "logits/chosen": -1.9106519222259521, + "logits/rejected": -1.854853630065918, + "logps/chosen": -0.2881598472595215, + "logps/rejected": -0.29906392097473145, + "loss": 0.6862886548042297, + "loss/core": 0.6862886548042297, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.429782390594482, + "rewards/margins": 2.9549190998077393, + "rewards/rejected": 1.4748632907867432, + "step": 525 + }, + { + "epoch": 0.5063895855726741, + "grad_norm": 0.068359375, + "learning_rate": 2.1712813430593434e-07, + "logits/chosen": -1.698067307472229, + "logits/rejected": -1.693956732749939, + "logps/chosen": -0.28551727533340454, + "logps/rejected": -0.2868136763572693, + "loss": 0.6893957853317261, + "loss/core": 0.6893957853317261, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.0555648803710938, + "rewards/margins": 1.5484126806259155, + "rewards/rejected": 1.5071521997451782, + "step": 530 + }, + { + "epoch": 0.5111668458139257, + "grad_norm": 0.62890625, + "learning_rate": 2.123285719376292e-07, + "logits/chosen": -1.7900841236114502, + "logits/rejected": -1.7464206218719482, + "logps/chosen": -0.28272828459739685, + "logps/rejected": -0.27548089623451233, + "loss": 0.6883460283279419, + "loss/core": 0.6883460283279419, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.122218370437622, + "rewards/margins": 2.007307529449463, + "rewards/rejected": 1.1149110794067383, + "step": 535 + }, + { + "epoch": 0.5159441060551774, + "grad_norm": 0.10400390625, + "learning_rate": 2.0754317625276982e-07, + "logits/chosen": -1.7317934036254883, + "logits/rejected": -1.6521661281585693, + "logps/chosen": -0.27843254804611206, + "logps/rejected": -0.2827979624271393, + "loss": 0.693123996257782, + "loss/core": 0.693123996257782, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.5914747714996338, + "rewards/margins": 0.03556249290704727, + "rewards/rejected": 1.5559122562408447, + "step": 540 + }, + { + "epoch": 0.520721366296429, + "grad_norm": 1.578125, + "learning_rate": 2.0277374684296498e-07, + "logits/chosen": -1.9993709325790405, + "logits/rejected": -1.906482458114624, + "logps/chosen": -0.2804117798805237, + "logps/rejected": -0.29083675146102905, + "loss": 0.6889429092407227, + "loss/core": 0.6889429092407227, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.6694138050079346, + "rewards/margins": 1.7691431045532227, + "rewards/rejected": 0.9002708196640015, + "step": 545 + }, + { + "epoch": 0.5254986265376806, + "grad_norm": 0.265625, + "learning_rate": 1.980220772955602e-07, + "logits/chosen": -1.5809681415557861, + "logits/rejected": -1.5856965780258179, + "logps/chosen": -0.2750489413738251, + "logps/rejected": -0.2972986698150635, + "loss": 0.6903039216995239, + "loss/core": 0.6903039216995239, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.7791810035705566, + "rewards/margins": 1.1509668827056885, + "rewards/rejected": 1.628213882446289, + "step": 550 + }, + { + "epoch": 0.5302758867789323, + "grad_norm": 0.1396484375, + "learning_rate": 1.932899545191433e-07, + "logits/chosen": -1.8478825092315674, + "logits/rejected": -1.8661056756973267, + "logps/chosen": -0.31153231859207153, + "logps/rejected": -0.2762632966041565, + "loss": 0.6875492334365845, + "loss/core": 0.6875492334365845, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.2964320182800293, + "rewards/margins": 2.3140857219696045, + "rewards/rejected": 0.9823463559150696, + "step": 555 + }, + { + "epoch": 0.5350531470201839, + "grad_norm": 0.408203125, + "learning_rate": 1.885791580715609e-07, + "logits/chosen": -1.7739731073379517, + "logits/rejected": -1.8065807819366455, + "logps/chosen": -0.2748362421989441, + "logps/rejected": -0.2793380916118622, + "loss": 0.6899937987327576, + "loss/core": 0.6899937987327576, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.8241641521453857, + "rewards/margins": 1.279166340827942, + "rewards/rejected": 1.544998049736023, + "step": 560 + }, + { + "epoch": 0.5398304072614356, + "grad_norm": 0.376953125, + "learning_rate": 1.8389145949069951e-07, + "logits/chosen": -1.8410762548446655, + "logits/rejected": -1.8144041299819946, + "logps/chosen": -0.2863827347755432, + "logps/rejected": -0.30311352014541626, + "loss": 0.6904335618019104, + "loss/core": 0.6904335618019104, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.7921481132507324, + "rewards/margins": 1.1107183694839478, + "rewards/rejected": 1.6814296245574951, + "step": 565 + }, + { + "epoch": 0.5446076675026872, + "grad_norm": 1.046875, + "learning_rate": 1.792286216282824e-07, + "logits/chosen": -1.79177725315094, + "logits/rejected": -1.8119325637817383, + "logps/chosen": -0.2888721525669098, + "logps/rejected": -0.2847359776496887, + "loss": 0.689185619354248, + "loss/core": 0.689185619354248, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.3419628143310547, + "rewards/margins": 1.616077184677124, + "rewards/rejected": 1.7258857488632202, + "step": 570 + }, + { + "epoch": 0.5493849277439389, + "grad_norm": 0.111328125, + "learning_rate": 1.745923979869336e-07, + "logits/chosen": -1.88888418674469, + "logits/rejected": -1.9161269664764404, + "logps/chosen": -0.29976314306259155, + "logps/rejected": -0.3220902383327484, + "loss": 0.6900423169136047, + "loss/core": 0.6900423169136047, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.8517236709594727, + "rewards/margins": 1.2799955606460571, + "rewards/rejected": 0.5717281103134155, + "step": 575 + }, + { + "epoch": 0.5541621879851905, + "grad_norm": 0.07568359375, + "learning_rate": 1.6998453206075708e-07, + "logits/chosen": -1.9753665924072266, + "logits/rejected": -2.079908847808838, + "logps/chosen": -0.26681944727897644, + "logps/rejected": -0.2805357575416565, + "loss": 0.6920329332351685, + "loss/core": 0.6920329332351685, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.4170547723770142, + "rewards/margins": 0.45395904779434204, + "rewards/rejected": 0.9630956649780273, + "step": 580 + }, + { + "epoch": 0.5589394482264421, + "grad_norm": 0.2265625, + "learning_rate": 1.6540675667967973e-07, + "logits/chosen": -1.9732328653335571, + "logits/rejected": -1.867099404335022, + "logps/chosen": -0.2693113684654236, + "logps/rejected": -0.28243768215179443, + "loss": 0.6910297274589539, + "loss/core": 0.6910297274589539, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.231765031814575, + "rewards/margins": 0.8599586486816406, + "rewards/rejected": 1.3718063831329346, + "step": 585 + }, + { + "epoch": 0.5637167084676937, + "grad_norm": 0.1318359375, + "learning_rate": 1.60860793357805e-07, + "logits/chosen": -1.827453851699829, + "logits/rejected": -1.8766067028045654, + "logps/chosen": -0.2868167757987976, + "logps/rejected": -0.2861761450767517, + "loss": 0.6869019269943237, + "loss/core": 0.6869019269943237, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.6281845569610596, + "rewards/margins": 2.5879273414611816, + "rewards/rejected": 1.0402569770812988, + "step": 590 + }, + { + "epoch": 0.5684939687089454, + "grad_norm": 0.1767578125, + "learning_rate": 1.5634835164602196e-07, + "logits/chosen": -1.771225929260254, + "logits/rejected": -1.781171202659607, + "logps/chosen": -0.27533966302871704, + "logps/rejected": -0.2719757854938507, + "loss": 0.6911031603813171, + "loss/core": 0.6911031603813171, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.7094968557357788, + "rewards/margins": 0.8211535215377808, + "rewards/rejected": 0.8883434534072876, + "step": 595 + }, + { + "epoch": 0.5732712289501971, + "grad_norm": 0.314453125, + "learning_rate": 1.518711284891132e-07, + "logits/chosen": -1.8096263408660889, + "logits/rejected": -1.7713695764541626, + "logps/chosen": -0.26596006751060486, + "logps/rejected": -0.28354257345199585, + "loss": 0.691169023513794, + "loss/core": 0.691169023513794, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.876749038696289, + "rewards/margins": 0.8061097860336304, + "rewards/rejected": 1.0706393718719482, + "step": 600 + }, + { + "epoch": 0.5780484891914487, + "grad_norm": 0.6953125, + "learning_rate": 1.47430807587603e-07, + "logits/chosen": -1.9364635944366455, + "logits/rejected": -2.0032143592834473, + "logps/chosen": -0.2760660648345947, + "logps/rejected": -0.2748982906341553, + "loss": 0.6860328316688538, + "loss/core": 0.6860328316688538, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 4.224491596221924, + "rewards/margins": 3.010608196258545, + "rewards/rejected": 1.2138831615447998, + "step": 605 + }, + { + "epoch": 0.5828257494327004, + "grad_norm": 0.5078125, + "learning_rate": 1.430290587645865e-07, + "logits/chosen": -1.6363016366958618, + "logits/rejected": -1.6160857677459717, + "logps/chosen": -0.27192258834838867, + "logps/rejected": -0.2753311097621918, + "loss": 0.6874152421951294, + "loss/core": 0.6874152421951294, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 4.3153300285339355, + "rewards/margins": 2.379615068435669, + "rewards/rejected": 1.9357149600982666, + "step": 610 + }, + { + "epoch": 0.587603009673952, + "grad_norm": 0.19921875, + "learning_rate": 1.3866753733777765e-07, + "logits/chosen": -1.4840619564056396, + "logits/rejected": -1.5672607421875, + "logps/chosen": -0.3140578866004944, + "logps/rejected": -0.30204764008522034, + "loss": 0.6901291012763977, + "loss/core": 0.6901291012763977, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.7394516468048096, + "rewards/margins": 1.2340558767318726, + "rewards/rejected": 1.505395770072937, + "step": 615 + }, + { + "epoch": 0.5923802699152036, + "grad_norm": 0.1572265625, + "learning_rate": 1.343478834970121e-07, + "logits/chosen": -1.865546464920044, + "logits/rejected": -1.8963212966918945, + "logps/chosen": -0.28501859307289124, + "logps/rejected": -0.2918963134288788, + "loss": 0.692618191242218, + "loss/core": 0.692618191242218, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.7005846500396729, + "rewards/margins": 0.2428729087114334, + "rewards/rejected": 1.4577115774154663, + "step": 620 + }, + { + "epoch": 0.5971575301564552, + "grad_norm": 3.3125, + "learning_rate": 1.3007172168743852e-07, + "logits/chosen": -1.7032978534698486, + "logits/rejected": -1.6907075643539429, + "logps/chosen": -0.30372241139411926, + "logps/rejected": -0.33234214782714844, + "loss": 0.6876958012580872, + "loss/core": 0.6876958012580872, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.8974437713623047, + "rewards/margins": 2.3479323387145996, + "rewards/rejected": 1.5495115518569946, + "step": 625 + }, + { + "epoch": 0.6019347903977069, + "grad_norm": 0.2255859375, + "learning_rate": 1.25840659998631e-07, + "logits/chosen": -1.7279109954833984, + "logits/rejected": -1.8030815124511719, + "logps/chosen": -0.2717967629432678, + "logps/rejected": -0.2828608751296997, + "loss": 0.6890043020248413, + "loss/core": 0.6890043020248413, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.9497199058532715, + "rewards/margins": 1.700548529624939, + "rewards/rejected": 1.249171257019043, + "step": 630 + }, + { + "epoch": 0.6067120506389586, + "grad_norm": 1.0, + "learning_rate": 1.2165628955985313e-07, + "logits/chosen": -1.7578712701797485, + "logits/rejected": -1.8161617517471313, + "logps/chosen": -0.303678959608078, + "logps/rejected": -0.2981964647769928, + "loss": 0.6900124549865723, + "loss/core": 0.6900124549865723, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.3181376457214355, + "rewards/margins": 1.2847710847854614, + "rewards/rejected": 1.0333666801452637, + "step": 635 + }, + { + "epoch": 0.6114893108802102, + "grad_norm": 0.125, + "learning_rate": 1.175201839416988e-07, + "logits/chosen": -1.9067970514297485, + "logits/rejected": -1.9922056198120117, + "logps/chosen": -0.29978805780410767, + "logps/rejected": -0.2979918420314789, + "loss": 0.6902927160263062, + "loss/core": 0.6902927160263062, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.974582314491272, + "rewards/margins": 1.1694762706756592, + "rewards/rejected": 0.8051063418388367, + "step": 640 + }, + { + "epoch": 0.6162665711214619, + "grad_norm": 0.28515625, + "learning_rate": 1.1343389856433658e-07, + "logits/chosen": -1.998485803604126, + "logits/rejected": -2.0181636810302734, + "logps/chosen": -0.28951841592788696, + "logps/rejected": -0.2937147319316864, + "loss": 0.6920903921127319, + "loss/core": 0.6920903921127319, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.685060739517212, + "rewards/margins": 0.43871307373046875, + "rewards/rejected": 1.2463476657867432, + "step": 645 + }, + { + "epoch": 0.6210438313627135, + "grad_norm": 0.1767578125, + "learning_rate": 1.0939897011258e-07, + "logits/chosen": -1.9956214427947998, + "logits/rejected": -1.9716342687606812, + "logps/chosen": -0.2891247570514679, + "logps/rejected": -0.2986387312412262, + "loss": 0.691946804523468, + "loss/core": 0.691946804523468, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.3371341228485107, + "rewards/margins": 0.49254298210144043, + "rewards/rejected": 0.8445911407470703, + "step": 650 + }, + { + "epoch": 0.6258210916039652, + "grad_norm": 0.8359375, + "learning_rate": 1.0541691595800336e-07, + "logits/chosen": -1.7673661708831787, + "logits/rejected": -1.8068790435791016, + "logps/chosen": -0.282105028629303, + "logps/rejected": -0.2889975309371948, + "loss": 0.690642774105072, + "loss/core": 0.690642774105072, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.0060832500457764, + "rewards/margins": 1.010461688041687, + "rewards/rejected": 0.9956215023994446, + "step": 655 + }, + { + "epoch": 0.6305983518452167, + "grad_norm": 0.11474609375, + "learning_rate": 1.0148923358832021e-07, + "logits/chosen": -1.9046757221221924, + "logits/rejected": -1.9204623699188232, + "logps/chosen": -0.29088854789733887, + "logps/rejected": -0.30732461810112, + "loss": 0.6896063685417175, + "loss/core": 0.6896063685417175, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.8072502613067627, + "rewards/margins": 1.5126233100891113, + "rewards/rejected": 1.2946269512176514, + "step": 660 + }, + { + "epoch": 0.6353756120864684, + "grad_norm": 0.5390625, + "learning_rate": 9.761740004423926e-08, + "logits/chosen": -1.9125967025756836, + "logits/rejected": -1.8837387561798096, + "logps/chosen": -0.2830875813961029, + "logps/rejected": -0.27817443013191223, + "loss": 0.6911165118217468, + "loss/core": 0.6911165118217468, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.3600566387176514, + "rewards/margins": 0.8475581407546997, + "rewards/rejected": 1.5124986171722412, + "step": 665 + }, + { + "epoch": 0.64015287232772, + "grad_norm": 0.11083984375, + "learning_rate": 9.380287136400999e-08, + "logits/chosen": -1.7506167888641357, + "logits/rejected": -1.8097076416015625, + "logps/chosen": -0.2727493345737457, + "logps/rejected": -0.3100685477256775, + "loss": 0.6875694394111633, + "loss/core": 0.6875694394111633, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.660663604736328, + "rewards/margins": 2.3024368286132812, + "rewards/rejected": 1.3582265377044678, + "step": 670 + }, + { + "epoch": 0.6449301325689717, + "grad_norm": 0.19921875, + "learning_rate": 9.004708203586628e-08, + "logits/chosen": -1.661525011062622, + "logits/rejected": -1.6602637767791748, + "logps/chosen": -0.2878963053226471, + "logps/rejected": -0.28721392154693604, + "loss": 0.6913235187530518, + "loss/core": 0.6913235187530518, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6275174617767334, + "rewards/margins": 0.7366446256637573, + "rewards/rejected": 0.8908728361129761, + "step": 675 + }, + { + "epoch": 0.6497073928102234, + "grad_norm": 0.10791015625, + "learning_rate": 8.635144445857407e-08, + "logits/chosen": -1.8359218835830688, + "logits/rejected": -1.7717132568359375, + "logps/chosen": -0.2822825312614441, + "logps/rejected": -0.2898341715335846, + "loss": 0.6913445591926575, + "loss/core": 0.6913445591926575, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.1436684131622314, + "rewards/margins": 0.7406994700431824, + "rewards/rejected": 1.4029688835144043, + "step": 680 + }, + { + "epoch": 0.654484653051475, + "grad_norm": 0.1455078125, + "learning_rate": 8.271734841028552e-08, + "logits/chosen": -1.6390689611434937, + "logits/rejected": -1.651624321937561, + "logps/chosen": -0.2526906132698059, + "logps/rejected": -0.27529576420783997, + "loss": 0.6899958848953247, + "loss/core": 0.6899958848953247, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.895305633544922, + "rewards/margins": 1.3431675434112549, + "rewards/rejected": 1.552138090133667, + "step": 685 + }, + { + "epoch": 0.6592619132927267, + "grad_norm": 0.1337890625, + "learning_rate": 7.91461605259007e-08, + "logits/chosen": -1.5240414142608643, + "logits/rejected": -1.5546308755874634, + "logps/chosen": -0.3081050515174866, + "logps/rejected": -0.32126057147979736, + "loss": 0.6916640996932983, + "loss/core": 0.6916640996932983, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.7689863443374634, + "rewards/margins": 0.6094117164611816, + "rewards/rejected": 1.1595746278762817, + "step": 690 + }, + { + "epoch": 0.6640391735339782, + "grad_norm": 0.1640625, + "learning_rate": 7.563922378313217e-08, + "logits/chosen": -1.8862450122833252, + "logits/rejected": -1.9309171438217163, + "logps/chosen": -0.30440554022789, + "logps/rejected": -0.2918163239955902, + "loss": 0.6900017857551575, + "loss/core": 0.6900017857551575, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9893100261688232, + "rewards/margins": 1.2960065603256226, + "rewards/rejected": 0.6933035850524902, + "step": 695 + }, + { + "epoch": 0.6688164337752299, + "grad_norm": 2.703125, + "learning_rate": 7.219785699746572e-08, + "logits/chosen": -1.7974674701690674, + "logits/rejected": -1.8728835582733154, + "logps/chosen": -0.2968365252017975, + "logps/rejected": -0.3178980350494385, + "loss": 0.6916329264640808, + "loss/core": 0.6916329264640808, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.230217933654785, + "rewards/margins": 0.6277396082878113, + "rewards/rejected": 1.60247802734375, + "step": 700 + }, + { + "epoch": 0.6735936940164815, + "grad_norm": 0.52734375, + "learning_rate": 6.882335432620779e-08, + "logits/chosen": -1.846928596496582, + "logits/rejected": -1.8282711505889893, + "logps/chosen": -0.28012657165527344, + "logps/rejected": -0.26722002029418945, + "loss": 0.6885994672775269, + "loss/core": 0.6885994672775269, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.4068984985351562, + "rewards/margins": 1.8730993270874023, + "rewards/rejected": 1.533799171447754, + "step": 705 + }, + { + "epoch": 0.6783709542577332, + "grad_norm": 1.6953125, + "learning_rate": 6.551698478180589e-08, + "logits/chosen": -1.6279361248016357, + "logits/rejected": -1.647020936012268, + "logps/chosen": -0.2771731913089752, + "logps/rejected": -0.29674896597862244, + "loss": 0.6890315413475037, + "loss/core": 0.6890315413475037, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.5971343517303467, + "rewards/margins": 1.6893699169158936, + "rewards/rejected": 1.9077644348144531, + "step": 710 + }, + { + "epoch": 0.6831482144989849, + "grad_norm": 0.1572265625, + "learning_rate": 6.22799917546252e-08, + "logits/chosen": -1.8586616516113281, + "logits/rejected": -1.8125330209732056, + "logps/chosen": -0.2845900058746338, + "logps/rejected": -0.32791072130203247, + "loss": 0.6905220746994019, + "loss/core": 0.6905220746994019, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8608309030532837, + "rewards/margins": 1.073857307434082, + "rewards/rejected": 0.7869734168052673, + "step": 715 + }, + { + "epoch": 0.6879254747402365, + "grad_norm": 0.423828125, + "learning_rate": 5.9113592545359944e-08, + "logits/chosen": -1.7632392644882202, + "logits/rejected": -1.7893062829971313, + "logps/chosen": -0.2961721122264862, + "logps/rejected": -0.300367534160614, + "loss": 0.6917133331298828, + "loss/core": 0.6917133331298828, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.103755474090576, + "rewards/margins": 0.6050199866294861, + "rewards/rejected": 1.4987354278564453, + "step": 720 + }, + { + "epoch": 0.6927027349814882, + "grad_norm": 0.1708984375, + "learning_rate": 5.601897790725643e-08, + "logits/chosen": -1.9446632862091064, + "logits/rejected": -1.969635248184204, + "logps/chosen": -0.2848110795021057, + "logps/rejected": -0.27734309434890747, + "loss": 0.6910138130187988, + "loss/core": 0.6910138130187988, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.5936205387115479, + "rewards/margins": 0.8628653287887573, + "rewards/rejected": 0.7307552099227905, + "step": 725 + }, + { + "epoch": 0.6974799952227397, + "grad_norm": 1.4375, + "learning_rate": 5.299731159831952e-08, + "logits/chosen": -1.901888132095337, + "logits/rejected": -1.9483245611190796, + "logps/chosen": -0.2863468527793884, + "logps/rejected": -0.2913190722465515, + "loss": 0.6928960680961609, + "loss/core": 0.6928960680961609, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.0203444957733154, + "rewards/margins": 0.1184670701622963, + "rewards/rejected": 0.9018775224685669, + "step": 730 + }, + { + "epoch": 0.7022572554639914, + "grad_norm": 0.408203125, + "learning_rate": 5.0049729943671013e-08, + "logits/chosen": -1.8452632427215576, + "logits/rejected": -1.8946354389190674, + "logps/chosen": -0.319896399974823, + "logps/rejected": -0.3175990581512451, + "loss": 0.6912761926651001, + "loss/core": 0.6912761926651001, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.4188251495361328, + "rewards/margins": 0.7536794543266296, + "rewards/rejected": 0.6651455760002136, + "step": 735 + }, + { + "epoch": 0.707034515705243, + "grad_norm": 0.1826171875, + "learning_rate": 4.7177341408223994e-08, + "logits/chosen": -1.7158161401748657, + "logits/rejected": -1.7437493801116943, + "logps/chosen": -0.31194645166397095, + "logps/rejected": -0.3125692307949066, + "loss": 0.6917774081230164, + "loss/core": 0.6917774081230164, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.575186252593994, + "rewards/margins": 0.7101010084152222, + "rewards/rejected": 1.865085244178772, + "step": 740 + }, + { + "epoch": 0.7118117759464947, + "grad_norm": 1.0546875, + "learning_rate": 4.438122617983442e-08, + "logits/chosen": -1.7589060068130493, + "logits/rejected": -1.833776831626892, + "logps/chosen": -0.28943413496017456, + "logps/rejected": -0.28058308362960815, + "loss": 0.6892428994178772, + "loss/core": 0.6892428994178772, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.131422758102417, + "rewards/margins": 1.6019914150238037, + "rewards/rejected": 1.5294312238693237, + "step": 745 + }, + { + "epoch": 0.7165890361877463, + "grad_norm": 0.451171875, + "learning_rate": 4.1662435763087114e-08, + "logits/chosen": -1.912740707397461, + "logits/rejected": -1.9157087802886963, + "logps/chosen": -0.31203845143318176, + "logps/rejected": -0.3028412461280823, + "loss": 0.6934337615966797, + "loss/core": 0.6934337615966797, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.2026363611221313, + "rewards/margins": -0.061604440212249756, + "rewards/rejected": 1.2642407417297363, + "step": 750 + }, + { + "epoch": 0.721366296428998, + "grad_norm": 0.373046875, + "learning_rate": 3.902199258386732e-08, + "logits/chosen": -1.7527711391448975, + "logits/rejected": -1.647605299949646, + "logps/chosen": -0.3076718747615814, + "logps/rejected": -0.2929893732070923, + "loss": 0.6915236115455627, + "loss/core": 0.6915236115455627, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.39526629447937, + "rewards/margins": 0.6723926663398743, + "rewards/rejected": 1.7228736877441406, + "step": 755 + }, + { + "epoch": 0.7261435566702497, + "grad_norm": 0.201171875, + "learning_rate": 3.646088960486862e-08, + "logits/chosen": -1.7982404232025146, + "logits/rejected": -1.840656042098999, + "logps/chosen": -0.3029170632362366, + "logps/rejected": -0.29293784499168396, + "loss": 0.6903898119926453, + "loss/core": 0.6903898119926453, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.9886500835418701, + "rewards/margins": 1.1183176040649414, + "rewards/rejected": 0.8703324198722839, + "step": 760 + }, + { + "epoch": 0.7309208169115012, + "grad_norm": 0.328125, + "learning_rate": 3.398008995217988e-08, + "logits/chosen": -1.7162113189697266, + "logits/rejected": -1.7504310607910156, + "logps/chosen": -0.2765401601791382, + "logps/rejected": -0.2717045247554779, + "loss": 0.691356360912323, + "loss/core": 0.691356360912323, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.7894961833953857, + "rewards/margins": 0.7241676449775696, + "rewards/rejected": 1.0653284788131714, + "step": 765 + }, + { + "epoch": 0.7356980771527529, + "grad_norm": 0.53125, + "learning_rate": 3.1580526553093315e-08, + "logits/chosen": -1.6956571340560913, + "logits/rejected": -1.7221298217773438, + "logps/chosen": -0.3087131679058075, + "logps/rejected": -0.28246551752090454, + "loss": 0.6899880170822144, + "loss/core": 0.6899880170822144, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.7439963817596436, + "rewards/margins": 1.3693008422851562, + "rewards/rejected": 2.3746957778930664, + "step": 770 + }, + { + "epoch": 0.7404753373940045, + "grad_norm": 0.07861328125, + "learning_rate": 2.9263101785268252e-08, + "logits/chosen": -1.8691062927246094, + "logits/rejected": -1.896593451499939, + "logps/chosen": -0.306645005941391, + "logps/rejected": -0.3000980317592621, + "loss": 0.6917179822921753, + "loss/core": 0.6917179822921753, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.5224562883377075, + "rewards/margins": 0.5784978866577148, + "rewards/rejected": 0.9439584016799927, + "step": 775 + }, + { + "epoch": 0.7452525976352562, + "grad_norm": 0.287109375, + "learning_rate": 2.7028687137384264e-08, + "logits/chosen": -1.7383358478546143, + "logits/rejected": -1.7722053527832031, + "logps/chosen": -0.301580548286438, + "logps/rejected": -0.3000149130821228, + "loss": 0.6916896104812622, + "loss/core": 0.6916896104812622, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.896596908569336, + "rewards/margins": 0.5912862420082092, + "rewards/rejected": 1.305310606956482, + "step": 780 + }, + { + "epoch": 0.7500298578765078, + "grad_norm": 0.107421875, + "learning_rate": 2.4878122881409447e-08, + "logits/chosen": -1.5884864330291748, + "logits/rejected": -1.562572717666626, + "logps/chosen": -0.30816543102264404, + "logps/rejected": -0.2903262674808502, + "loss": 0.6882777810096741, + "loss/core": 0.6882777810096741, + "rewards/accuracies": 0.8125, + "rewards/chosen": 3.4592926502227783, + "rewards/margins": 2.0797040462493896, + "rewards/rejected": 1.3795884847640991, + "step": 785 + }, + { + "epoch": 0.7548071181177595, + "grad_norm": 3.609375, + "learning_rate": 2.2812217756608937e-08, + "logits/chosen": -1.8189325332641602, + "logits/rejected": -1.710569143295288, + "logps/chosen": -0.28520792722702026, + "logps/rejected": -0.317575603723526, + "loss": 0.69153892993927, + "loss/core": 0.69153892993927, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.6281354427337646, + "rewards/margins": 0.6913524866104126, + "rewards/rejected": 1.9367828369140625, + "step": 790 + }, + { + "epoch": 0.7595843783590112, + "grad_norm": 0.400390625, + "learning_rate": 2.0831748665410763e-08, + "logits/chosen": -1.7783273458480835, + "logits/rejected": -1.7968485355377197, + "logps/chosen": -0.30635857582092285, + "logps/rejected": -0.29732808470726013, + "loss": 0.6901718378067017, + "loss/core": 0.6901718378067017, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.3910913467407227, + "rewards/margins": 1.2171629667282104, + "rewards/rejected": 1.173928141593933, + "step": 795 + }, + { + "epoch": 0.7643616386002627, + "grad_norm": 0.154296875, + "learning_rate": 1.8937460381244967e-08, + "logits/chosen": -1.8456178903579712, + "logits/rejected": -1.8204724788665771, + "logps/chosen": -0.2890564501285553, + "logps/rejected": -0.28678005933761597, + "loss": 0.6904189586639404, + "loss/core": 0.6904189586639404, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.2987735271453857, + "rewards/margins": 1.1124370098114014, + "rewards/rejected": 1.186336636543274, + "step": 800 + }, + { + "epoch": 0.7691388988415144, + "grad_norm": 0.25, + "learning_rate": 1.713006526846439e-08, + "logits/chosen": -1.7271690368652344, + "logits/rejected": -1.7207380533218384, + "logps/chosen": -0.2747926414012909, + "logps/rejected": -0.26928240060806274, + "loss": 0.69256192445755, + "loss/core": 0.69256192445755, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.2577762603759766, + "rewards/margins": 0.2586982250213623, + "rewards/rejected": 1.9990781545639038, + "step": 805 + }, + { + "epoch": 0.773916159082766, + "grad_norm": 0.15234375, + "learning_rate": 1.541024301445404e-08, + "logits/chosen": -1.8081337213516235, + "logits/rejected": -1.7127681970596313, + "logps/chosen": -0.3074837327003479, + "logps/rejected": -0.316093385219574, + "loss": 0.6930532455444336, + "loss/core": 0.6930532455444336, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.5384314060211182, + "rewards/margins": 0.054566673934459686, + "rewards/rejected": 1.4838647842407227, + "step": 810 + }, + { + "epoch": 0.7786934193240177, + "grad_norm": 0.353515625, + "learning_rate": 1.3778640374027983e-08, + "logits/chosen": -1.8720842599868774, + "logits/rejected": -1.9219611883163452, + "logps/chosen": -0.28327468037605286, + "logps/rejected": -0.28367432951927185, + "loss": 0.6905892491340637, + "loss/core": 0.6905892491340637, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.1285481452941895, + "rewards/margins": 1.0284489393234253, + "rewards/rejected": 1.1000992059707642, + "step": 815 + }, + { + "epoch": 0.7834706795652693, + "grad_norm": 0.115234375, + "learning_rate": 1.2235870926211616e-08, + "logits/chosen": -1.7334009408950806, + "logits/rejected": -1.737236738204956, + "logps/chosen": -0.28783631324768066, + "logps/rejected": -0.3172575533390045, + "loss": 0.6899994611740112, + "loss/core": 0.6899994611740112, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.9646492004394531, + "rewards/margins": 1.2791537046432495, + "rewards/rejected": 0.6854956150054932, + "step": 820 + }, + { + "epoch": 0.788247939806521, + "grad_norm": 0.546875, + "learning_rate": 1.0782514843499652e-08, + "logits/chosen": -1.7240092754364014, + "logits/rejected": -1.8827893733978271, + "logps/chosen": -0.2997475564479828, + "logps/rejected": -0.283795028924942, + "loss": 0.690019965171814, + "loss/core": 0.690019965171814, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.4867701530456543, + "rewards/margins": 1.2625161409378052, + "rewards/rejected": 1.22425377368927, + "step": 825 + }, + { + "epoch": 0.7930252000477725, + "grad_norm": 0.2412109375, + "learning_rate": 9.419118673676924e-09, + "logits/chosen": -1.9089800119400024, + "logits/rejected": -1.8700017929077148, + "logps/chosen": -0.27884355187416077, + "logps/rejected": -0.30505234003067017, + "loss": 0.6930972933769226, + "loss/core": 0.6930972933769226, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.7232449054718018, + "rewards/margins": 0.04113560914993286, + "rewards/rejected": 1.6821091175079346, + "step": 830 + }, + { + "epoch": 0.7978024602890242, + "grad_norm": 1.296875, + "learning_rate": 8.14619513428405e-09, + "logits/chosen": -1.7318191528320312, + "logits/rejected": -1.74942946434021, + "logps/chosen": -0.25466471910476685, + "logps/rejected": -0.30034616589546204, + "loss": 0.6893445253372192, + "loss/core": 0.6893445253372192, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.404738426208496, + "rewards/margins": 1.5665807723999023, + "rewards/rejected": 0.8381577730178833, + "step": 835 + }, + { + "epoch": 0.8025797205302759, + "grad_norm": 4.0, + "learning_rate": 6.96422291980539e-09, + "logits/chosen": -1.8222589492797852, + "logits/rejected": -1.8721163272857666, + "logps/chosen": -0.32260316610336304, + "logps/rejected": -0.30097338557243347, + "loss": 0.6889389157295227, + "loss/core": 0.6889389157295227, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.8018436431884766, + "rewards/margins": 1.7603601217269897, + "rewards/rejected": 1.0414836406707764, + "step": 840 + }, + { + "epoch": 0.8073569807715275, + "grad_norm": 0.1318359375, + "learning_rate": 5.8736465216517594e-09, + "logits/chosen": -1.7940136194229126, + "logits/rejected": -1.8399118185043335, + "logps/chosen": -0.2931648790836334, + "logps/rejected": -0.3112708628177643, + "loss": 0.6907740235328674, + "loss/core": 0.6907740235328674, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.4224978685379028, + "rewards/margins": 0.967786431312561, + "rewards/rejected": 0.45471152663230896, + "step": 845 + }, + { + "epoch": 0.8121342410127792, + "grad_norm": 0.703125, + "learning_rate": 4.874876061005173e-09, + "logits/chosen": -1.9608793258666992, + "logits/rejected": -2.056640625, + "logps/chosen": -0.25170543789863586, + "logps/rejected": -0.24661597609519958, + "loss": 0.6915278434753418, + "loss/core": 0.6915278434753418, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.5610125064849854, + "rewards/margins": 0.6514654755592346, + "rewards/rejected": 0.9095470309257507, + "step": 850 + }, + { + "epoch": 0.8169115012540308, + "grad_norm": 0.44140625, + "learning_rate": 3.968287134589188e-09, + "logits/chosen": -1.767511010169983, + "logits/rejected": -1.7454007863998413, + "logps/chosen": -0.30029794573783875, + "logps/rejected": -0.29418742656707764, + "loss": 0.6896105408668518, + "loss/core": 0.6896105408668518, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.694100856781006, + "rewards/margins": 1.4545179605484009, + "rewards/rejected": 1.2395830154418945, + "step": 855 + }, + { + "epoch": 0.8216887614952825, + "grad_norm": 0.095703125, + "learning_rate": 3.154220673422192e-09, + "logits/chosen": -1.6382343769073486, + "logits/rejected": -1.68215811252594, + "logps/chosen": -0.29962679743766785, + "logps/rejected": -0.2946147322654724, + "loss": 0.6880518198013306, + "loss/core": 0.6880518198013306, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.085516452789307, + "rewards/margins": 2.11319637298584, + "rewards/rejected": 1.9723198413848877, + "step": 860 + }, + { + "epoch": 0.826466021736534, + "grad_norm": 0.275390625, + "learning_rate": 2.4329828146074096e-09, + "logits/chosen": -1.656494379043579, + "logits/rejected": -1.6885122060775757, + "logps/chosen": -0.2906996011734009, + "logps/rejected": -0.2946203649044037, + "loss": 0.6910411715507507, + "loss/core": 0.6910411715507507, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.4540257453918457, + "rewards/margins": 0.8756850957870483, + "rewards/rejected": 1.5783402919769287, + "step": 865 + }, + { + "epoch": 0.8312432819777857, + "grad_norm": 0.236328125, + "learning_rate": 1.8048447862070714e-09, + "logits/chosen": -1.8312631845474243, + "logits/rejected": -1.8569021224975586, + "logps/chosen": -0.3106675148010254, + "logps/rejected": -0.29439225792884827, + "loss": 0.6878628134727478, + "loss/core": 0.6878628134727478, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.189491033554077, + "rewards/margins": 2.2258591651916504, + "rewards/rejected": 0.9636317491531372, + "step": 870 + }, + { + "epoch": 0.8360205422190374, + "grad_norm": 0.1787109375, + "learning_rate": 1.2700428052447033e-09, + "logits/chosen": -1.7763378620147705, + "logits/rejected": -1.8419290781021118, + "logps/chosen": -0.3076077103614807, + "logps/rejected": -0.3310580551624298, + "loss": 0.6911420822143555, + "loss/core": 0.6911420822143555, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.865380883216858, + "rewards/margins": 0.844092071056366, + "rewards/rejected": 1.0212888717651367, + "step": 875 + }, + { + "epoch": 0.840797802460289, + "grad_norm": 1.4375, + "learning_rate": 8.287779888734858e-10, + "logits/chosen": -1.8257224559783936, + "logits/rejected": -1.7873103618621826, + "logps/chosen": -0.29086047410964966, + "logps/rejected": -0.327653706073761, + "loss": 0.6917628049850464, + "loss/core": 0.6917628049850464, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.455867290496826, + "rewards/margins": 0.6519712209701538, + "rewards/rejected": 1.8038959503173828, + "step": 880 + }, + { + "epoch": 0.8455750627015407, + "grad_norm": 0.291015625, + "learning_rate": 4.812162787445062e-10, + "logits/chosen": -1.7672336101531982, + "logits/rejected": -1.8428665399551392, + "logps/chosen": -0.28123897314071655, + "logps/rejected": -0.27698567509651184, + "loss": 0.689162015914917, + "loss/core": 0.689162015914917, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.7381834983825684, + "rewards/margins": 1.6145788431167603, + "rewards/rejected": 1.123604655265808, + "step": 885 + }, + { + "epoch": 0.8503523229427923, + "grad_norm": 0.97265625, + "learning_rate": 2.2748837860270265e-10, + "logits/chosen": -1.7094967365264893, + "logits/rejected": -1.819848656654358, + "logps/chosen": -0.29713571071624756, + "logps/rejected": -0.2740597426891327, + "loss": 0.6906223297119141, + "loss/core": 0.6906223297119141, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.4846818447113037, + "rewards/margins": 1.043480396270752, + "rewards/rejected": 1.4412018060684204, + "step": 890 + }, + { + "epoch": 0.855129583184044, + "grad_norm": 0.451171875, + "learning_rate": 6.768970513457151e-11, + "logits/chosen": -1.7548071146011353, + "logits/rejected": -1.7341846227645874, + "logps/chosen": -0.2722460627555847, + "logps/rejected": -0.2857206165790558, + "loss": 0.6913274526596069, + "loss/core": 0.6913274526596069, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.302839517593384, + "rewards/margins": 0.7413143515586853, + "rewards/rejected": 1.5615251064300537, + "step": 895 + }, + { + "epoch": 0.8599068434252956, + "grad_norm": 0.40625, + "learning_rate": 1.8803520859811406e-12, + "logits/chosen": -1.736576795578003, + "logits/rejected": -1.8592958450317383, + "logps/chosen": -0.26908183097839355, + "logps/rejected": -0.2669498324394226, + "loss": 0.6877947449684143, + "loss/core": 0.6877947449684143, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 3.115830659866333, + "rewards/margins": 2.253107786178589, + "rewards/rejected": 0.8627226948738098, + "step": 900 + }, + { + "epoch": 0.8599068434252956, + "step": 900, + "total_flos": 0.0, + "train_loss": 0.690198503865136, + "train_runtime": 8363.1914, + "train_samples_per_second": 1.722, + "train_steps_per_second": 0.108 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..bab5d56 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b25446b8e43170ab70d92ee5bed12a8e7e4c53550e0a5cb6de4123d57ca1c5a +size 6993