From e209ebafaed190bebb5a216954039f5bfe2e162f Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Thu, 23 Jul 2026 20:00:10 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: promotion/qwen3-8b-aaai27-flagship-inpo-avg-s42 Source: Original Platform --- .gitattributes | 36 + README.md | 24 + all_results.json | 9 + chat_template.jinja | 89 ++ config.json | 71 + config.yaml | 58 + generation_config.json | 12 + job_status.json | 23 + model.safetensors | 3 + objective_protocol.json | 69 + pair_manifest.json | 34 + provenance.json | 13 + resource_profile.json | 21 + run_status.json | 14 + split_manifest.json | 23 + stability_gate.json | 38 + tokenizer.json | 3 + tokenizer_config.json | 30 + train_results.json | 9 + trainer_state.json | 2923 +++++++++++++++++++++++++++++++++++++++ training_args.bin | 3 + 21 files changed, 3505 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 config.yaml create mode 100644 generation_config.json create mode 100644 job_status.json create mode 100644 model.safetensors create mode 100644 objective_protocol.json create mode 100644 pair_manifest.json create mode 100644 provenance.json create mode 100644 resource_profile.json create mode 100644 run_status.json create mode 100644 split_manifest.json create mode 100644 stability_gate.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..eb398f3 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +--- +library_name: transformers +base_model: Qwen/Qwen3-8B +tags: +- ronpo +- mnpo +- preference-optimization +--- + +# qwen3-8b-aaai27-flagship-inpo-avg-s42 + +Research checkpoint for the RONPO AAAI revision experiments. + +- Method: inpo_avg +- Base model: `Qwen/Qwen3-8B` +- Seed: 42 +- Local source at upload time: `/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/inpo_avg/seed42/attempt1` +- Uploaded at UTC: 2026-07-13T04:37:29Z +- Run status metadata: `{"attempt": 1, "effective_batch_size": 16, "method": "inpo_avg", "objective_protocol": "objective_protocol.json", "optimizer_steps": 900, "resource_profile": "resource_profile.json", "seed": 42, "split_manifest": "split_manifest.json", "stability_gate": "stability_gate.json", "status": "completed", "wandb_run_id": "ca7feea19984", "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/ca7feea19984"}` + +AAAI-27 P1 matched budget: 900 optimizer steps, effective batch 16; passed non-thinking and collapse stability gates; sealed-test metrics are not used for checkpoint selection. + +Intended use: reproducibility and evaluation for the RONPO paper. This +checkpoint is not intended as a production assistant. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..0bf5a7f --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 4428.528517795139, + "train_runtime": 7133.5294, + "train_samples": 16746, + "train_samples_per_second": 2.019, + "train_steps_per_second": 0.126 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..1a812ca --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +model_name_or_path: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218 +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/avg: 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: inpo +eta: 0.0075 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.0 +preference_sft_weight: 0.0 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 5.0e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 42 +gradient_accumulation_steps: 16 +gradient_checkpointing: true +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 1 +per_device_eval_batch_size: 1 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/inpo_avg/seed42/attempt1 +run_name: aaai27-flagship-full-inpo_avg-s42-a1 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/job_status.json b/job_status.json new file mode 100644 index 0000000..8378411 --- /dev/null +++ b/job_status.json @@ -0,0 +1,23 @@ +{ + "status": "completed", + "stage": "full", + "method": "inpo_avg", + "seed": 42, + "attempt": 1, + "gpu": 7, + "gpus": [ + 7 + ], + "returncode": 0, + "finite_metrics": true, + "oom": false, + "traceback": false, + "reason": "passed", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "ca7feea19984", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/ca7feea19984", + "output_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/inpo_avg/seed42/attempt1", + "log": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/logs/full_inpo_avg_s42_a1.log", + "completed_at": "2026-07-13T04:34:19Z" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..486dbf6 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3bc4d2dcb72fa5fb580a6c13667e56939be11e58b78b8782a1995272626f6a2 +size 16381517208 diff --git a/objective_protocol.json b/objective_protocol.json new file mode 100644 index 0000000..35b3b44 --- /dev/null +++ b/objective_protocol.json @@ -0,0 +1,69 @@ +{ + "schema_version": 1, + "frozen_at_kst": "2026-07-12T21:40:00+09:00", + "base_model": "Qwen/Qwen3-8B", + "primary_triple": { + "helpfulness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "ultrafeedback-helpfulness", + "transform": "identity" + }, + "safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "conciseness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "helpsteer-verbosity", + "transform": "negate" + } + }, + "gate_thresholds": { + "raw_score_std_strictly_greater_than": 0.01, + "aggregate_prompt_pair_spearman_median_strictly_less_than": 0.0, + "aggregate_pairwise_top1_mismatch_at_least": 0.5 + }, + "gate_pool": "source-train Qwen3-8B response pool split into deterministic train/validation; sealed source-test excluded", + "training_seeds": [42, 43, 44], + "optimizer_steps": 900, + "effective_batch_size": 16, + "decode": { + "enable_thinking": false, + "temperature": 0.7, + "top_p": 0.9, + "max_new_tokens": 2048, + "decode_seed": 42 + }, + "methods": [ + "ronpo_full_expect", + "ronpo_k_only", + "dpo", + "ipo", + "simpo", + "kto", + "sppo_avg", + "inpo_avg", + "ht_mnpo_helpfulness", + "ht_mnpo_safety", + "ht_mnpo_conciseness" + ], + "selection_policy": "No sealed-test selection. One preregistered configuration per method; failures use only the declared stability retry recipe, at most three attempts.", + "stability_gate": { + "validation_prompts": 128, + "min_candidate_base_mean_word_ratio": 0.33, + "max_candidate_base_mean_word_ratio": 2.0, + "max_consecutive_identical_word_run": 20, + "think_tag_count": 0, + "empty_response_count": 0, + "retry_recipe": "attempt 2 halves learning rate, enables gradient checkpointing, and adds reference=0.02/SFT=0.002 anchoring to unanchored unified losses; attempt 3 quarters learning rate and uses reference=0.05/SFT=0.005; RONPO uses reference=0.05/SFT=0.005 from attempt 1" + }, + "primary_metric": "mean prompt-level minimum of per-prompt min-max-normalized helpfulness, safety, conciseness", + "bootstrap": { + "unit": "prompt", + "paired": true, + "resamples": 2000, + "interval": "percentile_95", + "seed": 42 + } +} diff --git a/pair_manifest.json b/pair_manifest.json new file mode 100644 index 0000000..be034b0 --- /dev/null +++ b/pair_manifest.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "objectives": [ + "helpfulness", + "safety", + "conciseness" + ], + "normalization": "per_prompt_minmax", + "adversary": { + "steps": 25, + "alpha": 1.0, + "kappa": 0.05, + "preference_scale": 8.0 + }, + "ronpo_estimator": "Rao-Blackwellized full sigma expectation with three common response pairs per prompt", + "counts": { + "train": { + "avg": 16746, + "ronpo_full_expect": 50340, + "ronpo_k_only": 50340, + "ht_mnpo_helpfulness": 16743, + "ht_mnpo_safety": 16764, + "ht_mnpo_conciseness": 16755 + }, + "validation": { + "avg": 1859, + "ronpo_full_expect": 5592, + "ronpo_k_only": 5592, + "ht_mnpo_helpfulness": 1864, + "ht_mnpo_safety": 1862, + "ht_mnpo_conciseness": 1858 + } + } +} diff --git a/provenance.json b/provenance.json new file mode 100644 index 0000000..d02041c --- /dev/null +++ b/provenance.json @@ -0,0 +1,13 @@ +{ + "method": "inpo_avg", + "seed": 42, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "ca7feea19984", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/ca7feea19984", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json" +} diff --git a/resource_profile.json b/resource_profile.json new file mode 100644 index 0000000..18eec47 --- /dev/null +++ b/resource_profile.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "measured_at_kst": "2026-07-12T22:38:00+09:00", + "hardware": "NVIDIA B200 183359 MiB reported total; 178.35 GiB PyTorch-visible capacity", + "smoke_evidence": { + "attempted_profile": {"per_device_batch_size": 2, "gradient_accumulation_steps": 8, "gradient_checkpointing": false, "effective_batch_size": 16}, + "outcome": "OOM before completing the 20-step smoke across every launched unified baseline; representative process used 176.87 GiB and failed a 1.62 GiB allocation. KTO also OOMed at the optimizer step.", + "paper_metric": false + }, + "selected_full_training_profile": { + "per_device_batch_size": 2, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": false, + "attention_implementation": "PyTorch SDPA with cuDNN SDPA disabled", + "effective_batch_size": 16, + "applies_symmetrically_to_all_unified_pairwise_methods": true, + "kto_implementation_constraint": "TRL KTO requires actual batch size > 1. KTO uses batch 2 / accumulation 8 and identical AdamW with foreach=False. PyTorch SDPA is used with cuDNN SDPA disabled. The fixed base reference model is placed on a second B200 and its exact logits are returned to the policy device; the policy/optimizer remain on the first B200. This avoids a measured ~6-hour reference-precompute path while leaving the KTO loss, actual/effective batch, optimizer, and 900-step budget unchanged." + }, + "retry_semantics": "Resource selection is separated from method-stability attempts. Full attempt 1 uses the smoke-validated SDPA batch-2 profile and retains the preregistered method LR/anchor; attempts 2 and 3 fall back to batch 1 / accumulation 16 / gradient checkpointing and apply the declared LR/anchor stabilization recipe.", + "provenance_note": "No reward or sealed-evaluation result was available when this hardware profile was selected." +} diff --git a/run_status.json b/run_status.json new file mode 100644 index 0000000..6897942 --- /dev/null +++ b/run_status.json @@ -0,0 +1,14 @@ +{ + "method": "inpo_avg", + "seed": 42, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "ca7feea19984", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/ca7feea19984", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json", + "status": "completed" +} diff --git a/split_manifest.json b/split_manifest.json new file mode 100644 index 0000000..4fb96bd --- /dev/null +++ b/split_manifest.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "split_rule": "sort source-train prompts by sha256('flagship-v1||'+prompt_id); first round(10%) validation", + "validation_fraction": 0.1, + "source_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/avg_oracle", + "base_model": "Qwen/Qwen3-8B", + "counts": { + "train": 16780, + "validation": 1864, + "sealed_test": 604 + }, + "prompt_id_sha256": { + "train": "fcf6f2ea5a0b7bcd41179e6f5288f4d850b2d04bdb729da91907bd18bf733a69", + "validation": "a87ba6f519192164daa487b109a6547dd8cbde46a27a373d6da201ca2d061b8f", + "sealed_test": "b398ced6af6cf73d731e1a5f32a0631d8060323d51eda1e375695566bf5f1867" + }, + "overlaps": { + "train_validation": 0, + "train_sealed_test": 0, + "validation_sealed_test": 0 + }, + "sealed_test_policy": "prompts only until S4a; no objective scores or model generations used in S0-S3" +} diff --git a/stability_gate.json b/stability_gate.json new file mode 100644 index 0000000..f75c104 --- /dev/null +++ b/stability_gate.json @@ -0,0 +1,38 @@ +{ + "status": "passed", + "passed": true, + "base": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 388.1484375, + "max_words": 1379, + "max_repeat_run": 4 + }, + "candidate": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 407.7421875, + "max_words": 1386, + "max_repeat_run": 4 + }, + "candidate_base_mean_word_ratio": 1.0504800434756356, + "thresholds": { + "min_length_ratio": 0.33, + "max_length_ratio": 2.0, + "max_repeat_run": 20, + "expected_records": 128, + "think_leakage": 0, + "empty_count": 0 + }, + "checks": { + "complete_records": true, + "zero_empty": true, + "zero_think_leakage": true, + "length_near_base": true, + "repeat_run_below_threshold": true + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ffd19d2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..0bf5a7f --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 4428.528517795139, + "train_runtime": 7133.5294, + "train_samples": 16746, + "train_samples_per_second": 2.019, + "train_steps_per_second": 0.126 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..833672e --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,2923 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8599068434252956, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.004777260241251642, + "grad_norm": 2112.0, + "learning_rate": 2.222222222222222e-08, + "logits/chosen": -1.9987274408340454, + "logits/rejected": -2.014296054840088, + "logps/chosen": -0.27934736013412476, + "logps/rejected": -0.2821010947227478, + "loss": 4425.13330078125, + "loss/core": 4425.13330078125, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.4205944538116455, + "rewards/margins": 1.4615087509155273, + "rewards/rejected": 0.9590854644775391, + "step": 5 + }, + { + "epoch": 0.009554520482503284, + "grad_norm": 11584.0, + "learning_rate": 5e-08, + "logits/chosen": -2.3447604179382324, + "logits/rejected": -2.391680955886841, + "logps/chosen": -0.2895873188972473, + "logps/rejected": -0.28846535086631775, + "loss": 4443.54931640625, + "loss/core": 4443.54931640625, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.6877996921539307, + "rewards/margins": 0.0745888203382492, + "rewards/rejected": 1.613210916519165, + "step": 10 + }, + { + "epoch": 0.014331780723754926, + "grad_norm": 2416.0, + "learning_rate": 7.777777777777778e-08, + "logits/chosen": -2.229412317276001, + "logits/rejected": -2.1927170753479004, + "logps/chosen": -0.27719348669052124, + "logps/rejected": -0.2841333746910095, + "loss": 4431.30712890625, + "loss/core": 4431.30712890625, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.136369228363037, + "rewards/margins": 0.9970209002494812, + "rewards/rejected": 1.1393483877182007, + "step": 15 + }, + { + "epoch": 0.01910904096500657, + "grad_norm": 1632.0, + "learning_rate": 1.0555555555555555e-07, + "logits/chosen": -2.1783900260925293, + "logits/rejected": -2.219207286834717, + "logps/chosen": -0.30116215348243713, + "logps/rejected": -0.2862641215324402, + "loss": 4430.82421875, + "loss/core": 4430.82421875, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.917729377746582, + "rewards/margins": 1.0351120233535767, + "rewards/rejected": 0.8826172947883606, + "step": 20 + }, + { + "epoch": 0.02388630120625821, + "grad_norm": 1272.0, + "learning_rate": 1.3333333333333334e-07, + "logits/chosen": -2.075169086456299, + "logits/rejected": -2.1347451210021973, + "logps/chosen": -0.27128368616104126, + "logps/rejected": -0.25881752371788025, + "loss": 4416.3876953125, + "loss/core": 4416.3876953125, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.8423290252685547, + "rewards/margins": 2.2059836387634277, + "rewards/rejected": 1.6363451480865479, + "step": 25 + }, + { + "epoch": 0.028663561447509853, + "grad_norm": 3216.0, + "learning_rate": 1.611111111111111e-07, + "logits/chosen": -2.348489284515381, + "logits/rejected": -2.3322410583496094, + "logps/chosen": -0.30126723647117615, + "logps/rejected": -0.3248973786830902, + "loss": 4428.22412109375, + "loss/core": 4428.22412109375, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.2666220664978027, + "rewards/margins": 1.2462260723114014, + "rewards/rejected": 1.0203959941864014, + "step": 30 + }, + { + "epoch": 0.033440821688761495, + "grad_norm": 696.0, + "learning_rate": 1.8888888888888888e-07, + "logits/chosen": -2.2013611793518066, + "logits/rejected": -2.183184862136841, + "logps/chosen": -0.26950961351394653, + "logps/rejected": -0.2648638188838959, + "loss": 4425.138671875, + "loss/core": 4425.138671875, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.6406617164611816, + "rewards/margins": 1.477004885673523, + "rewards/rejected": 1.1636569499969482, + "step": 35 + }, + { + "epoch": 0.03821808193001314, + "grad_norm": 2256.0, + "learning_rate": 2.1666666666666667e-07, + "logits/chosen": -2.0045735836029053, + "logits/rejected": -2.0654070377349854, + "logps/chosen": -0.2931990921497345, + "logps/rejected": -0.280799925327301, + "loss": 4407.8447265625, + "loss/core": 4407.8447265625, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.472355842590332, + "rewards/margins": 2.8589112758636475, + "rewards/rejected": 1.6134445667266846, + "step": 40 + }, + { + "epoch": 0.04299534217126478, + "grad_norm": 1032.0, + "learning_rate": 2.4444444444444445e-07, + "logits/chosen": -2.20025372505188, + "logits/rejected": -2.3626952171325684, + "logps/chosen": -0.2890096604824066, + "logps/rejected": -0.28794199228286743, + "loss": 4427.9345703125, + "loss/core": 4427.9345703125, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.0934605598449707, + "rewards/margins": 1.246740460395813, + "rewards/rejected": 0.8467203378677368, + "step": 45 + }, + { + "epoch": 0.04777260241251642, + "grad_norm": 5600.0, + "learning_rate": 2.7222222222222216e-07, + "logits/chosen": -2.1670279502868652, + "logits/rejected": -2.163269519805908, + "logps/chosen": -0.2812585234642029, + "logps/rejected": -0.28595098853111267, + "loss": 4425.6552734375, + "loss/core": 4425.6552734375, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.883915662765503, + "rewards/margins": 1.4437261819839478, + "rewards/rejected": 1.4401893615722656, + "step": 50 + }, + { + "epoch": 0.05254986265376806, + "grad_norm": 440.0, + "learning_rate": 3e-07, + "logits/chosen": -2.197618007659912, + "logits/rejected": -2.1856415271759033, + "logps/chosen": -0.28887230157852173, + "logps/rejected": -0.28767216205596924, + "loss": 4442.45166015625, + "loss/core": 4442.45166015625, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.2596747875213623, + "rewards/margins": 0.2099606990814209, + "rewards/rejected": 2.0497140884399414, + "step": 55 + }, + { + "epoch": 0.057327122895019705, + "grad_norm": 18688.0, + "learning_rate": 3.2777777777777776e-07, + "logits/chosen": -2.091808319091797, + "logits/rejected": -2.120185375213623, + "logps/chosen": -0.27423757314682007, + "logps/rejected": -0.2837786376476288, + "loss": 4436.55517578125, + "loss/core": 4436.55517578125, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.158466100692749, + "rewards/margins": 0.602463960647583, + "rewards/rejected": 1.556002140045166, + "step": 60 + }, + { + "epoch": 0.06210438313627135, + "grad_norm": 888.0, + "learning_rate": 3.5555555555555553e-07, + "logits/chosen": -1.9588630199432373, + "logits/rejected": -2.0604052543640137, + "logps/chosen": -0.3092983365058899, + "logps/rejected": -0.30547577142715454, + "loss": 4420.9130859375, + "loss/core": 4420.9130859375, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.9688713550567627, + "rewards/margins": 1.7887201309204102, + "rewards/rejected": 1.180151104927063, + "step": 65 + }, + { + "epoch": 0.06688164337752299, + "grad_norm": 2208.0, + "learning_rate": 3.8333333333333335e-07, + "logits/chosen": -1.9690837860107422, + "logits/rejected": -1.95412278175354, + "logps/chosen": -0.3884851932525635, + "logps/rejected": -0.2990027964115143, + "loss": 4427.15771484375, + "loss/core": 4427.15771484375, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.5522398948669434, + "rewards/margins": 1.389047384262085, + "rewards/rejected": 2.1631925106048584, + "step": 70 + }, + { + "epoch": 0.07165890361877464, + "grad_norm": 676.0, + "learning_rate": 4.1111111111111107e-07, + "logits/chosen": -2.0886988639831543, + "logits/rejected": -2.0533549785614014, + "logps/chosen": -0.3141383230686188, + "logps/rejected": -0.28872591257095337, + "loss": 4431.21728515625, + "loss/core": 4431.21728515625, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.615492582321167, + "rewards/margins": 1.0181725025177002, + "rewards/rejected": 1.597320318222046, + "step": 75 + }, + { + "epoch": 0.07643616386002627, + "grad_norm": 760.0, + "learning_rate": 4.3888888888888884e-07, + "logits/chosen": -2.0682148933410645, + "logits/rejected": -2.1878905296325684, + "logps/chosen": -0.2604108154773712, + "logps/rejected": -0.2718687355518341, + "loss": 4418.80078125, + "loss/core": 4418.80078125, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.2428879737854004, + "rewards/margins": 1.9694182872772217, + "rewards/rejected": 1.2734694480895996, + "step": 80 + }, + { + "epoch": 0.08121342410127792, + "grad_norm": 1072.0, + "learning_rate": 4.6666666666666666e-07, + "logits/chosen": -2.1902594566345215, + "logits/rejected": -2.2378265857696533, + "logps/chosen": -0.3089032471179962, + "logps/rejected": -0.2776981294155121, + "loss": 4416.4970703125, + "loss/core": 4416.4970703125, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.4842758178710938, + "rewards/margins": 2.1533937454223633, + "rewards/rejected": 1.3308818340301514, + "step": 85 + }, + { + "epoch": 0.08599068434252956, + "grad_norm": 436.0, + "learning_rate": 4.944444444444445e-07, + "logits/chosen": -2.421081066131592, + "logits/rejected": -2.4086060523986816, + "logps/chosen": -0.2822219729423523, + "logps/rejected": -0.29552292823791504, + "loss": 4432.59619140625, + "loss/core": 4432.59619140625, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.7451679706573486, + "rewards/margins": 0.8951616287231445, + "rewards/rejected": 0.8500064015388489, + "step": 90 + }, + { + "epoch": 0.09076794458378121, + "grad_norm": 2064.0, + "learning_rate": 4.999699149323369e-07, + "logits/chosen": -2.0548970699310303, + "logits/rejected": -2.1117641925811768, + "logps/chosen": -0.26747360825538635, + "logps/rejected": -0.27913323044776917, + "loss": 4430.07275390625, + "loss/core": 4430.07275390625, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.2089908123016357, + "rewards/margins": 1.1066198348999023, + "rewards/rejected": 2.1023709774017334, + "step": 95 + }, + { + "epoch": 0.09554520482503284, + "grad_norm": 1104.0, + "learning_rate": 4.998477067547739e-07, + "logits/chosen": -2.234778881072998, + "logits/rejected": -2.280416250228882, + "logps/chosen": -0.25745129585266113, + "logps/rejected": -0.2470175325870514, + "loss": 4428.8994140625, + "loss/core": 4428.8994140625, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.4600722789764404, + "rewards/margins": 1.177268147468567, + "rewards/rejected": 1.282804250717163, + "step": 100 + }, + { + "epoch": 0.10032246506628449, + "grad_norm": 4160.0, + "learning_rate": 4.996315410727229e-07, + "logits/chosen": -2.339501142501831, + "logits/rejected": -2.4241232872009277, + "logps/chosen": -0.2842095196247101, + "logps/rejected": -0.27984419465065, + "loss": 4428.7177734375, + "loss/core": 4428.7177734375, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.997147798538208, + "rewards/margins": 1.1996146440505981, + "rewards/rejected": 0.7975330948829651, + "step": 105 + }, + { + "epoch": 0.10509972530753613, + "grad_norm": 944.0, + "learning_rate": 4.993214991772562e-07, + "logits/chosen": -1.907444953918457, + "logits/rejected": -1.858888030052185, + "logps/chosen": -0.3371734619140625, + "logps/rejected": -0.2982056736946106, + "loss": 4410.75390625, + "loss/core": 4410.75390625, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.483543395996094, + "rewards/margins": 2.5996921062469482, + "rewards/rejected": 1.8838508129119873, + "step": 110 + }, + { + "epoch": 0.10987698554878778, + "grad_norm": 5120.0, + "learning_rate": 4.989176976624511e-07, + "logits/chosen": -2.087294816970825, + "logits/rejected": -2.0307278633117676, + "logps/chosen": -0.2998853623867035, + "logps/rejected": -0.2985258996486664, + "loss": 4440.2822265625, + "loss/core": 4440.2822265625, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.7805169820785522, + "rewards/margins": 0.33072328567504883, + "rewards/rejected": 1.4497936964035034, + "step": 115 + }, + { + "epoch": 0.11465424579003941, + "grad_norm": 8576.0, + "learning_rate": 4.984202883815428e-07, + "logits/chosen": -2.0372519493103027, + "logits/rejected": -2.075047492980957, + "logps/chosen": -0.30229058861732483, + "logps/rejected": -0.2958064079284668, + "loss": 4435.390625, + "loss/core": 4435.390625, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.808635950088501, + "rewards/margins": 0.6832652688026428, + "rewards/rejected": 1.1253705024719238, + "step": 120 + }, + { + "epoch": 0.11943150603129106, + "grad_norm": 712.0, + "learning_rate": 4.978294583898195e-07, + "logits/chosen": -2.1084086894989014, + "logits/rejected": -2.138524055480957, + "logps/chosen": -0.29876020550727844, + "logps/rejected": -0.306560218334198, + "loss": 4430.5625, + "loss/core": 4430.5625, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.067880153656006, + "rewards/margins": 1.0501195192337036, + "rewards/rejected": 1.0177606344223022, + "step": 125 + }, + { + "epoch": 0.1242087662725427, + "grad_norm": 1504.0, + "learning_rate": 4.971454298742779e-07, + "logits/chosen": -2.001467227935791, + "logits/rejected": -2.094036817550659, + "logps/chosen": -0.2802198827266693, + "logps/rejected": -0.27933305501937866, + "loss": 4432.16796875, + "loss/core": 4432.16796875, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.805000901222229, + "rewards/margins": 0.9235280752182007, + "rewards/rejected": 0.8814727067947388, + "step": 130 + }, + { + "epoch": 0.12898602651379434, + "grad_norm": 2192.0, + "learning_rate": 4.963684600700678e-07, + "logits/chosen": -2.0947656631469727, + "logits/rejected": -2.167937994003296, + "logps/chosen": -0.3018343448638916, + "logps/rejected": -0.29743117094039917, + "loss": 4417.396484375, + "loss/core": 4417.396484375, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.3767879009246826, + "rewards/margins": 2.0548617839813232, + "rewards/rejected": 1.3219258785247803, + "step": 135 + }, + { + "epoch": 0.13376328675504598, + "grad_norm": 860.0, + "learning_rate": 4.954988411637571e-07, + "logits/chosen": -2.454082489013672, + "logits/rejected": -2.513239622116089, + "logps/chosen": -0.282596230506897, + "logps/rejected": -0.2890459895133972, + "loss": 4432.37744140625, + "loss/core": 4432.37744140625, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6272993087768555, + "rewards/margins": 0.908282458782196, + "rewards/rejected": 0.7190167307853699, + "step": 140 + }, + { + "epoch": 0.13854054699629761, + "grad_norm": 884.0, + "learning_rate": 4.945369001834514e-07, + "logits/chosen": -1.781502366065979, + "logits/rejected": -1.8124113082885742, + "logps/chosen": -0.29897254705429077, + "logps/rejected": -0.3044045567512512, + "loss": 4420.0517578125, + "loss/core": 4420.0517578125, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.627842664718628, + "rewards/margins": 1.8691842555999756, + "rewards/rejected": 1.7586586475372314, + "step": 145 + }, + { + "epoch": 0.14331780723754928, + "grad_norm": 2224.0, + "learning_rate": 4.93482998875813e-07, + "logits/chosen": -2.2341880798339844, + "logits/rejected": -2.2840676307678223, + "logps/chosen": -0.3164155185222626, + "logps/rejected": -0.3073708713054657, + "loss": 4415.888671875, + "loss/core": 4415.888671875, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.64851713180542, + "rewards/margins": 2.182638168334961, + "rewards/rejected": 1.4658793210983276, + "step": 150 + }, + { + "epoch": 0.1480950674788009, + "grad_norm": 1064.0, + "learning_rate": 4.923375335700223e-07, + "logits/chosen": -2.100919008255005, + "logits/rejected": -2.178107738494873, + "logps/chosen": -0.28356048464775085, + "logps/rejected": -0.2917582392692566, + "loss": 4417.8515625, + "loss/core": 4417.8515625, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.8598971366882324, + "rewards/margins": 2.027747392654419, + "rewards/rejected": 0.8321496248245239, + "step": 155 + }, + { + "epoch": 0.15287232772005255, + "grad_norm": 5024.0, + "learning_rate": 4.911009350287347e-07, + "logits/chosen": -2.0568435192108154, + "logits/rejected": -2.0316061973571777, + "logps/chosen": -0.31080225110054016, + "logps/rejected": -0.3151562511920929, + "loss": 4421.96240234375, + "loss/core": 4421.96240234375, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.147416591644287, + "rewards/margins": 1.7445529699325562, + "rewards/rejected": 1.4028639793395996, + "step": 160 + }, + { + "epoch": 0.15764958796130418, + "grad_norm": 1232.0, + "learning_rate": 4.897736682860885e-07, + "logits/chosen": -2.222330331802368, + "logits/rejected": -2.154171943664551, + "logps/chosen": -0.3106082081794739, + "logps/rejected": -0.3049444556236267, + "loss": 4433.68994140625, + "loss/core": 4433.68994140625, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6072232723236084, + "rewards/margins": 0.810656726360321, + "rewards/rejected": 0.7965666055679321, + "step": 165 + }, + { + "epoch": 0.16242684820255585, + "grad_norm": 5184.0, + "learning_rate": 4.883562324728241e-07, + "logits/chosen": -2.189422130584717, + "logits/rejected": -2.1332600116729736, + "logps/chosen": -0.2748359739780426, + "logps/rejected": -0.2597963213920593, + "loss": 4447.82421875, + "loss/core": 4447.82421875, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.5958924293518066, + "rewards/margins": -0.17370417714118958, + "rewards/rejected": 2.769596576690674, + "step": 170 + }, + { + "epoch": 0.16720410844380748, + "grad_norm": 7424.0, + "learning_rate": 4.868491606285823e-07, + "logits/chosen": -2.1792263984680176, + "logits/rejected": -2.2180469036102295, + "logps/chosen": -0.28667014837265015, + "logps/rejected": -0.30290576815605164, + "loss": 4426.7734375, + "loss/core": 4426.7734375, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.5856590270996094, + "rewards/margins": 1.3379803895950317, + "rewards/rejected": 1.2476783990859985, + "step": 175 + }, + { + "epoch": 0.17198136868505912, + "grad_norm": 8320.0, + "learning_rate": 4.852530195014489e-07, + "logits/chosen": -2.2014615535736084, + "logits/rejected": -2.1121859550476074, + "logps/chosen": -0.27293267846107483, + "logps/rejected": -0.2612500488758087, + "loss": 4422.0908203125, + "loss/core": 4422.0908203125, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.2250092029571533, + "rewards/margins": 1.7280479669570923, + "rewards/rejected": 1.4969611167907715, + "step": 180 + }, + { + "epoch": 0.17675862892631075, + "grad_norm": 844.0, + "learning_rate": 4.835684093348244e-07, + "logits/chosen": -2.20589017868042, + "logits/rejected": -2.162938117980957, + "logps/chosen": -0.288524866104126, + "logps/rejected": -0.2919921875, + "loss": 4438.9091796875, + "loss/core": 4438.9091796875, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.0444185733795166, + "rewards/margins": 0.4165409207344055, + "rewards/rejected": 0.6278777122497559, + "step": 185 + }, + { + "epoch": 0.18153588916756241, + "grad_norm": 1832.0, + "learning_rate": 4.817959636416969e-07, + "logits/chosen": -2.3000447750091553, + "logits/rejected": -2.2772605419158936, + "logps/chosen": -0.2829676866531372, + "logps/rejected": -0.28464275598526, + "loss": 4423.5048828125, + "loss/core": 4423.5048828125, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.844500780105591, + "rewards/margins": 1.6092599630355835, + "rewards/rejected": 1.2352410554885864, + "step": 190 + }, + { + "epoch": 0.18631314940881405, + "grad_norm": 1216.0, + "learning_rate": 4.799363489664039e-07, + "logits/chosen": -1.9864647388458252, + "logits/rejected": -2.007460594177246, + "logps/chosen": -0.2905668616294861, + "logps/rejected": -0.2901652455329895, + "loss": 4422.435546875, + "loss/core": 4422.435546875, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.672790288925171, + "rewards/margins": 1.7109134197235107, + "rewards/rejected": 0.9618767499923706, + "step": 195 + }, + { + "epoch": 0.19109040965006568, + "grad_norm": 1080.0, + "learning_rate": 4.779902646339721e-07, + "logits/chosen": -1.8949693441390991, + "logits/rejected": -1.9352257251739502, + "logps/chosen": -0.3231773376464844, + "logps/rejected": -0.31668004393577576, + "loss": 4425.9697265625, + "loss/core": 4425.9697265625, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.618408441543579, + "rewards/margins": 1.3984508514404297, + "rewards/rejected": 1.2199573516845703, + "step": 200 + }, + { + "epoch": 0.19586766989131732, + "grad_norm": 752.0, + "learning_rate": 4.759584424871301e-07, + "logits/chosen": -2.1187617778778076, + "logits/rejected": -2.1708757877349854, + "logps/chosen": -0.2496471405029297, + "logps/rejected": -0.26736050844192505, + "loss": 4429.56640625, + "loss/core": 4429.56640625, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.2940521240234375, + "rewards/margins": 1.1699793338775635, + "rewards/rejected": 2.124073028564453, + "step": 205 + }, + { + "epoch": 0.20064493013256898, + "grad_norm": 1952.0, + "learning_rate": 4.738416466110917e-07, + "logits/chosen": -2.1127567291259766, + "logits/rejected": -2.137141227722168, + "logps/chosen": -0.31239503622055054, + "logps/rejected": -0.322940468788147, + "loss": 4426.55859375, + "loss/core": 4426.55859375, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.8321475982666016, + "rewards/margins": 1.3591885566711426, + "rewards/rejected": 1.4729589223861694, + "step": 210 + }, + { + "epoch": 0.20542219037382062, + "grad_norm": 1848.0, + "learning_rate": 4.716406730462153e-07, + "logits/chosen": -2.0887973308563232, + "logits/rejected": -2.079000949859619, + "logps/chosen": -0.27418142557144165, + "logps/rejected": -0.2853352427482605, + "loss": 4425.31494140625, + "loss/core": 4425.31494140625, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.2043747901916504, + "rewards/margins": 1.5126924514770508, + "rewards/rejected": 1.6916821002960205, + "step": 215 + }, + { + "epoch": 0.21019945061507225, + "grad_norm": 1888.0, + "learning_rate": 4.693563494886454e-07, + "logits/chosen": -2.104156017303467, + "logits/rejected": -2.01837420463562, + "logps/chosen": -0.3019450902938843, + "logps/rejected": -0.3014443516731262, + "loss": 4433.5771484375, + "loss/core": 4433.5771484375, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.8912023305892944, + "rewards/margins": 0.8183046579360962, + "rewards/rejected": 1.0728976726531982, + "step": 220 + }, + { + "epoch": 0.2149767108563239, + "grad_norm": 1040.0, + "learning_rate": 4.6698953497905016e-07, + "logits/chosen": -2.190114974975586, + "logits/rejected": -2.1780142784118652, + "logps/chosen": -0.2899002134799957, + "logps/rejected": -0.2752569317817688, + "loss": 4415.31396484375, + "loss/core": 4415.31396484375, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.6290431022644043, + "rewards/margins": 2.217606782913208, + "rewards/rejected": 1.4114364385604858, + "step": 225 + }, + { + "epoch": 0.21975397109757555, + "grad_norm": 10112.0, + "learning_rate": 4.645411195795709e-07, + "logits/chosen": -2.2766060829162598, + "logits/rejected": -2.2850639820098877, + "logps/chosen": -0.28285735845565796, + "logps/rejected": -0.3064045310020447, + "loss": 4430.21240234375, + "loss/core": 4430.21240234375, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8430675268173218, + "rewards/margins": 1.0752555131912231, + "rewards/rejected": 0.7678121328353882, + "step": 230 + }, + { + "epoch": 0.2245312313388272, + "grad_norm": 1328.0, + "learning_rate": 4.6201202403910643e-07, + "logits/chosen": -2.0158145427703857, + "logits/rejected": -2.0498404502868652, + "logps/chosen": -0.2925497591495514, + "logps/rejected": -0.28553271293640137, + "loss": 4437.0009765625, + "loss/core": 4437.0009765625, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.9056549072265625, + "rewards/margins": 0.5666888952255249, + "rewards/rejected": 1.3389660120010376, + "step": 235 + }, + { + "epoch": 0.22930849158007882, + "grad_norm": 1200.0, + "learning_rate": 4.594031994470573e-07, + "logits/chosen": -2.028263568878174, + "logits/rejected": -2.0478172302246094, + "logps/chosen": -0.29387903213500977, + "logps/rejected": -0.2936249375343323, + "loss": 4434.69873046875, + "loss/core": 4434.69873046875, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7645080089569092, + "rewards/margins": 0.732918381690979, + "rewards/rejected": 1.0315898656845093, + "step": 240 + }, + { + "epoch": 0.23408575182133046, + "grad_norm": 10048.0, + "learning_rate": 4.567156268756593e-07, + "logits/chosen": -2.0705394744873047, + "logits/rejected": -2.0584795475006104, + "logps/chosen": -0.2776476740837097, + "logps/rejected": -0.2890920341014862, + "loss": 4413.03466796875, + "loss/core": 4413.03466796875, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.6031315326690674, + "rewards/margins": 2.431091547012329, + "rewards/rejected": 1.1720398664474487, + "step": 245 + }, + { + "epoch": 0.23886301206258212, + "grad_norm": 1512.0, + "learning_rate": 4.5395031701104303e-07, + "logits/chosen": -1.9100065231323242, + "logits/rejected": -1.989465355873108, + "logps/chosen": -0.29988300800323486, + "logps/rejected": -0.2941280007362366, + "loss": 4427.71923828125, + "loss/core": 4427.71923828125, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.3385276794433594, + "rewards/margins": 1.2655916213989258, + "rewards/rejected": 1.0729362964630127, + "step": 250 + }, + { + "epoch": 0.24364027230383375, + "grad_norm": 916.0, + "learning_rate": 4.511083097731555e-07, + "logits/chosen": -2.2194790840148926, + "logits/rejected": -2.2149035930633545, + "logps/chosen": -0.2853752374649048, + "logps/rejected": -0.2876095473766327, + "loss": 4433.2822265625, + "loss/core": 4433.2822265625, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.4470086097717285, + "rewards/margins": 0.8454030156135559, + "rewards/rejected": 1.6016056537628174, + "step": 255 + }, + { + "epoch": 0.2484175325450854, + "grad_norm": 458.0, + "learning_rate": 4.481906739246894e-07, + "logits/chosen": -2.1570799350738525, + "logits/rejected": -2.2298617362976074, + "logps/chosen": -0.28772681951522827, + "logps/rejected": -0.26979178190231323, + "loss": 4433.95703125, + "loss/core": 4433.95703125, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.3997950553894043, + "rewards/margins": 0.8103548288345337, + "rewards/rejected": 1.5894405841827393, + "step": 260 + }, + { + "epoch": 0.25319479278633705, + "grad_norm": 448.0, + "learning_rate": 4.451985066691648e-07, + "logits/chosen": -2.0369317531585693, + "logits/rejected": -2.0460586547851562, + "logps/chosen": -0.29983657598495483, + "logps/rejected": -0.3035128712654114, + "loss": 4443.8427734375, + "loss/core": 4443.8427734375, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.2268575429916382, + "rewards/margins": 0.0723171979188919, + "rewards/rejected": 1.1545404195785522, + "step": 265 + }, + { + "epoch": 0.2579720530275887, + "grad_norm": 868.0, + "learning_rate": 4.421329332383158e-07, + "logits/chosen": -1.9414504766464233, + "logits/rejected": -2.0309865474700928, + "logps/chosen": -0.28761979937553406, + "logps/rejected": -0.28891196846961975, + "loss": 4429.82275390625, + "loss/core": 4429.82275390625, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.290050506591797, + "rewards/margins": 1.1456321477890015, + "rewards/rejected": 2.144418716430664, + "step": 270 + }, + { + "epoch": 0.2627493132688403, + "grad_norm": 8704.0, + "learning_rate": 4.3899510646893696e-07, + "logits/chosen": -2.0923826694488525, + "logits/rejected": -2.0464677810668945, + "logps/chosen": -0.29613757133483887, + "logps/rejected": -0.30172640085220337, + "loss": 4428.80517578125, + "loss/core": 4428.80517578125, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.328432083129883, + "rewards/margins": 1.1818605661392212, + "rewards/rejected": 1.146571397781372, + "step": 275 + }, + { + "epoch": 0.26752657351009196, + "grad_norm": 760.0, + "learning_rate": 4.357862063693485e-07, + "logits/chosen": -2.060886859893799, + "logits/rejected": -1.9781144857406616, + "logps/chosen": -0.2877367436885834, + "logps/rejected": -0.30556520819664, + "loss": 4427.63134765625, + "loss/core": 4427.63134765625, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.049919605255127, + "rewards/margins": 1.2704269886016846, + "rewards/rejected": 0.7794923782348633, + "step": 280 + }, + { + "epoch": 0.2723038337513436, + "grad_norm": 4608.0, + "learning_rate": 4.3250743967564364e-07, + "logits/chosen": -2.0870399475097656, + "logits/rejected": -2.082958221435547, + "logps/chosen": -0.29648491740226746, + "logps/rejected": -0.2911376357078552, + "loss": 4431.78466796875, + "loss/core": 4431.78466796875, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.612039089202881, + "rewards/margins": 0.9668930768966675, + "rewards/rejected": 1.6451460123062134, + "step": 285 + }, + { + "epoch": 0.27708109399259523, + "grad_norm": 10176.0, + "learning_rate": 4.29160039397884e-07, + "logits/chosen": -2.1645455360412598, + "logits/rejected": -2.1757516860961914, + "logps/chosen": -0.2864384949207306, + "logps/rejected": -0.28760045766830444, + "loss": 4444.3701171875, + "loss/core": 4444.3701171875, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.4827371835708618, + "rewards/margins": 0.028162401169538498, + "rewards/rejected": 1.4545748233795166, + "step": 290 + }, + { + "epoch": 0.28185835423384686, + "grad_norm": 924.0, + "learning_rate": 4.2574526435641546e-07, + "logits/chosen": -2.5665838718414307, + "logits/rejected": -2.5299837589263916, + "logps/chosen": -0.24450433254241943, + "logps/rejected": -0.2568889260292053, + "loss": 4435.9375, + "loss/core": 4435.9375, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.310236930847168, + "rewards/margins": 0.6390492916107178, + "rewards/rejected": 0.6711876392364502, + "step": 295 + }, + { + "epoch": 0.28663561447509855, + "grad_norm": 12928.0, + "learning_rate": 4.22264398708477e-07, + "logits/chosen": -1.9790928363800049, + "logits/rejected": -1.9473092555999756, + "logps/chosen": -0.3141786456108093, + "logps/rejected": -0.3128449320793152, + "loss": 4410.31787109375, + "loss/core": 4410.31787109375, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.7060604095458984, + "rewards/margins": 2.605220317840576, + "rewards/rejected": 1.1008400917053223, + "step": 300 + }, + { + "epoch": 0.2914128747163502, + "grad_norm": 1080.0, + "learning_rate": 4.187187514652819e-07, + "logits/chosen": -2.1226131916046143, + "logits/rejected": -2.1846141815185547, + "logps/chosen": -0.2690519690513611, + "logps/rejected": -0.26041924953460693, + "loss": 4427.720703125, + "loss/core": 4427.720703125, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.1630797386169434, + "rewards/margins": 1.2623918056488037, + "rewards/rejected": 0.9006881713867188, + "step": 305 + }, + { + "epoch": 0.2961901349576018, + "grad_norm": 13568.0, + "learning_rate": 4.151096559997519e-07, + "logits/chosen": -2.1288938522338867, + "logits/rejected": -2.0386104583740234, + "logps/chosen": -0.27233630418777466, + "logps/rejected": -0.279468834400177, + "loss": 4437.134765625, + "loss/core": 4437.134765625, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.6192855834960938, + "rewards/margins": 0.5920935869216919, + "rewards/rejected": 2.027191638946533, + "step": 310 + }, + { + "epoch": 0.30096739519885346, + "grad_norm": 840.0, + "learning_rate": 4.114384695450905e-07, + "logits/chosen": -1.9616706371307373, + "logits/rejected": -2.0680670738220215, + "logps/chosen": -0.2692381739616394, + "logps/rejected": -0.27612730860710144, + "loss": 4423.5947265625, + "loss/core": 4423.5947265625, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.912264347076416, + "rewards/margins": 1.5709517002105713, + "rewards/rejected": 1.3413128852844238, + "step": 315 + }, + { + "epoch": 0.3057446554401051, + "grad_norm": 708.0, + "learning_rate": 4.077065726843828e-07, + "logits/chosen": -2.000558614730835, + "logits/rejected": -2.0157294273376465, + "logps/chosen": -0.2892259359359741, + "logps/rejected": -0.29263320565223694, + "loss": 4439.705078125, + "loss/core": 4439.705078125, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.5899919271469116, + "rewards/margins": 0.36150193214416504, + "rewards/rejected": 1.2284899950027466, + "step": 320 + }, + { + "epoch": 0.31052191568135673, + "grad_norm": 7104.0, + "learning_rate": 4.039153688314145e-07, + "logits/chosen": -2.257622718811035, + "logits/rejected": -2.2041261196136475, + "logps/chosen": -0.2870021462440491, + "logps/rejected": -0.2847207188606262, + "loss": 4416.87060546875, + "loss/core": 4416.87060546875, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.6918609142303467, + "rewards/margins": 2.167919635772705, + "rewards/rejected": 1.5239416360855103, + "step": 325 + }, + { + "epoch": 0.31529917592260837, + "grad_norm": 568.0, + "learning_rate": 4.0006628370290613e-07, + "logits/chosen": -2.2275662422180176, + "logits/rejected": -2.2161643505096436, + "logps/chosen": -0.2746240496635437, + "logps/rejected": -0.271222323179245, + "loss": 4431.955078125, + "loss/core": 4431.955078125, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.07051682472229, + "rewards/margins": 0.9463024139404297, + "rewards/rejected": 1.12421452999115, + "step": 330 + }, + { + "epoch": 0.32007643616386, + "grad_norm": 17920.0, + "learning_rate": 3.9616076478235826e-07, + "logits/chosen": -2.0493054389953613, + "logits/rejected": -2.0343852043151855, + "logps/chosen": -0.3021050691604614, + "logps/rejected": -0.32243406772613525, + "loss": 4425.26953125, + "loss/core": 4425.26953125, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.1218528747558594, + "rewards/margins": 1.4721834659576416, + "rewards/rejected": 0.6496694684028625, + "step": 335 + }, + { + "epoch": 0.3248536964051117, + "grad_norm": 5632.0, + "learning_rate": 3.922002807757129e-07, + "logits/chosen": -2.0967724323272705, + "logits/rejected": -2.2191970348358154, + "logps/chosen": -0.2803352475166321, + "logps/rejected": -0.2822166979312897, + "loss": 4428.95849609375, + "loss/core": 4428.95849609375, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.037506580352783, + "rewards/margins": 1.1726973056793213, + "rewards/rejected": 0.8648093342781067, + "step": 340 + }, + { + "epoch": 0.3296309566463633, + "grad_norm": 1392.0, + "learning_rate": 3.8818632105903315e-07, + "logits/chosen": -2.3210458755493164, + "logits/rejected": -2.3820576667785645, + "logps/chosen": -0.2815735936164856, + "logps/rejected": -0.2927202582359314, + "loss": 4433.78076171875, + "loss/core": 4433.78076171875, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.6497310400009155, + "rewards/margins": 0.8072711825370789, + "rewards/rejected": 0.8424596786499023, + "step": 345 + }, + { + "epoch": 0.33440821688761496, + "grad_norm": 1080.0, + "learning_rate": 3.841203951184094e-07, + "logits/chosen": -2.062190532684326, + "logits/rejected": -2.041247844696045, + "logps/chosen": -0.2606438100337982, + "logps/rejected": -0.2558349072933197, + "loss": 4430.6279296875, + "loss/core": 4430.6279296875, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.995518445968628, + "rewards/margins": 1.0421174764633179, + "rewards/rejected": 0.9534010887145996, + "step": 350 + }, + { + "epoch": 0.3391854771288666, + "grad_norm": 1096.0, + "learning_rate": 3.800040319823038e-07, + "logits/chosen": -2.1064131259918213, + "logits/rejected": -2.233365535736084, + "logps/chosen": -0.2812296450138092, + "logps/rejected": -0.29490309953689575, + "loss": 4437.8212890625, + "loss/core": 4437.8212890625, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8455944061279297, + "rewards/margins": 0.5175111889839172, + "rewards/rejected": 1.3280833959579468, + "step": 355 + }, + { + "epoch": 0.34396273737011823, + "grad_norm": 7392.0, + "learning_rate": 3.75838779646545e-07, + "logits/chosen": -2.132842540740967, + "logits/rejected": -2.196658134460449, + "logps/chosen": -0.2782808542251587, + "logps/rejected": -0.28051522374153137, + "loss": 4426.58642578125, + "loss/core": 4426.58642578125, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.652100086212158, + "rewards/margins": 1.363356351852417, + "rewards/rejected": 1.2887436151504517, + "step": 360 + }, + { + "epoch": 0.34873999761136987, + "grad_norm": 6272.0, + "learning_rate": 3.7162620449218993e-07, + "logits/chosen": -2.0724411010742188, + "logits/rejected": -2.1809439659118652, + "logps/chosen": -0.2859554886817932, + "logps/rejected": -0.28384047746658325, + "loss": 4417.919921875, + "loss/core": 4417.919921875, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.864577054977417, + "rewards/margins": 2.0546889305114746, + "rewards/rejected": 0.8098880648612976, + "step": 365 + }, + { + "epoch": 0.3535172578526215, + "grad_norm": 2048.0, + "learning_rate": 3.673678906964727e-07, + "logits/chosen": -2.2656424045562744, + "logits/rejected": -2.311187744140625, + "logps/chosen": -0.28399044275283813, + "logps/rejected": -0.28443023562431335, + "loss": 4436.71484375, + "loss/core": 4436.71484375, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.6527982950210571, + "rewards/margins": 0.5823618173599243, + "rewards/rejected": 1.0704364776611328, + "step": 370 + }, + { + "epoch": 0.35829451809387314, + "grad_norm": 1048.0, + "learning_rate": 3.6306543963705936e-07, + "logits/chosen": -2.04667067527771, + "logits/rejected": -2.024685859680176, + "logps/chosen": -0.2581968307495117, + "logps/rejected": -0.2569906711578369, + "loss": 4434.41162109375, + "loss/core": 4434.41162109375, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.0193209648132324, + "rewards/margins": 0.787818968296051, + "rewards/rejected": 2.231502056121826, + "step": 375 + }, + { + "epoch": 0.36307177833512483, + "grad_norm": 1440.0, + "learning_rate": 3.5872046928983623e-07, + "logits/chosen": -2.1206488609313965, + "logits/rejected": -2.1304726600646973, + "logps/chosen": -0.2783791422843933, + "logps/rejected": -0.2880426049232483, + "loss": 4438.98291015625, + "loss/core": 4438.98291015625, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.067422866821289, + "rewards/margins": 0.4526546597480774, + "rewards/rejected": 1.6147682666778564, + "step": 380 + }, + { + "epoch": 0.36784903857637646, + "grad_norm": 1912.0, + "learning_rate": 3.5433461362045447e-07, + "logits/chosen": -2.113156318664551, + "logits/rejected": -2.1326403617858887, + "logps/chosen": -0.2700130343437195, + "logps/rejected": -0.27478334307670593, + "loss": 4435.85009765625, + "loss/core": 4435.85009765625, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.1801278591156006, + "rewards/margins": 0.6569812893867493, + "rewards/rejected": 1.523146390914917, + "step": 385 + }, + { + "epoch": 0.3726262988176281, + "grad_norm": 584.0, + "learning_rate": 3.4990952196986305e-07, + "logits/chosen": -1.8711488246917725, + "logits/rejected": -1.948341727256775, + "logps/chosen": -0.3067759573459625, + "logps/rejected": -0.3026246428489685, + "loss": 4432.92138671875, + "loss/core": 4432.92138671875, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.7659752368927002, + "rewards/margins": 0.8714345693588257, + "rewards/rejected": 0.8945406079292297, + "step": 390 + }, + { + "epoch": 0.37740355905887973, + "grad_norm": 1744.0, + "learning_rate": 3.4544685843405875e-07, + "logits/chosen": -2.2217893600463867, + "logits/rejected": -2.32525897026062, + "logps/chosen": -0.29468852281570435, + "logps/rejected": -0.2896806001663208, + "loss": 4432.77392578125, + "loss/core": 4432.77392578125, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 1.685217261314392, + "rewards/margins": 0.8789496421813965, + "rewards/rejected": 0.8062676191329956, + "step": 395 + }, + { + "epoch": 0.38218081930013137, + "grad_norm": 1272.0, + "learning_rate": 3.4094830123828786e-07, + "logits/chosen": -2.1023941040039062, + "logits/rejected": -2.1372156143188477, + "logps/chosen": -0.29075393080711365, + "logps/rejected": -0.29359036684036255, + "loss": 4417.08642578125, + "loss/core": 4417.08642578125, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.9458072185516357, + "rewards/margins": 2.109001636505127, + "rewards/rejected": 0.836805522441864, + "step": 400 + }, + { + "epoch": 0.386958079541383, + "grad_norm": 940.0, + "learning_rate": 3.3641554210593414e-07, + "logits/chosen": -2.132859706878662, + "logits/rejected": -2.1874256134033203, + "logps/chosen": -0.31743866205215454, + "logps/rejected": -0.3030361235141754, + "loss": 4430.7001953125, + "loss/core": 4430.7001953125, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.2906603813171387, + "rewards/margins": 1.0478847026824951, + "rewards/rejected": 1.242775797843933, + "step": 405 + }, + { + "epoch": 0.39173533978263464, + "grad_norm": 1176.0, + "learning_rate": 3.3185028562233107e-07, + "logits/chosen": -2.2498955726623535, + "logits/rejected": -2.280546188354492, + "logps/chosen": -0.2766816020011902, + "logps/rejected": -0.28772997856140137, + "loss": 4432.49267578125, + "loss/core": 4432.49267578125, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.0912842750549316, + "rewards/margins": 0.8995159268379211, + "rewards/rejected": 1.1917682886123657, + "step": 410 + }, + { + "epoch": 0.3965126000238863, + "grad_norm": 1160.0, + "learning_rate": 3.272542485937368e-07, + "logits/chosen": -2.11924409866333, + "logits/rejected": -2.1979384422302246, + "logps/chosen": -0.3298892378807068, + "logps/rejected": -0.32105594873428345, + "loss": 4436.1533203125, + "loss/core": 4436.1533203125, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 0.9991945028305054, + "rewards/margins": 0.6242727041244507, + "rewards/rejected": 0.37492185831069946, + "step": 415 + }, + { + "epoch": 0.40128986026513797, + "grad_norm": 3312.0, + "learning_rate": 3.226291594017137e-07, + "logits/chosen": -2.350698947906494, + "logits/rejected": -2.317554473876953, + "logps/chosen": -0.283860981464386, + "logps/rejected": -0.2982264757156372, + "loss": 4427.1689453125, + "loss/core": 4427.1689453125, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.040308952331543, + "rewards/margins": 1.3134760856628418, + "rewards/rejected": 0.7268326282501221, + "step": 420 + }, + { + "epoch": 0.4060671205063896, + "grad_norm": 4608.0, + "learning_rate": 3.1797675735315454e-07, + "logits/chosen": -2.014967679977417, + "logits/rejected": -2.074613094329834, + "logps/chosen": -0.2912288010120392, + "logps/rejected": -0.26653727889060974, + "loss": 4436.998046875, + "loss/core": 4436.998046875, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.238948345184326, + "rewards/margins": 0.6072303652763367, + "rewards/rejected": 1.6317180395126343, + "step": 425 + }, + { + "epoch": 0.41084438074764124, + "grad_norm": 5952.0, + "learning_rate": 3.1329879202620047e-07, + "logits/chosen": -2.0681891441345215, + "logits/rejected": -2.092362403869629, + "logps/chosen": -0.2822319567203522, + "logps/rejected": -0.26612913608551025, + "loss": 4406.2177734375, + "loss/core": 4406.2177734375, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.376003265380859, + "rewards/margins": 2.9532055854797363, + "rewards/rejected": 1.422797679901123, + "step": 430 + }, + { + "epoch": 0.41562164098889287, + "grad_norm": 1256.0, + "learning_rate": 3.0859702261229613e-07, + "logits/chosen": -2.350803852081299, + "logits/rejected": -2.3343265056610107, + "logps/chosen": -0.27796900272369385, + "logps/rejected": -0.2925384044647217, + "loss": 4436.28271484375, + "loss/core": 4436.28271484375, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.6284303665161133, + "rewards/margins": 0.6162401437759399, + "rewards/rejected": 1.0121902227401733, + "step": 435 + }, + { + "epoch": 0.4203989012301445, + "grad_norm": 5120.0, + "learning_rate": 3.0387321725463e-07, + "logits/chosen": -2.056842565536499, + "logits/rejected": -2.1276373863220215, + "logps/chosen": -0.2973445951938629, + "logps/rejected": -0.300473153591156, + "loss": 4422.72900390625, + "loss/core": 4422.72900390625, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.0658655166625977, + "rewards/margins": 1.684059500694275, + "rewards/rejected": 1.3818061351776123, + "step": 440 + }, + { + "epoch": 0.42517616147139614, + "grad_norm": 2016.0, + "learning_rate": 2.991291523832075e-07, + "logits/chosen": -1.9376929998397827, + "logits/rejected": -1.9126249551773071, + "logps/chosen": -0.2954959571361542, + "logps/rejected": -0.2955567538738251, + "loss": 4426.203125, + "loss/core": 4426.203125, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.961683988571167, + "rewards/margins": 1.3829456567764282, + "rewards/rejected": 1.5787384510040283, + "step": 445 + }, + { + "epoch": 0.4299534217126478, + "grad_norm": 8832.0, + "learning_rate": 2.943666120468088e-07, + "logits/chosen": -1.97494375705719, + "logits/rejected": -1.992226004600525, + "logps/chosen": -0.2974433898925781, + "logps/rejected": -0.28895604610443115, + "loss": 4420.197265625, + "loss/core": 4420.197265625, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.466543197631836, + "rewards/margins": 1.8552677631378174, + "rewards/rejected": 1.6112754344940186, + "step": 450 + }, + { + "epoch": 0.4347306819538994, + "grad_norm": 3072.0, + "learning_rate": 2.8958738724208073e-07, + "logits/chosen": -1.9352457523345947, + "logits/rejected": -2.1265578269958496, + "logps/chosen": -0.32452231645584106, + "logps/rejected": -0.31393322348594666, + "loss": 4418.81005859375, + "loss/core": 4418.81005859375, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.4475817680358887, + "rewards/margins": 1.9687623977661133, + "rewards/rejected": 1.4788196086883545, + "step": 455 + }, + { + "epoch": 0.4395079421951511, + "grad_norm": 2192.0, + "learning_rate": 2.8479327524001633e-07, + "logits/chosen": -2.062018871307373, + "logits/rejected": -2.0496106147766113, + "logps/chosen": -0.29210391640663147, + "logps/rejected": -0.30656731128692627, + "loss": 4420.3466796875, + "loss/core": 4420.3466796875, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.7037289142608643, + "rewards/margins": 1.8343826532363892, + "rewards/rejected": 1.8693459033966064, + "step": 460 + }, + { + "epoch": 0.44428520243640274, + "grad_norm": 892.0, + "learning_rate": 2.7998607891007493e-07, + "logits/chosen": -1.8918297290802002, + "logits/rejected": -1.9900699853897095, + "logps/chosen": -0.29650312662124634, + "logps/rejected": -0.2915659546852112, + "loss": 4426.185546875, + "loss/core": 4426.185546875, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.307586908340454, + "rewards/margins": 1.3769824504852295, + "rewards/rejected": 0.9306044578552246, + "step": 465 + }, + { + "epoch": 0.4490624626776544, + "grad_norm": 10496.0, + "learning_rate": 2.7516760604219616e-07, + "logits/chosen": -2.1318187713623047, + "logits/rejected": -2.199493646621704, + "logps/chosen": -0.27676263451576233, + "logps/rejected": -0.26010042428970337, + "loss": 4420.11279296875, + "loss/core": 4420.11279296875, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.0329606533050537, + "rewards/margins": 1.8643920421600342, + "rewards/rejected": 1.1685688495635986, + "step": 470 + }, + { + "epoch": 0.453839722918906, + "grad_norm": 3424.0, + "learning_rate": 2.703396686669646e-07, + "logits/chosen": -2.0941619873046875, + "logits/rejected": -2.0339064598083496, + "logps/chosen": -0.294299840927124, + "logps/rejected": -0.3125583529472351, + "loss": 4440.9580078125, + "loss/core": 4440.9580078125, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8425333499908447, + "rewards/margins": 0.29318827390670776, + "rewards/rejected": 1.5493448972702026, + "step": 475 + }, + { + "epoch": 0.45861698316015764, + "grad_norm": 996.0, + "learning_rate": 2.6550408237417884e-07, + "logits/chosen": -2.262141704559326, + "logits/rejected": -2.3195528984069824, + "logps/chosen": -0.275444895029068, + "logps/rejected": -0.2715856432914734, + "loss": 4427.11572265625, + "loss/core": 4427.11572265625, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.358044147491455, + "rewards/margins": 1.3245257139205933, + "rewards/rejected": 1.033518671989441, + "step": 480 + }, + { + "epoch": 0.4633942434014093, + "grad_norm": 884.0, + "learning_rate": 2.6066266563008265e-07, + "logits/chosen": -2.2081265449523926, + "logits/rejected": -2.2153499126434326, + "logps/chosen": -0.2904752790927887, + "logps/rejected": -0.29524365067481995, + "loss": 4432.99755859375, + "loss/core": 4432.99755859375, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.753950834274292, + "rewards/margins": 0.8617108464241028, + "rewards/rejected": 0.892240047454834, + "step": 485 + }, + { + "epoch": 0.4681715036426609, + "grad_norm": 1744.0, + "learning_rate": 2.5581723909351404e-07, + "logits/chosen": -2.158295154571533, + "logits/rejected": -2.2005717754364014, + "logps/chosen": -0.3060145378112793, + "logps/rejected": -0.29342374205589294, + "loss": 4430.84033203125, + "loss/core": 4430.84033203125, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.088890314102173, + "rewards/margins": 1.0260167121887207, + "rewards/rejected": 1.0628736019134521, + "step": 490 + }, + { + "epoch": 0.47294876388391255, + "grad_norm": 1896.0, + "learning_rate": 2.509696249312301e-07, + "logits/chosen": -2.228339433670044, + "logits/rejected": -2.3221817016601562, + "logps/chosen": -0.2925832271575928, + "logps/rejected": -0.29101797938346863, + "loss": 4421.3046875, + "loss/core": 4421.3046875, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.663743495941162, + "rewards/margins": 1.77371084690094, + "rewards/rejected": 0.8900324702262878, + "step": 495 + }, + { + "epoch": 0.47772602412516424, + "grad_norm": 2352.0, + "learning_rate": 2.461216461326642e-07, + "logits/chosen": -2.023076295852661, + "logits/rejected": -1.9727592468261719, + "logps/chosen": -0.2806738018989563, + "logps/rejected": -0.29887765645980835, + "loss": 4417.65283203125, + "loss/core": 4417.65283203125, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 4.167788505554199, + "rewards/margins": 2.10418963432312, + "rewards/rejected": 2.0635993480682373, + "step": 500 + }, + { + "epoch": 0.4825032843664159, + "grad_norm": 7104.0, + "learning_rate": 2.412751258243748e-07, + "logits/chosen": -2.1479763984680176, + "logits/rejected": -2.180921792984009, + "logps/chosen": -0.2815966010093689, + "logps/rejected": -0.27384239435195923, + "loss": 4422.50537109375, + "loss/core": 4422.50537109375, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 3.650578022003174, + "rewards/margins": 1.758052110671997, + "rewards/rejected": 1.8925259113311768, + "step": 505 + }, + { + "epoch": 0.4872805446076675, + "grad_norm": 324.0, + "learning_rate": 2.3643188658444158e-07, + "logits/chosen": -2.1493828296661377, + "logits/rejected": -2.2821457386016846, + "logps/chosen": -0.3144438862800598, + "logps/rejected": -0.29132357239723206, + "loss": 4428.4169921875, + "loss/core": 4428.4169921875, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.025472640991211, + "rewards/margins": 1.2251968383789062, + "rewards/rejected": 0.8002756834030151, + "step": 510 + }, + { + "epoch": 0.49205780484891914, + "grad_norm": 1976.0, + "learning_rate": 2.315937497570688e-07, + "logits/chosen": -2.0333778858184814, + "logits/rejected": -2.035680055618286, + "logps/chosen": -0.28305697441101074, + "logps/rejected": -0.30175352096557617, + "loss": 4433.373046875, + "loss/core": 4433.373046875, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.540903091430664, + "rewards/margins": 0.8402018547058105, + "rewards/rejected": 1.7007014751434326, + "step": 515 + }, + { + "epoch": 0.4968350650901708, + "grad_norm": 2752.0, + "learning_rate": 2.2676253476765194e-07, + "logits/chosen": -2.0594825744628906, + "logits/rejected": -2.229219675064087, + "logps/chosen": -0.3165486454963684, + "logps/rejected": -0.33373939990997314, + "loss": 4416.2041015625, + "loss/core": 4416.2041015625, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.1289918422698975, + "rewards/margins": 2.1620774269104004, + "rewards/rejected": 0.9669145345687866, + "step": 520 + }, + { + "epoch": 0.5016123253314224, + "grad_norm": 7648.0, + "learning_rate": 2.2194005843856633e-07, + "logits/chosen": -2.1951212882995605, + "logits/rejected": -2.163513660430908, + "logps/chosen": -0.29069000482559204, + "logps/rejected": -0.29456523060798645, + "loss": 4407.59716796875, + "loss/core": 4407.59716796875, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.40447998046875, + "rewards/margins": 2.884629726409912, + "rewards/rejected": 1.519850254058838, + "step": 525 + }, + { + "epoch": 0.5063895855726741, + "grad_norm": 368.0, + "learning_rate": 2.1712813430593434e-07, + "logits/chosen": -2.039048433303833, + "logits/rejected": -2.001903772354126, + "logps/chosen": -0.2846403121948242, + "logps/rejected": -0.28684523701667786, + "loss": 4424.06103515625, + "loss/core": 4424.06103515625, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.0643351078033447, + "rewards/margins": 1.5574977397918701, + "rewards/rejected": 1.5068371295928955, + "step": 530 + }, + { + "epoch": 0.5111668458139257, + "grad_norm": 3616.0, + "learning_rate": 2.123285719376292e-07, + "logits/chosen": -2.187757968902588, + "logits/rejected": -2.110003709793091, + "logps/chosen": -0.28276944160461426, + "logps/rejected": -0.27659210562705994, + "loss": 4418.234375, + "loss/core": 4418.234375, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.121807098388672, + "rewards/margins": 2.0180087089538574, + "rewards/rejected": 1.1037986278533936, + "step": 535 + }, + { + "epoch": 0.5159441060551774, + "grad_norm": 560.0, + "learning_rate": 2.0754317625276982e-07, + "logits/chosen": -2.1087703704833984, + "logits/rejected": -2.086918592453003, + "logps/chosen": -0.2788047194480896, + "logps/rejected": -0.28258147835731506, + "loss": 4444.259765625, + "loss/core": 4444.259765625, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.587753176689148, + "rewards/margins": 0.029676269739866257, + "rewards/rejected": 1.5580768585205078, + "step": 540 + }, + { + "epoch": 0.520721366296429, + "grad_norm": 10176.0, + "learning_rate": 2.0277374684296498e-07, + "logits/chosen": -2.3330397605895996, + "logits/rejected": -2.172541379928589, + "logps/chosen": -0.28407856822013855, + "logps/rejected": -0.2892422080039978, + "loss": 4422.20458984375, + "loss/core": 4422.20458984375, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.6327457427978516, + "rewards/margins": 1.716529130935669, + "rewards/rejected": 0.9162166714668274, + "step": 545 + }, + { + "epoch": 0.5254986265376806, + "grad_norm": 1416.0, + "learning_rate": 1.980220772955602e-07, + "logits/chosen": -1.896632194519043, + "logits/rejected": -1.9163414239883423, + "logps/chosen": -0.2775726616382599, + "logps/rejected": -0.2957395911216736, + "loss": 4429.7470703125, + "loss/core": 4429.7470703125, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.7539432048797607, + "rewards/margins": 1.1101391315460205, + "rewards/rejected": 1.6438043117523193, + "step": 550 + }, + { + "epoch": 0.5302758867789323, + "grad_norm": 756.0, + "learning_rate": 1.932899545191433e-07, + "logits/chosen": -2.0946218967437744, + "logits/rejected": -2.152243137359619, + "logps/chosen": -0.3078138530254364, + "logps/rejected": -0.2756267786026001, + "loss": 4413.80322265625, + "loss/core": 4413.80322265625, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.3336167335510254, + "rewards/margins": 2.344905376434326, + "rewards/rejected": 0.9887111783027649, + "step": 555 + }, + { + "epoch": 0.5350531470201839, + "grad_norm": 2208.0, + "learning_rate": 1.885791580715609e-07, + "logits/chosen": -2.1799540519714355, + "logits/rejected": -2.1663684844970703, + "logps/chosen": -0.2761627435684204, + "logps/rejected": -0.28063830733299255, + "loss": 4427.53564453125, + "loss/core": 4427.53564453125, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.810899496078491, + "rewards/margins": 1.2789033651351929, + "rewards/rejected": 1.5319960117340088, + "step": 560 + }, + { + "epoch": 0.5398304072614356, + "grad_norm": 2080.0, + "learning_rate": 1.8389145949069951e-07, + "logits/chosen": -2.0762481689453125, + "logits/rejected": -2.101166009902954, + "logps/chosen": -0.28619739413261414, + "logps/rejected": -0.30411672592163086, + "loss": 4429.6884765625, + "loss/core": 4429.6884765625, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.794001579284668, + "rewards/margins": 1.1226036548614502, + "rewards/rejected": 1.6713975667953491, + "step": 565 + }, + { + "epoch": 0.5446076675026872, + "grad_norm": 5824.0, + "learning_rate": 1.792286216282824e-07, + "logits/chosen": -2.077721118927002, + "logits/rejected": -2.128166913986206, + "logps/chosen": -0.28693076968193054, + "logps/rejected": -0.2858390212059021, + "loss": 4422.75732421875, + "loss/core": 4422.75732421875, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.361377000808716, + "rewards/margins": 1.646521806716919, + "rewards/rejected": 1.7148549556732178, + "step": 570 + }, + { + "epoch": 0.5493849277439389, + "grad_norm": 588.0, + "learning_rate": 1.745923979869336e-07, + "logits/chosen": -2.156691074371338, + "logits/rejected": -2.1882190704345703, + "logps/chosen": -0.3002692461013794, + "logps/rejected": -0.3224543035030365, + "loss": 4427.705078125, + "loss/core": 4427.705078125, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.8466622829437256, + "rewards/margins": 1.278574824333191, + "rewards/rejected": 0.5680874586105347, + "step": 575 + }, + { + "epoch": 0.5541621879851905, + "grad_norm": 402.0, + "learning_rate": 1.6998453206075708e-07, + "logits/chosen": -2.4301846027374268, + "logits/rejected": -2.4865431785583496, + "logps/chosen": -0.26671233773231506, + "logps/rejected": -0.27991726994514465, + "loss": 4438.52734375, + "loss/core": 4438.52734375, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.4181255102157593, + "rewards/margins": 0.4488455653190613, + "rewards/rejected": 0.9692800641059875, + "step": 580 + }, + { + "epoch": 0.5589394482264421, + "grad_norm": 1120.0, + "learning_rate": 1.6540675667967973e-07, + "logits/chosen": -2.1363842487335205, + "logits/rejected": -2.100966453552246, + "logps/chosen": -0.2700468599796295, + "logps/rejected": -0.2819743752479553, + "loss": 4433.23828125, + "loss/core": 4433.23828125, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.22441029548645, + "rewards/margins": 0.8479706048965454, + "rewards/rejected": 1.3764398097991943, + "step": 585 + }, + { + "epoch": 0.5637167084676937, + "grad_norm": 688.0, + "learning_rate": 1.60860793357805e-07, + "logits/chosen": -2.0266470909118652, + "logits/rejected": -2.1219282150268555, + "logps/chosen": -0.2864200174808502, + "logps/rejected": -0.2867039442062378, + "loss": 4410.529296875, + "loss/core": 4410.529296875, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.6321520805358887, + "rewards/margins": 2.5971732139587402, + "rewards/rejected": 1.0349791049957275, + "step": 590 + }, + { + "epoch": 0.5684939687089454, + "grad_norm": 940.0, + "learning_rate": 1.5634835164602196e-07, + "logits/chosen": -2.047276020050049, + "logits/rejected": -2.041769027709961, + "logps/chosen": -0.27496907114982605, + "logps/rejected": -0.27177929878234863, + "loss": 4433.5009765625, + "loss/core": 4433.5009765625, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.713202714920044, + "rewards/margins": 0.8228942155838013, + "rewards/rejected": 0.8903085589408875, + "step": 595 + }, + { + "epoch": 0.5732712289501971, + "grad_norm": 1688.0, + "learning_rate": 1.518711284891132e-07, + "logits/chosen": -2.2743325233459473, + "logits/rejected": -2.264472484588623, + "logps/chosen": -0.2655165493488312, + "logps/rejected": -0.28108400106430054, + "loss": 4434.0869140625, + "loss/core": 4434.0869140625, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.8811838626861572, + "rewards/margins": 0.7859585881233215, + "rewards/rejected": 1.0952253341674805, + "step": 600 + }, + { + "epoch": 0.5780484891914487, + "grad_norm": 3680.0, + "learning_rate": 1.47430807587603e-07, + "logits/chosen": -2.117419481277466, + "logits/rejected": -2.192072868347168, + "logps/chosen": -0.27613067626953125, + "logps/rejected": -0.2757434844970703, + "loss": 4405.54541015625, + "loss/core": 4405.54541015625, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.223845481872559, + "rewards/margins": 3.018414258956909, + "rewards/rejected": 1.205431342124939, + "step": 605 + }, + { + "epoch": 0.5828257494327004, + "grad_norm": 2688.0, + "learning_rate": 1.430290587645865e-07, + "logits/chosen": -2.1161704063415527, + "logits/rejected": -2.132699489593506, + "logps/chosen": -0.2725360095500946, + "logps/rejected": -0.27246853709220886, + "loss": 4413.8681640625, + "loss/core": 4413.8681640625, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 4.309195518493652, + "rewards/margins": 2.3448550701141357, + "rewards/rejected": 1.9643408060073853, + "step": 610 + }, + { + "epoch": 0.587603009673952, + "grad_norm": 1112.0, + "learning_rate": 1.3866753733777765e-07, + "logits/chosen": -1.9690643548965454, + "logits/rejected": -2.0342695713043213, + "logps/chosen": -0.3134908080101013, + "logps/rejected": -0.3014810383319855, + "loss": 4428.2001953125, + "loss/core": 4428.2001953125, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.745121717453003, + "rewards/margins": 1.2340598106384277, + "rewards/rejected": 1.5110619068145752, + "step": 615 + }, + { + "epoch": 0.5923802699152036, + "grad_norm": 828.0, + "learning_rate": 1.343478834970121e-07, + "logits/chosen": -2.141674518585205, + "logits/rejected": -2.1674609184265137, + "logps/chosen": -0.28517812490463257, + "logps/rejected": -0.29004770517349243, + "loss": 4441.734375, + "loss/core": 4441.734375, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.6989895105361938, + "rewards/margins": 0.22279176115989685, + "rewards/rejected": 1.4761978387832642, + "step": 620 + }, + { + "epoch": 0.5971575301564552, + "grad_norm": 19968.0, + "learning_rate": 1.3007172168743852e-07, + "logits/chosen": -1.933323621749878, + "logits/rejected": -1.9620736837387085, + "logps/chosen": -0.3021884560585022, + "logps/rejected": -0.3325551450252533, + "loss": 4414.35498046875, + "loss/core": 4414.35498046875, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.912782669067383, + "rewards/margins": 2.36540150642395, + "rewards/rejected": 1.5473812818527222, + "step": 625 + }, + { + "epoch": 0.6019347903977069, + "grad_norm": 1264.0, + "learning_rate": 1.25840659998631e-07, + "logits/chosen": -1.9333274364471436, + "logits/rejected": -2.048060894012451, + "logps/chosen": -0.27091148495674133, + "logps/rejected": -0.2834457755088806, + "loss": 4421.9296875, + "loss/core": 4421.9296875, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.9585728645324707, + "rewards/margins": 1.7152507305145264, + "rewards/rejected": 1.2433220148086548, + "step": 630 + }, + { + "epoch": 0.6067120506389586, + "grad_norm": 5376.0, + "learning_rate": 1.2165628955985313e-07, + "logits/chosen": -1.983690857887268, + "logits/rejected": -2.0693118572235107, + "logps/chosen": -0.3024982810020447, + "logps/rejected": -0.29896295070648193, + "loss": 4427.31494140625, + "loss/core": 4427.31494140625, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.329944372177124, + "rewards/margins": 1.3042433261871338, + "rewards/rejected": 1.0257014036178589, + "step": 635 + }, + { + "epoch": 0.6114893108802102, + "grad_norm": 668.0, + "learning_rate": 1.175201839416988e-07, + "logits/chosen": -2.2102410793304443, + "logits/rejected": -2.2873520851135254, + "logps/chosen": -0.29883983731269836, + "logps/rejected": -0.2977871298789978, + "loss": 4428.97705078125, + "loss/core": 4428.97705078125, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.9840648174285889, + "rewards/margins": 1.1769111156463623, + "rewards/rejected": 0.807153582572937, + "step": 640 + }, + { + "epoch": 0.6162665711214619, + "grad_norm": 1560.0, + "learning_rate": 1.1343389856433658e-07, + "logits/chosen": -2.1556880474090576, + "logits/rejected": -2.131955623626709, + "logps/chosen": -0.2893293499946594, + "logps/rejected": -0.2930451035499573, + "loss": 4438.78759765625, + "loss/core": 4438.78759765625, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.6869512796401978, + "rewards/margins": 0.43390756845474243, + "rewards/rejected": 1.2530437707901, + "step": 645 + }, + { + "epoch": 0.6210438313627135, + "grad_norm": 932.0, + "learning_rate": 1.0939897011258e-07, + "logits/chosen": -2.222175359725952, + "logits/rejected": -2.172823429107666, + "logps/chosen": -0.2886146605014801, + "logps/rejected": -0.29807931184768677, + "loss": 4437.9853515625, + "loss/core": 4437.9853515625, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.3422349691390991, + "rewards/margins": 0.49204936623573303, + "rewards/rejected": 0.8501856923103333, + "step": 650 + }, + { + "epoch": 0.6258210916039652, + "grad_norm": 4224.0, + "learning_rate": 1.0541691595800336e-07, + "logits/chosen": -2.060539960861206, + "logits/rejected": -2.123028039932251, + "logps/chosen": -0.28187817335128784, + "logps/rejected": -0.2881320118904114, + "loss": 4431.12353515625, + "loss/core": 4431.12353515625, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.0083515644073486, + "rewards/margins": 1.0040746927261353, + "rewards/rejected": 1.004276990890503, + "step": 655 + }, + { + "epoch": 0.6305983518452167, + "grad_norm": 616.0, + "learning_rate": 1.0148923358832021e-07, + "logits/chosen": -2.032500982284546, + "logits/rejected": -2.027933359146118, + "logps/chosen": -0.29038962721824646, + "logps/rejected": -0.30487364530563354, + "loss": 4425.306640625, + "loss/core": 4425.306640625, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.8122401237487793, + "rewards/margins": 1.49310302734375, + "rewards/rejected": 1.3191370964050293, + "step": 660 + }, + { + "epoch": 0.6353756120864684, + "grad_norm": 2960.0, + "learning_rate": 9.761740004423926e-08, + "logits/chosen": -2.1880297660827637, + "logits/rejected": -2.1642916202545166, + "logps/chosen": -0.28237125277519226, + "logps/rejected": -0.27983298897743225, + "loss": 4433.11083984375, + "loss/core": 4433.11083984375, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.367220163345337, + "rewards/margins": 0.8713071942329407, + "rewards/rejected": 1.495913028717041, + "step": 665 + }, + { + "epoch": 0.64015287232772, + "grad_norm": 588.0, + "learning_rate": 9.380287136400999e-08, + "logits/chosen": -1.982226014137268, + "logits/rejected": -2.0288047790527344, + "logps/chosen": -0.2729211151599884, + "logps/rejected": -0.31506603956222534, + "loss": 4413.7041015625, + "loss/core": 4413.7041015625, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.6589457988739014, + "rewards/margins": 2.350693702697754, + "rewards/rejected": 1.308251976966858, + "step": 670 + }, + { + "epoch": 0.6449301325689717, + "grad_norm": 1032.0, + "learning_rate": 9.004708203586628e-08, + "logits/chosen": -2.0349762439727783, + "logits/rejected": -2.077690601348877, + "logps/chosen": -0.28860360383987427, + "logps/rejected": -0.2868594527244568, + "loss": 4434.82080078125, + "loss/core": 4434.82080078125, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.6204445362091064, + "rewards/margins": 0.7260267734527588, + "rewards/rejected": 0.8944176435470581, + "step": 675 + }, + { + "epoch": 0.6497073928102234, + "grad_norm": 596.0, + "learning_rate": 8.635144445857407e-08, + "logits/chosen": -2.1599338054656982, + "logits/rejected": -2.0785868167877197, + "logps/chosen": -0.28200381994247437, + "logps/rejected": -0.29134702682495117, + "loss": 4434.482421875, + "loss/core": 4434.482421875, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.1464552879333496, + "rewards/margins": 0.7586153149604797, + "rewards/rejected": 1.3878400325775146, + "step": 680 + }, + { + "epoch": 0.654484653051475, + "grad_norm": 780.0, + "learning_rate": 8.271734841028552e-08, + "logits/chosen": -2.0853731632232666, + "logits/rejected": -2.0709526538848877, + "logps/chosen": -0.25362181663513184, + "logps/rejected": -0.2743147313594818, + "loss": 4427.447265625, + "loss/core": 4427.447265625, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.885993719100952, + "rewards/margins": 1.3240457773208618, + "rewards/rejected": 1.5619480609893799, + "step": 685 + }, + { + "epoch": 0.6592619132927267, + "grad_norm": 708.0, + "learning_rate": 7.91461605259007e-08, + "logits/chosen": -1.8858239650726318, + "logits/rejected": -1.9080051183700562, + "logps/chosen": -0.30707043409347534, + "logps/rejected": -0.31979310512542725, + "loss": 4436.5107421875, + "loss/core": 4436.5107421875, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.7793325185775757, + "rewards/margins": 0.6050835251808167, + "rewards/rejected": 1.1742490530014038, + "step": 690 + }, + { + "epoch": 0.6640391735339782, + "grad_norm": 916.0, + "learning_rate": 7.563922378313217e-08, + "logits/chosen": -2.273056745529175, + "logits/rejected": -2.28229022026062, + "logps/chosen": -0.30402398109436035, + "logps/rejected": -0.29174989461898804, + "loss": 4427.4306640625, + "loss/core": 4427.4306640625, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.9931259155273438, + "rewards/margins": 1.2991578578948975, + "rewards/rejected": 0.6939681768417358, + "step": 695 + }, + { + "epoch": 0.6688164337752299, + "grad_norm": 14528.0, + "learning_rate": 7.219785699746572e-08, + "logits/chosen": -2.105236768722534, + "logits/rejected": -2.20990252494812, + "logps/chosen": -0.2958459258079529, + "logps/rejected": -0.31115972995758057, + "loss": 4437.0419921875, + "loss/core": 4437.0419921875, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.240123748779297, + "rewards/margins": 0.5702623128890991, + "rewards/rejected": 1.6698611974716187, + "step": 700 + }, + { + "epoch": 0.6735936940164815, + "grad_norm": 2880.0, + "learning_rate": 6.882335432620779e-08, + "logits/chosen": -2.197855234146118, + "logits/rejected": -2.154949903488159, + "logps/chosen": -0.28139424324035645, + "logps/rejected": -0.2661704421043396, + "loss": 4420.2060546875, + "loss/core": 4420.2060546875, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.394221067428589, + "rewards/margins": 1.8499267101287842, + "rewards/rejected": 1.5442945957183838, + "step": 705 + }, + { + "epoch": 0.6783709542577332, + "grad_norm": 9536.0, + "learning_rate": 6.551698478180589e-08, + "logits/chosen": -1.889145851135254, + "logits/rejected": -1.876465082168579, + "logps/chosen": -0.27592915296554565, + "logps/rejected": -0.2931953966617584, + "loss": 4422.54833984375, + "loss/core": 4422.54833984375, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 3.609574794769287, + "rewards/margins": 1.6662747859954834, + "rewards/rejected": 1.9433000087738037, + "step": 710 + }, + { + "epoch": 0.6831482144989849, + "grad_norm": 792.0, + "learning_rate": 6.22799917546252e-08, + "logits/chosen": -2.253756284713745, + "logits/rejected": -2.234802484512329, + "logps/chosen": -0.2832790017127991, + "logps/rejected": -0.3265189826488495, + "loss": 4430.35302734375, + "loss/core": 4430.35302734375, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.8739410638809204, + "rewards/margins": 1.0730499029159546, + "rewards/rejected": 0.8008909225463867, + "step": 715 + }, + { + "epoch": 0.6879254747402365, + "grad_norm": 2096.0, + "learning_rate": 5.9113592545359944e-08, + "logits/chosen": -1.9960432052612305, + "logits/rejected": -2.0051181316375732, + "logps/chosen": -0.2970682680606842, + "logps/rejected": -0.29701724648475647, + "loss": 4437.2138671875, + "loss/core": 4437.2138671875, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.0947940349578857, + "rewards/margins": 0.5625554919242859, + "rewards/rejected": 1.5322383642196655, + "step": 720 + }, + { + "epoch": 0.6927027349814882, + "grad_norm": 932.0, + "learning_rate": 5.601897790725643e-08, + "logits/chosen": -2.124847888946533, + "logits/rejected": -2.2395145893096924, + "logps/chosen": -0.28501376509666443, + "logps/rejected": -0.27728787064552307, + "loss": 4433.05029296875, + "loss/core": 4433.05029296875, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.5915935039520264, + "rewards/margins": 0.8602860569953918, + "rewards/rejected": 0.7313073873519897, + "step": 725 + }, + { + "epoch": 0.6974799952227397, + "grad_norm": 6816.0, + "learning_rate": 5.299731159831952e-08, + "logits/chosen": -2.2217555046081543, + "logits/rejected": -2.272338628768921, + "logps/chosen": -0.28637298941612244, + "logps/rejected": -0.2902549207210541, + "loss": 4443.1611328125, + "loss/core": 4443.1611328125, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.0200834274291992, + "rewards/margins": 0.1075645312666893, + "rewards/rejected": 0.9125189781188965, + "step": 730 + }, + { + "epoch": 0.7022572554639914, + "grad_norm": 2208.0, + "learning_rate": 5.0049729943671013e-08, + "logits/chosen": -2.0483860969543457, + "logits/rejected": -2.0965161323547363, + "logps/chosen": -0.3195610046386719, + "logps/rejected": -0.3177678883075714, + "loss": 4434.37109375, + "loss/core": 4434.37109375, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.4221794605255127, + "rewards/margins": 0.7587219476699829, + "rewards/rejected": 0.6634576320648193, + "step": 735 + }, + { + "epoch": 0.707034515705243, + "grad_norm": 972.0, + "learning_rate": 4.7177341408223994e-08, + "logits/chosen": -2.103890895843506, + "logits/rejected": -2.122037172317505, + "logps/chosen": -0.3093516230583191, + "logps/rejected": -0.31388068199157715, + "loss": 4435.8095703125, + "loss/core": 4435.8095703125, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.601134777069092, + "rewards/margins": 0.7491642236709595, + "rewards/rejected": 1.8519706726074219, + "step": 740 + }, + { + "epoch": 0.7118117759464947, + "grad_norm": 5824.0, + "learning_rate": 4.438122617983442e-08, + "logits/chosen": -2.053230047225952, + "logits/rejected": -2.097982406616211, + "logps/chosen": -0.28634682297706604, + "logps/rejected": -0.28023213148117065, + "loss": 4423.0634765625, + "loss/core": 4423.0634765625, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.1622960567474365, + "rewards/margins": 1.6293554306030273, + "rewards/rejected": 1.5329406261444092, + "step": 745 + }, + { + "epoch": 0.7165890361877463, + "grad_norm": 2544.0, + "learning_rate": 4.1662435763087114e-08, + "logits/chosen": -2.1885745525360107, + "logits/rejected": -2.1799445152282715, + "logps/chosen": -0.3118075728416443, + "logps/rejected": -0.3031180500984192, + "loss": 4445.6220703125, + "loss/core": 4445.6220703125, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.2049453258514404, + "rewards/margins": -0.05652730539441109, + "rewards/rejected": 1.261472463607788, + "step": 750 + }, + { + "epoch": 0.721366296428998, + "grad_norm": 2000.0, + "learning_rate": 3.902199258386732e-08, + "logits/chosen": -1.9294588565826416, + "logits/rejected": -1.839930534362793, + "logps/chosen": -0.3078926205635071, + "logps/rejected": -0.29328030347824097, + "loss": 4435.65625, + "loss/core": 4435.65625, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.3930587768554688, + "rewards/margins": 0.6730942726135254, + "rewards/rejected": 1.719964623451233, + "step": 755 + }, + { + "epoch": 0.7261435566702497, + "grad_norm": 1056.0, + "learning_rate": 3.646088960486862e-08, + "logits/chosen": -2.1844775676727295, + "logits/rejected": -2.2019870281219482, + "logps/chosen": -0.30291616916656494, + "logps/rejected": -0.292309045791626, + "loss": 4429.7392578125, + "loss/core": 4429.7392578125, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.988659143447876, + "rewards/margins": 1.112038254737854, + "rewards/rejected": 0.8766206502914429, + "step": 760 + }, + { + "epoch": 0.7309208169115012, + "grad_norm": 1656.0, + "learning_rate": 3.398008995217988e-08, + "logits/chosen": -2.0841526985168457, + "logits/rejected": -2.1565730571746826, + "logps/chosen": -0.2759454846382141, + "logps/rejected": -0.2714347541332245, + "loss": 4434.8095703125, + "loss/core": 4434.8095703125, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.7954429388046265, + "rewards/margins": 0.7274172902107239, + "rewards/rejected": 1.0680259466171265, + "step": 765 + }, + { + "epoch": 0.7356980771527529, + "grad_norm": 2832.0, + "learning_rate": 3.1580526553093315e-08, + "logits/chosen": -1.9811954498291016, + "logits/rejected": -1.9468730688095093, + "logps/chosen": -0.3071630895137787, + "logps/rejected": -0.2786198556423187, + "loss": 4427.37744140625, + "loss/core": 4427.37744140625, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.7594974040985107, + "rewards/margins": 1.3463454246520996, + "rewards/rejected": 2.4131522178649902, + "step": 770 + }, + { + "epoch": 0.7404753373940045, + "grad_norm": 412.0, + "learning_rate": 2.9263101785268252e-08, + "logits/chosen": -2.095184803009033, + "logits/rejected": -2.1245932579040527, + "logps/chosen": -0.30588921904563904, + "logps/rejected": -0.29908287525177, + "loss": 4436.82177734375, + "loss/core": 4436.82177734375, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.5300137996673584, + "rewards/margins": 0.5759037733078003, + "rewards/rejected": 0.9541099667549133, + "step": 775 + }, + { + "epoch": 0.7452525976352562, + "grad_norm": 1480.0, + "learning_rate": 2.7028687137384264e-08, + "logits/chosen": -2.1838436126708984, + "logits/rejected": -2.197448492050171, + "logps/chosen": -0.30141058564186096, + "logps/rejected": -0.29812097549438477, + "loss": 4436.86181640625, + "loss/core": 4436.86181640625, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8982967138290405, + "rewards/margins": 0.5740464925765991, + "rewards/rejected": 1.3242502212524414, + "step": 780 + }, + { + "epoch": 0.7500298578765078, + "grad_norm": 564.0, + "learning_rate": 2.4878122881409447e-08, + "logits/chosen": -1.9970121383666992, + "logits/rejected": -1.9551795721054077, + "logps/chosen": -0.31093713641166687, + "logps/rejected": -0.2889935374259949, + "loss": 4418.28271484375, + "loss/core": 4418.28271484375, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.431575298309326, + "rewards/margins": 2.0386598110198975, + "rewards/rejected": 1.3929154872894287, + "step": 785 + }, + { + "epoch": 0.7548071181177595, + "grad_norm": 19840.0, + "learning_rate": 2.2812217756608937e-08, + "logits/chosen": -2.1677725315093994, + "logits/rejected": -1.9948818683624268, + "logps/chosen": -0.2858007550239563, + "logps/rejected": -0.31171754002571106, + "loss": 4436.4150390625, + "loss/core": 4436.4150390625, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.6222074031829834, + "rewards/margins": 0.6268441081047058, + "rewards/rejected": 1.9953632354736328, + "step": 790 + }, + { + "epoch": 0.7595843783590112, + "grad_norm": 2176.0, + "learning_rate": 2.0831748665410763e-08, + "logits/chosen": -1.9716495275497437, + "logits/rejected": -1.9900119304656982, + "logps/chosen": -0.3067142367362976, + "logps/rejected": -0.29664263129234314, + "loss": 4428.5654296875, + "loss/core": 4428.5654296875, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.3875348567962646, + "rewards/margins": 1.2067521810531616, + "rewards/rejected": 1.1807825565338135, + "step": 795 + }, + { + "epoch": 0.7643616386002627, + "grad_norm": 824.0, + "learning_rate": 1.8937460381244967e-08, + "logits/chosen": -2.1559529304504395, + "logits/rejected": -2.1331565380096436, + "logps/chosen": -0.289201557636261, + "logps/rejected": -0.28383710980415344, + "loss": 4430.1826171875, + "loss/core": 4430.1826171875, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.2973222732543945, + "rewards/margins": 1.0815565586090088, + "rewards/rejected": 1.2157660722732544, + "step": 800 + }, + { + "epoch": 0.7691388988415144, + "grad_norm": 1240.0, + "learning_rate": 1.713006526846439e-08, + "logits/chosen": -2.0271267890930176, + "logits/rejected": -2.0145370960235596, + "logps/chosen": -0.27510371804237366, + "logps/rejected": -0.2665782868862152, + "loss": 4441.60107421875, + "loss/core": 4441.60107421875, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.2546658515930176, + "rewards/margins": 0.22854623198509216, + "rewards/rejected": 2.0261194705963135, + "step": 805 + }, + { + "epoch": 0.773916159082766, + "grad_norm": 840.0, + "learning_rate": 1.541024301445404e-08, + "logits/chosen": -2.2316105365753174, + "logits/rejected": -2.135343074798584, + "logps/chosen": -0.30771583318710327, + "logps/rejected": -0.31672921776771545, + "loss": 4443.7978515625, + "loss/core": 4443.7978515625, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.536110520362854, + "rewards/margins": 0.05860384181141853, + "rewards/rejected": 1.4775065183639526, + "step": 810 + }, + { + "epoch": 0.7786934193240177, + "grad_norm": 1800.0, + "learning_rate": 1.3778640374027983e-08, + "logits/chosen": -2.0983340740203857, + "logits/rejected": -2.187591552734375, + "logps/chosen": -0.2833598256111145, + "logps/rejected": -0.28382277488708496, + "loss": 4430.76611328125, + "loss/core": 4430.76611328125, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.1276965141296387, + "rewards/margins": 1.0290817022323608, + "rewards/rejected": 1.0986146926879883, + "step": 815 + }, + { + "epoch": 0.7834706795652693, + "grad_norm": 636.0, + "learning_rate": 1.2235870926211616e-08, + "logits/chosen": -2.1024179458618164, + "logits/rejected": -2.1378636360168457, + "logps/chosen": -0.28758540749549866, + "logps/rejected": -0.31663960218429565, + "loss": 4427.5947265625, + "loss/core": 4427.5947265625, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.9671586751937866, + "rewards/margins": 1.2754833698272705, + "rewards/rejected": 0.6916751265525818, + "step": 820 + }, + { + "epoch": 0.788247939806521, + "grad_norm": 2944.0, + "learning_rate": 1.0782514843499652e-08, + "logits/chosen": -2.1443867683410645, + "logits/rejected": -2.248300075531006, + "logps/chosen": -0.2994143068790436, + "logps/rejected": -0.28394490480422974, + "loss": 4427.640625, + "loss/core": 4427.640625, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.490103006362915, + "rewards/margins": 1.267347812652588, + "rewards/rejected": 1.2227551937103271, + "step": 825 + }, + { + "epoch": 0.7930252000477725, + "grad_norm": 1272.0, + "learning_rate": 9.419118673676924e-09, + "logits/chosen": -2.18145489692688, + "logits/rejected": -2.130202293395996, + "logps/chosen": -0.2789270877838135, + "logps/rejected": -0.3010575771331787, + "loss": 4444.62744140625, + "loss/core": 4444.62744140625, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.7224094867706299, + "rewards/margins": 0.0003525257052388042, + "rewards/rejected": 1.7220569849014282, + "step": 830 + }, + { + "epoch": 0.7978024602890242, + "grad_norm": 7616.0, + "learning_rate": 8.14619513428405e-09, + "logits/chosen": -2.0989551544189453, + "logits/rejected": -2.0986316204071045, + "logps/chosen": -0.25472623109817505, + "logps/rejected": -0.3020690977573395, + "loss": 4423.72119140625, + "loss/core": 4423.72119140625, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.4041237831115723, + "rewards/margins": 1.5831953287124634, + "rewards/rejected": 0.8209284543991089, + "step": 835 + }, + { + "epoch": 0.8025797205302759, + "grad_norm": 21504.0, + "learning_rate": 6.96422291980539e-09, + "logits/chosen": -2.036388874053955, + "logits/rejected": -2.085162401199341, + "logps/chosen": -0.32136353850364685, + "logps/rejected": -0.3007509410381317, + "loss": 4421.49072265625, + "loss/core": 4421.49072265625, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.8142402172088623, + "rewards/margins": 1.7705316543579102, + "rewards/rejected": 1.0437085628509521, + "step": 840 + }, + { + "epoch": 0.8073569807715275, + "grad_norm": 764.0, + "learning_rate": 5.8736465216517594e-09, + "logits/chosen": -2.175672769546509, + "logits/rejected": -2.2510218620300293, + "logps/chosen": -0.2935303747653961, + "logps/rejected": -0.3111540675163269, + "loss": 4431.751953125, + "loss/core": 4431.751953125, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.4188427925109863, + "rewards/margins": 0.9629634618759155, + "rewards/rejected": 0.4558793008327484, + "step": 845 + }, + { + "epoch": 0.8121342410127792, + "grad_norm": 3712.0, + "learning_rate": 4.874876061005173e-09, + "logits/chosen": -2.2930002212524414, + "logits/rejected": -2.3410279750823975, + "logps/chosen": -0.251552551984787, + "logps/rejected": -0.24615788459777832, + "loss": 4435.8291015625, + "loss/core": 4435.8291015625, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.5625412464141846, + "rewards/margins": 0.6484128832817078, + "rewards/rejected": 0.9141284227371216, + "step": 850 + }, + { + "epoch": 0.8169115012540308, + "grad_norm": 2384.0, + "learning_rate": 3.968287134589188e-09, + "logits/chosen": -2.006430149078369, + "logits/rejected": -2.0440707206726074, + "logps/chosen": -0.301272451877594, + "logps/rejected": -0.29304254055023193, + "loss": 4425.640625, + "loss/core": 4425.640625, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.6843559741973877, + "rewards/margins": 1.433323621749878, + "rewards/rejected": 1.2510322332382202, + "step": 855 + }, + { + "epoch": 0.8216887614952825, + "grad_norm": 508.0, + "learning_rate": 3.154220673422192e-09, + "logits/chosen": -1.952792763710022, + "logits/rejected": -2.0415608882904053, + "logps/chosen": -0.2997278571128845, + "logps/rejected": -0.29428717494010925, + "loss": 4416.9169921875, + "loss/core": 4416.9169921875, + "rewards/accuracies": 0.875, + "rewards/chosen": 4.084506034851074, + "rewards/margins": 2.108910083770752, + "rewards/rejected": 1.975595474243164, + "step": 860 + }, + { + "epoch": 0.826466021736534, + "grad_norm": 1544.0, + "learning_rate": 2.4329828146074096e-09, + "logits/chosen": -1.961377739906311, + "logits/rejected": -2.034229040145874, + "logps/chosen": -0.2911316156387329, + "logps/rejected": -0.2944122850894928, + "loss": 4433.12158203125, + "loss/core": 4433.12158203125, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.4497056007385254, + "rewards/margins": 0.8692841529846191, + "rewards/rejected": 1.5804215669631958, + "step": 865 + }, + { + "epoch": 0.8312432819777857, + "grad_norm": 1304.0, + "learning_rate": 1.8048447862070714e-09, + "logits/chosen": -2.1136727333068848, + "logits/rejected": -2.121825695037842, + "logps/chosen": -0.3133166432380676, + "logps/rejected": -0.29426655173301697, + "loss": 4415.9951171875, + "loss/core": 4415.9951171875, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.1630001068115234, + "rewards/margins": 2.198111057281494, + "rewards/rejected": 0.9648886919021606, + "step": 870 + }, + { + "epoch": 0.8360205422190374, + "grad_norm": 968.0, + "learning_rate": 1.2700428052447033e-09, + "logits/chosen": -2.1847896575927734, + "logits/rejected": -2.2328908443450928, + "logps/chosen": -0.3081599771976471, + "logps/rejected": -0.3277169167995453, + "loss": 4434.0029296875, + "loss/core": 4434.0029296875, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.8598581552505493, + "rewards/margins": 0.8051584959030151, + "rewards/rejected": 1.0546996593475342, + "step": 875 + }, + { + "epoch": 0.840797802460289, + "grad_norm": 8000.0, + "learning_rate": 8.287779888734858e-10, + "logits/chosen": -2.1225297451019287, + "logits/rejected": -2.1040260791778564, + "logps/chosen": -0.290223628282547, + "logps/rejected": -0.32425063848495483, + "loss": 4436.90234375, + "loss/core": 4436.90234375, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.4622364044189453, + "rewards/margins": 0.6243095993995667, + "rewards/rejected": 1.8379265069961548, + "step": 880 + }, + { + "epoch": 0.8455750627015407, + "grad_norm": 1528.0, + "learning_rate": 4.812162787445062e-10, + "logits/chosen": -2.043444871902466, + "logits/rejected": -2.1212260723114014, + "logps/chosen": -0.2810841202735901, + "logps/rejected": -0.277138888835907, + "loss": 4423.04150390625, + "loss/core": 4423.04150390625, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.739732265472412, + "rewards/margins": 1.617659568786621, + "rewards/rejected": 1.122072458267212, + "step": 885 + }, + { + "epoch": 0.8503523229427923, + "grad_norm": 5856.0, + "learning_rate": 2.2748837860270265e-10, + "logits/chosen": -1.9764293432235718, + "logits/rejected": -2.0967178344726562, + "logps/chosen": -0.29462671279907227, + "logps/rejected": -0.27400484681129456, + "loss": 4430.4931640625, + "loss/core": 4430.4931640625, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.5097718238830566, + "rewards/margins": 1.0680214166641235, + "rewards/rejected": 1.4417507648468018, + "step": 890 + }, + { + "epoch": 0.855129583184044, + "grad_norm": 2288.0, + "learning_rate": 6.768970513457151e-11, + "logits/chosen": -2.046212673187256, + "logits/rejected": -2.080132007598877, + "logps/chosen": -0.2724604308605194, + "logps/rejected": -0.2843073904514313, + "loss": 4434.88623046875, + "loss/core": 4434.88623046875, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.3006958961486816, + "rewards/margins": 0.7250384092330933, + "rewards/rejected": 1.5756577253341675, + "step": 895 + }, + { + "epoch": 0.8599068434252956, + "grad_norm": 2272.0, + "learning_rate": 1.8803520859811406e-12, + "logits/chosen": -2.057281970977783, + "logits/rejected": -2.177447557449341, + "logps/chosen": -0.2676500976085663, + "logps/rejected": -0.26869815587997437, + "loss": 4414.8955078125, + "loss/core": 4414.8955078125, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 3.1301474571228027, + "rewards/margins": 2.2849082946777344, + "rewards/rejected": 0.8452390432357788, + "step": 900 + }, + { + "epoch": 0.8599068434252956, + "step": 900, + "total_flos": 0.0, + "train_loss": 4428.528517795139, + "train_runtime": 7133.5294, + "train_samples_per_second": 2.019, + "train_steps_per_second": 0.126 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..84a0cc9 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ce07509af78d766e3855704e08329dcc31596eeeb4a67fae847c23a8c3590fc +size 7057