From 5da0fd25b7cad6fb4f3ce9a05cba330ec5d72f11 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Thu, 23 Jul 2026 19:45:13 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: promotion/qwen3-8b-aaai27-flagship-sppo-avg-s42 Source: Original Platform --- .gitattributes | 36 + README.md | 24 + all_results.json | 9 + chat_template.jinja | 89 ++ config.json | 71 + config.yaml | 58 + generation_config.json | 12 + job_status.json | 23 + model.safetensors | 3 + objective_protocol.json | 69 + pair_manifest.json | 34 + provenance.json | 13 + resource_profile.json | 21 + run_status.json | 14 + split_manifest.json | 23 + stability_gate.json | 38 + tokenizer.json | 3 + tokenizer_config.json | 30 + train_results.json | 9 + trainer_state.json | 2923 +++++++++++++++++++++++++++++++++++++++ training_args.bin | 3 + 21 files changed, 3505 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 config.yaml create mode 100644 generation_config.json create mode 100644 job_status.json create mode 100644 model.safetensors create mode 100644 objective_protocol.json create mode 100644 pair_manifest.json create mode 100644 provenance.json create mode 100644 resource_profile.json create mode 100644 run_status.json create mode 100644 split_manifest.json create mode 100644 stability_gate.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..dbaab71 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +--- +library_name: transformers +base_model: Qwen/Qwen3-8B +tags: +- ronpo +- mnpo +- preference-optimization +--- + +# qwen3-8b-aaai27-flagship-sppo-avg-s42 + +Research checkpoint for the RONPO AAAI revision experiments. + +- Method: sppo_avg +- Base model: `Qwen/Qwen3-8B` +- Seed: 42 +- Local source at upload time: `/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/sppo_avg/seed42/attempt1` +- Uploaded at UTC: 2026-07-13T04:31:51Z +- Run status metadata: `{"attempt": 1, "effective_batch_size": 16, "method": "sppo_avg", "objective_protocol": "objective_protocol.json", "optimizer_steps": 900, "resource_profile": "resource_profile.json", "seed": 42, "split_manifest": "split_manifest.json", "stability_gate": "stability_gate.json", "status": "completed", "wandb_run_id": "279449de1b21", "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/279449de1b21"}` + +AAAI-27 P1 matched budget: 900 optimizer steps, effective batch 16; passed non-thinking and collapse stability gates; sealed-test metrics are not used for checkpoint selection. + +Intended use: reproducibility and evaluation for the RONPO paper. This +checkpoint is not intended as a production assistant. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..39aaffa --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 0.7102717447943158, + "train_runtime": 7093.1544, + "train_samples": 16746, + "train_samples_per_second": 2.03, + "train_steps_per_second": 0.127 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..5b3388f --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +model_name_or_path: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218 +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/avg: 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: sppo +eta: 0.0075 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.0 +preference_sft_weight: 0.0 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 5.0e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 42 +gradient_accumulation_steps: 16 +gradient_checkpointing: true +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 1 +per_device_eval_batch_size: 1 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/sppo_avg/seed42/attempt1 +run_name: aaai27-flagship-full-sppo_avg-s42-a1 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/job_status.json b/job_status.json new file mode 100644 index 0000000..a9eafc6 --- /dev/null +++ b/job_status.json @@ -0,0 +1,23 @@ +{ + "status": "completed", + "stage": "full", + "method": "sppo_avg", + "seed": 42, + "attempt": 1, + "gpu": 0, + "gpus": [ + 0 + ], + "returncode": 0, + "finite_metrics": true, + "oom": false, + "traceback": false, + "reason": "passed", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "279449de1b21", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/279449de1b21", + "output_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/sppo_avg/seed42/attempt1", + "log": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/logs/full_sppo_avg_s42_a1.log", + "completed_at": "2026-07-13T04:29:35Z" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..4ff8dde --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37d91e589f3a3cc1cd69c5b1f3c113842e7002b379ab84ea8b0904066fb515da +size 16381517208 diff --git a/objective_protocol.json b/objective_protocol.json new file mode 100644 index 0000000..35b3b44 --- /dev/null +++ b/objective_protocol.json @@ -0,0 +1,69 @@ +{ + "schema_version": 1, + "frozen_at_kst": "2026-07-12T21:40:00+09:00", + "base_model": "Qwen/Qwen3-8B", + "primary_triple": { + "helpfulness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "ultrafeedback-helpfulness", + "transform": "identity" + }, + "safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "conciseness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "helpsteer-verbosity", + "transform": "negate" + } + }, + "gate_thresholds": { + "raw_score_std_strictly_greater_than": 0.01, + "aggregate_prompt_pair_spearman_median_strictly_less_than": 0.0, + "aggregate_pairwise_top1_mismatch_at_least": 0.5 + }, + "gate_pool": "source-train Qwen3-8B response pool split into deterministic train/validation; sealed source-test excluded", + "training_seeds": [42, 43, 44], + "optimizer_steps": 900, + "effective_batch_size": 16, + "decode": { + "enable_thinking": false, + "temperature": 0.7, + "top_p": 0.9, + "max_new_tokens": 2048, + "decode_seed": 42 + }, + "methods": [ + "ronpo_full_expect", + "ronpo_k_only", + "dpo", + "ipo", + "simpo", + "kto", + "sppo_avg", + "inpo_avg", + "ht_mnpo_helpfulness", + "ht_mnpo_safety", + "ht_mnpo_conciseness" + ], + "selection_policy": "No sealed-test selection. One preregistered configuration per method; failures use only the declared stability retry recipe, at most three attempts.", + "stability_gate": { + "validation_prompts": 128, + "min_candidate_base_mean_word_ratio": 0.33, + "max_candidate_base_mean_word_ratio": 2.0, + "max_consecutive_identical_word_run": 20, + "think_tag_count": 0, + "empty_response_count": 0, + "retry_recipe": "attempt 2 halves learning rate, enables gradient checkpointing, and adds reference=0.02/SFT=0.002 anchoring to unanchored unified losses; attempt 3 quarters learning rate and uses reference=0.05/SFT=0.005; RONPO uses reference=0.05/SFT=0.005 from attempt 1" + }, + "primary_metric": "mean prompt-level minimum of per-prompt min-max-normalized helpfulness, safety, conciseness", + "bootstrap": { + "unit": "prompt", + "paired": true, + "resamples": 2000, + "interval": "percentile_95", + "seed": 42 + } +} diff --git a/pair_manifest.json b/pair_manifest.json new file mode 100644 index 0000000..be034b0 --- /dev/null +++ b/pair_manifest.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "objectives": [ + "helpfulness", + "safety", + "conciseness" + ], + "normalization": "per_prompt_minmax", + "adversary": { + "steps": 25, + "alpha": 1.0, + "kappa": 0.05, + "preference_scale": 8.0 + }, + "ronpo_estimator": "Rao-Blackwellized full sigma expectation with three common response pairs per prompt", + "counts": { + "train": { + "avg": 16746, + "ronpo_full_expect": 50340, + "ronpo_k_only": 50340, + "ht_mnpo_helpfulness": 16743, + "ht_mnpo_safety": 16764, + "ht_mnpo_conciseness": 16755 + }, + "validation": { + "avg": 1859, + "ronpo_full_expect": 5592, + "ronpo_k_only": 5592, + "ht_mnpo_helpfulness": 1864, + "ht_mnpo_safety": 1862, + "ht_mnpo_conciseness": 1858 + } + } +} diff --git a/provenance.json b/provenance.json new file mode 100644 index 0000000..c5e9158 --- /dev/null +++ b/provenance.json @@ -0,0 +1,13 @@ +{ + "method": "sppo_avg", + "seed": 42, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "279449de1b21", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/279449de1b21", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json" +} diff --git a/resource_profile.json b/resource_profile.json new file mode 100644 index 0000000..18eec47 --- /dev/null +++ b/resource_profile.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "measured_at_kst": "2026-07-12T22:38:00+09:00", + "hardware": "NVIDIA B200 183359 MiB reported total; 178.35 GiB PyTorch-visible capacity", + "smoke_evidence": { + "attempted_profile": {"per_device_batch_size": 2, "gradient_accumulation_steps": 8, "gradient_checkpointing": false, "effective_batch_size": 16}, + "outcome": "OOM before completing the 20-step smoke across every launched unified baseline; representative process used 176.87 GiB and failed a 1.62 GiB allocation. KTO also OOMed at the optimizer step.", + "paper_metric": false + }, + "selected_full_training_profile": { + "per_device_batch_size": 2, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": false, + "attention_implementation": "PyTorch SDPA with cuDNN SDPA disabled", + "effective_batch_size": 16, + "applies_symmetrically_to_all_unified_pairwise_methods": true, + "kto_implementation_constraint": "TRL KTO requires actual batch size > 1. KTO uses batch 2 / accumulation 8 and identical AdamW with foreach=False. PyTorch SDPA is used with cuDNN SDPA disabled. The fixed base reference model is placed on a second B200 and its exact logits are returned to the policy device; the policy/optimizer remain on the first B200. This avoids a measured ~6-hour reference-precompute path while leaving the KTO loss, actual/effective batch, optimizer, and 900-step budget unchanged." + }, + "retry_semantics": "Resource selection is separated from method-stability attempts. Full attempt 1 uses the smoke-validated SDPA batch-2 profile and retains the preregistered method LR/anchor; attempts 2 and 3 fall back to batch 1 / accumulation 16 / gradient checkpointing and apply the declared LR/anchor stabilization recipe.", + "provenance_note": "No reward or sealed-evaluation result was available when this hardware profile was selected." +} diff --git a/run_status.json b/run_status.json new file mode 100644 index 0000000..90c15c0 --- /dev/null +++ b/run_status.json @@ -0,0 +1,14 @@ +{ + "method": "sppo_avg", + "seed": 42, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "279449de1b21", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/279449de1b21", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json", + "status": "completed" +} diff --git a/split_manifest.json b/split_manifest.json new file mode 100644 index 0000000..4fb96bd --- /dev/null +++ b/split_manifest.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "split_rule": "sort source-train prompts by sha256('flagship-v1||'+prompt_id); first round(10%) validation", + "validation_fraction": 0.1, + "source_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/avg_oracle", + "base_model": "Qwen/Qwen3-8B", + "counts": { + "train": 16780, + "validation": 1864, + "sealed_test": 604 + }, + "prompt_id_sha256": { + "train": "fcf6f2ea5a0b7bcd41179e6f5288f4d850b2d04bdb729da91907bd18bf733a69", + "validation": "a87ba6f519192164daa487b109a6547dd8cbde46a27a373d6da201ca2d061b8f", + "sealed_test": "b398ced6af6cf73d731e1a5f32a0631d8060323d51eda1e375695566bf5f1867" + }, + "overlaps": { + "train_validation": 0, + "train_sealed_test": 0, + "validation_sealed_test": 0 + }, + "sealed_test_policy": "prompts only until S4a; no objective scores or model generations used in S0-S3" +} diff --git a/stability_gate.json b/stability_gate.json new file mode 100644 index 0000000..b6d2a36 --- /dev/null +++ b/stability_gate.json @@ -0,0 +1,38 @@ +{ + "status": "passed", + "passed": true, + "base": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 388.1484375, + "max_words": 1379, + "max_repeat_run": 4 + }, + "candidate": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 402.5625, + "max_words": 1403, + "max_repeat_run": 3 + }, + "candidate_base_mean_word_ratio": 1.037135438681239, + "thresholds": { + "min_length_ratio": 0.33, + "max_length_ratio": 2.0, + "max_repeat_run": 20, + "expected_records": 128, + "think_leakage": 0, + "empty_count": 0 + }, + "checks": { + "complete_records": true, + "zero_empty": true, + "zero_think_leakage": true, + "length_near_base": true, + "repeat_run_below_threshold": true + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ffd19d2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..39aaffa --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 0.7102717447943158, + "train_runtime": 7093.1544, + "train_samples": 16746, + "train_samples_per_second": 2.03, + "train_steps_per_second": 0.127 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..418fec7 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,2923 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8599068434252956, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.004777260241251642, + "grad_norm": 24.0, + "learning_rate": 2.222222222222222e-08, + "logits/chosen": -1.997597098350525, + "logits/rejected": -2.0135111808776855, + "logps/chosen": -0.2795145511627197, + "logps/rejected": -0.28208082914352417, + "loss": 0.38181036710739136, + "loss/core": 0.38181036710739136, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.418922185897827, + "rewards/margins": 1.4596340656280518, + "rewards/rejected": 0.9592880010604858, + "step": 5 + }, + { + "epoch": 0.009554520482503284, + "grad_norm": 190.0, + "learning_rate": 5e-08, + "logits/chosen": -2.3454456329345703, + "logits/rejected": -2.392616033554077, + "logps/chosen": -0.2907744348049164, + "logps/rejected": -0.28635361790657043, + "loss": 1.0456370115280151, + "loss/core": 1.0456370115280151, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.6759283542633057, + "rewards/margins": 0.041599858552217484, + "rewards/rejected": 1.6343282461166382, + "step": 10 + }, + { + "epoch": 0.014331780723754926, + "grad_norm": 81.5, + "learning_rate": 7.777777777777778e-08, + "logits/chosen": -2.2278778553009033, + "logits/rejected": -2.1911208629608154, + "logps/chosen": -0.2769435942173004, + "logps/rejected": -0.28479596972465515, + "loss": 0.3574354350566864, + "loss/core": 0.3574354350566864, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.138868570327759, + "rewards/margins": 1.00614595413208, + "rewards/rejected": 1.1327224969863892, + "step": 15 + }, + { + "epoch": 0.01910904096500657, + "grad_norm": 21.75, + "learning_rate": 1.0555555555555555e-07, + "logits/chosen": -2.1759560108184814, + "logits/rejected": -2.218474864959717, + "logps/chosen": -0.300750195980072, + "logps/rejected": -0.2876606285572052, + "loss": 0.4318433701992035, + "loss/core": 0.4318433701992035, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.921849012374878, + "rewards/margins": 1.0531971454620361, + "rewards/rejected": 0.8686518669128418, + "step": 20 + }, + { + "epoch": 0.02388630120625821, + "grad_norm": 25.5, + "learning_rate": 1.3333333333333334e-07, + "logits/chosen": -2.0754263401031494, + "logits/rejected": -2.136157512664795, + "logps/chosen": -0.27347683906555176, + "logps/rejected": -0.2596595585346222, + "loss": 1.702324628829956, + "loss/core": 1.702324628829956, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.820397138595581, + "rewards/margins": 2.1924726963043213, + "rewards/rejected": 1.6279243230819702, + "step": 25 + }, + { + "epoch": 0.028663561447509853, + "grad_norm": 274.0, + "learning_rate": 1.611111111111111e-07, + "logits/chosen": -2.3495566844940186, + "logits/rejected": -2.331704616546631, + "logps/chosen": -0.30199751257896423, + "logps/rejected": -0.3243906497955322, + "loss": 0.6514055132865906, + "loss/core": 0.6514055132865906, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.259319543838501, + "rewards/margins": 1.2338563203811646, + "rewards/rejected": 1.0254631042480469, + "step": 30 + }, + { + "epoch": 0.033440821688761495, + "grad_norm": 2.265625, + "learning_rate": 1.8888888888888888e-07, + "logits/chosen": -2.204205274581909, + "logits/rejected": -2.1856656074523926, + "logps/chosen": -0.2655085027217865, + "logps/rejected": -0.26616171002388, + "loss": 0.814064621925354, + "loss/core": 0.814064621925354, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.6806728839874268, + "rewards/margins": 1.5299947261810303, + "rewards/rejected": 1.1506781578063965, + "step": 35 + }, + { + "epoch": 0.03821808193001314, + "grad_norm": 80.0, + "learning_rate": 2.1666666666666667e-07, + "logits/chosen": -2.0009946823120117, + "logits/rejected": -2.060720682144165, + "logps/chosen": -0.2918882966041565, + "logps/rejected": -0.28243204951286316, + "loss": 1.9676904678344727, + "loss/core": 1.9676904678344727, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.485463619232178, + "rewards/margins": 2.8883402347564697, + "rewards/rejected": 1.5971235036849976, + "step": 40 + }, + { + "epoch": 0.04299534217126478, + "grad_norm": 8.125, + "learning_rate": 2.4444444444444445e-07, + "logits/chosen": -2.197145938873291, + "logits/rejected": -2.359992504119873, + "logps/chosen": -0.28900036215782166, + "logps/rejected": -0.28796130418777466, + "loss": 0.2774832248687744, + "loss/core": 0.2774832248687744, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.0935540199279785, + "rewards/margins": 1.2470275163650513, + "rewards/rejected": 0.8465266227722168, + "step": 45 + }, + { + "epoch": 0.04777260241251642, + "grad_norm": 77.0, + "learning_rate": 2.7222222222222216e-07, + "logits/chosen": -2.166928768157959, + "logits/rejected": -2.1634888648986816, + "logps/chosen": -0.28184059262275696, + "logps/rejected": -0.28433579206466675, + "loss": 0.7104421257972717, + "loss/core": 0.7104421257972717, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.8780951499938965, + "rewards/margins": 1.4217535257339478, + "rewards/rejected": 1.4563415050506592, + "step": 50 + }, + { + "epoch": 0.05254986265376806, + "grad_norm": 0.87109375, + "learning_rate": 3e-07, + "logits/chosen": -2.197507858276367, + "logits/rejected": -2.185612916946411, + "logps/chosen": -0.2892429828643799, + "logps/rejected": -0.2890060245990753, + "loss": 1.496902346611023, + "loss/core": 1.496902346611023, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.2559685707092285, + "rewards/margins": 0.21959257125854492, + "rewards/rejected": 2.0363757610321045, + "step": 55 + }, + { + "epoch": 0.057327122895019705, + "grad_norm": 520.0, + "learning_rate": 3.2777777777777776e-07, + "logits/chosen": -2.09147572517395, + "logits/rejected": -2.1203830242156982, + "logps/chosen": -0.2743294835090637, + "logps/rejected": -0.28498584032058716, + "loss": 0.337458074092865, + "loss/core": 0.337458074092865, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.1575465202331543, + "rewards/margins": 0.6136163473129272, + "rewards/rejected": 1.543930172920227, + "step": 60 + }, + { + "epoch": 0.06210438313627135, + "grad_norm": 7.1875, + "learning_rate": 3.5555555555555553e-07, + "logits/chosen": -1.9565308094024658, + "logits/rejected": -2.059638023376465, + "logps/chosen": -0.3096534311771393, + "logps/rejected": -0.30584970116615295, + "loss": 0.9800235629081726, + "loss/core": 0.9800235629081726, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.9653208255767822, + "rewards/margins": 1.7889091968536377, + "rewards/rejected": 1.176411747932434, + "step": 65 + }, + { + "epoch": 0.06688164337752299, + "grad_norm": 54.25, + "learning_rate": 3.8333333333333335e-07, + "logits/chosen": -1.9700387716293335, + "logits/rejected": -1.955226182937622, + "logps/chosen": -0.3858351707458496, + "logps/rejected": -0.2979809641838074, + "loss": 1.823669195175171, + "loss/core": 1.823669195175171, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.578740358352661, + "rewards/margins": 1.4053298234939575, + "rewards/rejected": 2.1734108924865723, + "step": 70 + }, + { + "epoch": 0.07165890361877464, + "grad_norm": 6.875, + "learning_rate": 4.1111111111111107e-07, + "logits/chosen": -2.0864429473876953, + "logits/rejected": -2.051452398300171, + "logps/chosen": -0.3145933449268341, + "logps/rejected": -0.2891804873943329, + "loss": 0.7372748255729675, + "loss/core": 0.7372748255729675, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.610942840576172, + "rewards/margins": 1.0181684494018555, + "rewards/rejected": 1.5927742719650269, + "step": 75 + }, + { + "epoch": 0.07643616386002627, + "grad_norm": 4.15625, + "learning_rate": 4.3888888888888884e-07, + "logits/chosen": -2.066887855529785, + "logits/rejected": -2.186957836151123, + "logps/chosen": -0.26004475355148315, + "logps/rejected": -0.27356016635894775, + "loss": 0.9191620945930481, + "loss/core": 0.9191620945930481, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.2465484142303467, + "rewards/margins": 1.9899933338165283, + "rewards/rejected": 1.2565548419952393, + "step": 80 + }, + { + "epoch": 0.08121342410127792, + "grad_norm": 5.75, + "learning_rate": 4.6666666666666666e-07, + "logits/chosen": -2.1885199546813965, + "logits/rejected": -2.2368600368499756, + "logps/chosen": -0.30883267521858215, + "logps/rejected": -0.2775411307811737, + "loss": 1.6861064434051514, + "loss/core": 1.6861064434051514, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.484980821609497, + "rewards/margins": 2.152528762817383, + "rewards/rejected": 1.3324521780014038, + "step": 85 + }, + { + "epoch": 0.08599068434252956, + "grad_norm": 0.98828125, + "learning_rate": 4.944444444444445e-07, + "logits/chosen": -2.4196181297302246, + "logits/rejected": -2.4065661430358887, + "logps/chosen": -0.281704306602478, + "logps/rejected": -0.2961405813694, + "loss": 0.30765506625175476, + "loss/core": 0.30765506625175476, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.7503442764282227, + "rewards/margins": 0.9065144658088684, + "rewards/rejected": 0.8438299894332886, + "step": 90 + }, + { + "epoch": 0.09076794458378121, + "grad_norm": 62.5, + "learning_rate": 4.999699149323369e-07, + "logits/chosen": -2.0543789863586426, + "logits/rejected": -2.11065673828125, + "logps/chosen": -0.26749688386917114, + "logps/rejected": -0.2789869010448456, + "loss": 0.906001091003418, + "loss/core": 0.906001091003418, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.2087581157684326, + "rewards/margins": 1.1049243211746216, + "rewards/rejected": 2.1038336753845215, + "step": 95 + }, + { + "epoch": 0.09554520482503284, + "grad_norm": 10.625, + "learning_rate": 4.998477067547739e-07, + "logits/chosen": -2.231830596923828, + "logits/rejected": -2.2774977684020996, + "logps/chosen": -0.25898346304893494, + "logps/rejected": -0.24633264541625977, + "loss": 0.3568304777145386, + "loss/core": 0.3568304777145386, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.444751024246216, + "rewards/margins": 1.1550977230072021, + "rewards/rejected": 1.2896530628204346, + "step": 100 + }, + { + "epoch": 0.10032246506628449, + "grad_norm": 298.0, + "learning_rate": 4.996315410727229e-07, + "logits/chosen": -2.3348228931427, + "logits/rejected": -2.4207358360290527, + "logps/chosen": -0.2856779098510742, + "logps/rejected": -0.279996782541275, + "loss": 0.3823498785495758, + "loss/core": 0.3823498785495758, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.9824638366699219, + "rewards/margins": 1.1864570379257202, + "rewards/rejected": 0.7960068583488464, + "step": 105 + }, + { + "epoch": 0.10509972530753613, + "grad_norm": 11.875, + "learning_rate": 4.993214991772562e-07, + "logits/chosen": -1.9041261672973633, + "logits/rejected": -1.8569024801254272, + "logps/chosen": -0.3389981985092163, + "logps/rejected": -0.29700320959091187, + "loss": 1.7510207891464233, + "loss/core": 1.7510207891464233, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.465295314788818, + "rewards/margins": 2.569420576095581, + "rewards/rejected": 1.8958752155303955, + "step": 110 + }, + { + "epoch": 0.10987698554878778, + "grad_norm": 9.875, + "learning_rate": 4.989176976624511e-07, + "logits/chosen": -2.0836009979248047, + "logits/rejected": -2.027132749557495, + "logps/chosen": -0.2996644973754883, + "logps/rejected": -0.30102577805519104, + "loss": 0.4198575019836426, + "loss/core": 0.4198575019836426, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7827256917953491, + "rewards/margins": 0.3579304814338684, + "rewards/rejected": 1.424795389175415, + "step": 115 + }, + { + "epoch": 0.11465424579003941, + "grad_norm": 184.0, + "learning_rate": 4.984202883815428e-07, + "logits/chosen": -2.0339629650115967, + "logits/rejected": -2.0718021392822266, + "logps/chosen": -0.30192404985427856, + "logps/rejected": -0.29958024621009827, + "loss": 0.16679374873638153, + "loss/core": 0.16679374873638153, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.8123009204864502, + "rewards/margins": 0.7246686220169067, + "rewards/rejected": 1.0876322984695435, + "step": 120 + }, + { + "epoch": 0.11943150603129106, + "grad_norm": 5.5625, + "learning_rate": 4.978294583898195e-07, + "logits/chosen": -2.104386806488037, + "logits/rejected": -2.1359241008758545, + "logps/chosen": -0.2988458275794983, + "logps/rejected": -0.3063589632511139, + "loss": 0.2692359387874603, + "loss/core": 0.2692359387874603, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.067023992538452, + "rewards/margins": 1.047250747680664, + "rewards/rejected": 1.019773244857788, + "step": 125 + }, + { + "epoch": 0.1242087662725427, + "grad_norm": 10.6875, + "learning_rate": 4.971454298742779e-07, + "logits/chosen": -1.994943380355835, + "logits/rejected": -2.0879642963409424, + "logps/chosen": -0.2800437808036804, + "logps/rejected": -0.279203325510025, + "loss": 0.14723244309425354, + "loss/core": 0.14723244309425354, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.806762456893921, + "rewards/margins": 0.9239923357963562, + "rewards/rejected": 0.8827699422836304, + "step": 130 + }, + { + "epoch": 0.12898602651379434, + "grad_norm": 21.75, + "learning_rate": 4.963684600700678e-07, + "logits/chosen": -2.0894951820373535, + "logits/rejected": -2.16270112991333, + "logps/chosen": -0.3019745349884033, + "logps/rejected": -0.30207985639572144, + "loss": 0.8815671801567078, + "loss/core": 0.8815671801567078, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.3753859996795654, + "rewards/margins": 2.099947214126587, + "rewards/rejected": 1.2754385471343994, + "step": 135 + }, + { + "epoch": 0.13376328675504598, + "grad_norm": 11.1875, + "learning_rate": 4.954988411637571e-07, + "logits/chosen": -2.44862699508667, + "logits/rejected": -2.508138656616211, + "logps/chosen": -0.28244370222091675, + "logps/rejected": -0.2900107800960541, + "loss": 0.1148979440331459, + "loss/core": 0.1148979440331459, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6288248300552368, + "rewards/margins": 0.9194561243057251, + "rewards/rejected": 0.7093685865402222, + "step": 140 + }, + { + "epoch": 0.13854054699629761, + "grad_norm": 7.9375, + "learning_rate": 4.945369001834514e-07, + "logits/chosen": -1.7729772329330444, + "logits/rejected": -1.803558111190796, + "logps/chosen": -0.2981434464454651, + "logps/rejected": -0.30744919180870056, + "loss": 2.0826029777526855, + "loss/core": 2.0826029777526855, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.636133909225464, + "rewards/margins": 1.9079220294952393, + "rewards/rejected": 1.7282119989395142, + "step": 145 + }, + { + "epoch": 0.14331780723754928, + "grad_norm": 111.0, + "learning_rate": 4.93482998875813e-07, + "logits/chosen": -2.2263197898864746, + "logits/rejected": -2.274482488632202, + "logps/chosen": -0.31686437129974365, + "logps/rejected": -0.3086201250553131, + "loss": 1.6554409265518188, + "loss/core": 1.6554409265518188, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.644029140472412, + "rewards/margins": 2.1906416416168213, + "rewards/rejected": 1.4533871412277222, + "step": 150 + }, + { + "epoch": 0.1480950674788009, + "grad_norm": 4.875, + "learning_rate": 4.923375335700223e-07, + "logits/chosen": -2.094820022583008, + "logits/rejected": -2.17297101020813, + "logps/chosen": -0.2831822335720062, + "logps/rejected": -0.2986108958721161, + "loss": 0.7216386198997498, + "loss/core": 0.7216386198997498, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.8636791706085205, + "rewards/margins": 2.1000559329986572, + "rewards/rejected": 0.7636229395866394, + "step": 155 + }, + { + "epoch": 0.15287232772005255, + "grad_norm": 272.0, + "learning_rate": 4.911009350287347e-07, + "logits/chosen": -2.0500402450561523, + "logits/rejected": -2.0241525173187256, + "logps/chosen": -0.3109878599643707, + "logps/rejected": -0.3158418536186218, + "loss": 1.3895409107208252, + "loss/core": 1.3895409107208252, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.1455612182617188, + "rewards/margins": 1.7495533227920532, + "rewards/rejected": 1.396007776260376, + "step": 160 + }, + { + "epoch": 0.15764958796130418, + "grad_norm": 5.09375, + "learning_rate": 4.897736682860885e-07, + "logits/chosen": -2.216140031814575, + "logits/rejected": -2.146721363067627, + "logps/chosen": -0.31132304668426514, + "logps/rejected": -0.30651506781578064, + "loss": 0.1788981854915619, + "loss/core": 0.1788981854915619, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.6000747680664062, + "rewards/margins": 0.8192144632339478, + "rewards/rejected": 0.7808603048324585, + "step": 165 + }, + { + "epoch": 0.16242684820255585, + "grad_norm": 402.0, + "learning_rate": 4.883562324728241e-07, + "logits/chosen": -2.18127179145813, + "logits/rejected": -2.126049518585205, + "logps/chosen": -0.2751903235912323, + "logps/rejected": -0.261197030544281, + "loss": 1.8691749572753906, + "loss/core": 1.8691749572753906, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.59234881401062, + "rewards/margins": -0.16324080526828766, + "rewards/rejected": 2.755589485168457, + "step": 170 + }, + { + "epoch": 0.16720410844380748, + "grad_norm": 460.0, + "learning_rate": 4.868491606285823e-07, + "logits/chosen": -2.1704468727111816, + "logits/rejected": -2.2109594345092773, + "logps/chosen": -0.2882474958896637, + "logps/rejected": -0.3017005920410156, + "loss": 1.1066371202468872, + "loss/core": 1.1066371202468872, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.569884777069092, + "rewards/margins": 1.3101545572280884, + "rewards/rejected": 1.259730339050293, + "step": 175 + }, + { + "epoch": 0.17198136868505912, + "grad_norm": 640.0, + "learning_rate": 4.852530195014489e-07, + "logits/chosen": -2.193326711654663, + "logits/rejected": -2.104137897491455, + "logps/chosen": -0.2730237543582916, + "logps/rejected": -0.26204752922058105, + "loss": 1.1543080806732178, + "loss/core": 1.1543080806732178, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.2240982055664062, + "rewards/margins": 1.7351115942001343, + "rewards/rejected": 1.4889863729476929, + "step": 180 + }, + { + "epoch": 0.17675862892631075, + "grad_norm": 3.40625, + "learning_rate": 4.835684093348244e-07, + "logits/chosen": -2.1960487365722656, + "logits/rejected": -2.154365062713623, + "logps/chosen": -0.2888965308666229, + "logps/rejected": -0.29239413142204285, + "loss": 0.06165605038404465, + "loss/core": 0.06165605038404465, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.0407021045684814, + "rewards/margins": 0.41684430837631226, + "rewards/rejected": 0.6238579154014587, + "step": 185 + }, + { + "epoch": 0.18153588916756241, + "grad_norm": 28.875, + "learning_rate": 4.817959636416969e-07, + "logits/chosen": -2.2871928215026855, + "logits/rejected": -2.264662981033325, + "logps/chosen": -0.28313952684402466, + "logps/rejected": -0.28482240438461304, + "loss": 1.1405375003814697, + "loss/core": 1.1405375003814697, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.842782497406006, + "rewards/margins": 1.609337568283081, + "rewards/rejected": 1.2334448099136353, + "step": 190 + }, + { + "epoch": 0.18631314940881405, + "grad_norm": 23.5, + "learning_rate": 4.799363489664039e-07, + "logits/chosen": -1.9776523113250732, + "logits/rejected": -1.999280571937561, + "logps/chosen": -0.289995938539505, + "logps/rejected": -0.2919548749923706, + "loss": 1.1198909282684326, + "loss/core": 1.1198909282684326, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.678499937057495, + "rewards/margins": 1.7345187664031982, + "rewards/rejected": 0.9439811706542969, + "step": 195 + }, + { + "epoch": 0.19109040965006568, + "grad_norm": 20.625, + "learning_rate": 4.779902646339721e-07, + "logits/chosen": -1.88909113407135, + "logits/rejected": -1.9297901391983032, + "logps/chosen": -0.32461681962013245, + "logps/rejected": -0.3156924545764923, + "loss": 0.4924043118953705, + "loss/core": 0.4924043118953705, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.604013681411743, + "rewards/margins": 1.374180555343628, + "rewards/rejected": 1.2298332452774048, + "step": 200 + }, + { + "epoch": 0.19586766989131732, + "grad_norm": 9.875, + "learning_rate": 4.759584424871301e-07, + "logits/chosen": -2.110816478729248, + "logits/rejected": -2.1626930236816406, + "logps/chosen": -0.25076472759246826, + "logps/rejected": -0.2683349549770355, + "loss": 1.2883168458938599, + "loss/core": 1.2883168458938599, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.282876491546631, + "rewards/margins": 1.168548345565796, + "rewards/rejected": 2.114327907562256, + "step": 205 + }, + { + "epoch": 0.20064493013256898, + "grad_norm": 27.875, + "learning_rate": 4.738416466110917e-07, + "logits/chosen": -2.1067051887512207, + "logits/rejected": -2.1317334175109863, + "logps/chosen": -0.3131873607635498, + "logps/rejected": -0.32463163137435913, + "loss": 1.1303998231887817, + "loss/core": 1.1303998231887817, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.8242244720458984, + "rewards/margins": 1.3681774139404297, + "rewards/rejected": 1.4560470581054688, + "step": 210 + }, + { + "epoch": 0.20542219037382062, + "grad_norm": 6.875, + "learning_rate": 4.716406730462153e-07, + "logits/chosen": -2.079124927520752, + "logits/rejected": -2.069756031036377, + "logps/chosen": -0.2739131450653076, + "logps/rejected": -0.2875369191169739, + "loss": 1.5732624530792236, + "loss/core": 1.5732624530792236, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.207057237625122, + "rewards/margins": 1.5373919010162354, + "rewards/rejected": 1.6696653366088867, + "step": 215 + }, + { + "epoch": 0.21019945061507225, + "grad_norm": 30.375, + "learning_rate": 4.693563494886454e-07, + "logits/chosen": -2.0912394523620605, + "logits/rejected": -2.0069117546081543, + "logps/chosen": -0.3027765452861786, + "logps/rejected": -0.30113840103149414, + "loss": 0.21921542286872864, + "loss/core": 0.21921542286872864, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.8828872442245483, + "rewards/margins": 0.8069303631782532, + "rewards/rejected": 1.0759570598602295, + "step": 220 + }, + { + "epoch": 0.2149767108563239, + "grad_norm": 8.25, + "learning_rate": 4.6698953497905016e-07, + "logits/chosen": -2.1752381324768066, + "logits/rejected": -2.1640679836273193, + "logps/chosen": -0.2903914749622345, + "logps/rejected": -0.2762240171432495, + "loss": 0.8437272310256958, + "loss/core": 0.8437272310256958, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.6241302490234375, + "rewards/margins": 2.222364902496338, + "rewards/rejected": 1.4017655849456787, + "step": 225 + }, + { + "epoch": 0.21975397109757555, + "grad_norm": 54.0, + "learning_rate": 4.645411195795709e-07, + "logits/chosen": -2.266545057296753, + "logits/rejected": -2.2732253074645996, + "logps/chosen": -0.2827457785606384, + "logps/rejected": -0.31045886874198914, + "loss": 0.1818530261516571, + "loss/core": 0.1818530261516571, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8441836833953857, + "rewards/margins": 1.116914987564087, + "rewards/rejected": 0.7272688150405884, + "step": 230 + }, + { + "epoch": 0.2245312313388272, + "grad_norm": 21.0, + "learning_rate": 4.6201202403910643e-07, + "logits/chosen": -2.0017685890197754, + "logits/rejected": -2.0373404026031494, + "logps/chosen": -0.29275330901145935, + "logps/rejected": -0.2876564562320709, + "loss": 0.45062655210494995, + "loss/core": 0.45062655210494995, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.9036191701889038, + "rewards/margins": 0.5858906507492065, + "rewards/rejected": 1.3177284002304077, + "step": 235 + }, + { + "epoch": 0.22930849158007882, + "grad_norm": 11.125, + "learning_rate": 4.594031994470573e-07, + "logits/chosen": -2.016340732574463, + "logits/rejected": -2.035101890563965, + "logps/chosen": -0.29441291093826294, + "logps/rejected": -0.29516950249671936, + "loss": 0.14202021062374115, + "loss/core": 0.14202021062374115, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7591701745986938, + "rewards/margins": 0.7430254220962524, + "rewards/rejected": 1.0161445140838623, + "step": 240 + }, + { + "epoch": 0.23408575182133046, + "grad_norm": 300.0, + "learning_rate": 4.567156268756593e-07, + "logits/chosen": -2.0561702251434326, + "logits/rejected": -2.043266773223877, + "logps/chosen": -0.2777637839317322, + "logps/rejected": -0.2947208285331726, + "loss": 1.193971037864685, + "loss/core": 1.193971037864685, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.6019701957702637, + "rewards/margins": 2.486217975616455, + "rewards/rejected": 1.1157519817352295, + "step": 245 + }, + { + "epoch": 0.23886301206258212, + "grad_norm": 21.5, + "learning_rate": 4.5395031701104303e-07, + "logits/chosen": -1.8958946466445923, + "logits/rejected": -1.9753758907318115, + "logps/chosen": -0.30067577958106995, + "logps/rejected": -0.295326292514801, + "loss": 0.28905177116394043, + "loss/core": 0.28905177116394043, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.3306000232696533, + "rewards/margins": 1.2696471214294434, + "rewards/rejected": 1.0609527826309204, + "step": 250 + }, + { + "epoch": 0.24364027230383375, + "grad_norm": 12.875, + "learning_rate": 4.511083097731555e-07, + "logits/chosen": -2.2028048038482666, + "logits/rejected": -2.1979146003723145, + "logps/chosen": -0.2854974865913391, + "logps/rejected": -0.2871183156967163, + "loss": 0.6217617392539978, + "loss/core": 0.6217617392539978, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.4457859992980957, + "rewards/margins": 0.8392683863639832, + "rewards/rejected": 1.6065175533294678, + "step": 255 + }, + { + "epoch": 0.2484175325450854, + "grad_norm": 0.78515625, + "learning_rate": 4.481906739246894e-07, + "logits/chosen": -2.1431796550750732, + "logits/rejected": -2.214958906173706, + "logps/chosen": -0.28957638144493103, + "logps/rejected": -0.27189141511917114, + "loss": 0.5298699736595154, + "loss/core": 0.5298699736595154, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.3812990188598633, + "rewards/margins": 0.8128549456596375, + "rewards/rejected": 1.568444013595581, + "step": 260 + }, + { + "epoch": 0.25319479278633705, + "grad_norm": 1.1796875, + "learning_rate": 4.451985066691648e-07, + "logits/chosen": -2.025351047515869, + "logits/rejected": -2.0318732261657715, + "logps/chosen": -0.3007538318634033, + "logps/rejected": -0.305298388004303, + "loss": 0.405862033367157, + "loss/core": 0.405862033367157, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.2176848649978638, + "rewards/margins": 0.080999456346035, + "rewards/rejected": 1.1366853713989258, + "step": 265 + }, + { + "epoch": 0.2579720530275887, + "grad_norm": 12.4375, + "learning_rate": 4.421329332383158e-07, + "logits/chosen": -1.928206205368042, + "logits/rejected": -2.016925096511841, + "logps/chosen": -0.2899303734302521, + "logps/rejected": -0.29195359349250793, + "loss": 1.1587004661560059, + "loss/core": 1.1587004661560059, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.2669448852539062, + "rewards/margins": 1.1529427766799927, + "rewards/rejected": 2.114002227783203, + "step": 270 + }, + { + "epoch": 0.2627493132688403, + "grad_norm": 68.0, + "learning_rate": 4.3899510646893696e-07, + "logits/chosen": -2.078123092651367, + "logits/rejected": -2.032747745513916, + "logps/chosen": -0.2979772388935089, + "logps/rejected": -0.3040463328361511, + "loss": 0.45543140172958374, + "loss/core": 0.45543140172958374, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.310035467147827, + "rewards/margins": 1.1866633892059326, + "rewards/rejected": 1.123372197151184, + "step": 275 + }, + { + "epoch": 0.26752657351009196, + "grad_norm": 4.03125, + "learning_rate": 4.357862063693485e-07, + "logits/chosen": -2.0485012531280518, + "logits/rejected": -1.9654176235198975, + "logps/chosen": -0.28849276900291443, + "logps/rejected": -0.3105738162994385, + "loss": 0.21645864844322205, + "loss/core": 0.21645864844322205, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.0423591136932373, + "rewards/margins": 1.3129526376724243, + "rewards/rejected": 0.7294065952301025, + "step": 280 + }, + { + "epoch": 0.2723038337513436, + "grad_norm": 244.0, + "learning_rate": 4.3250743967564364e-07, + "logits/chosen": -2.072444438934326, + "logits/rejected": -2.069185733795166, + "logps/chosen": -0.29705238342285156, + "logps/rejected": -0.29413706064224243, + "loss": 0.5292507410049438, + "loss/core": 0.5292507410049438, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.6063647270202637, + "rewards/margins": 0.9912127256393433, + "rewards/rejected": 1.6151517629623413, + "step": 285 + }, + { + "epoch": 0.27708109399259523, + "grad_norm": 668.0, + "learning_rate": 4.29160039397884e-07, + "logits/chosen": -2.148799419403076, + "logits/rejected": -2.1607887744903564, + "logps/chosen": -0.2877974808216095, + "logps/rejected": -0.2921397387981415, + "loss": 0.3775253891944885, + "loss/core": 0.3775253891944885, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.4691474437713623, + "rewards/margins": 0.05996546894311905, + "rewards/rejected": 1.4091819524765015, + "step": 290 + }, + { + "epoch": 0.28185835423384686, + "grad_norm": 8.125, + "learning_rate": 4.2574526435641546e-07, + "logits/chosen": -2.5482234954833984, + "logits/rejected": -2.5131990909576416, + "logps/chosen": -0.24539580941200256, + "logps/rejected": -0.2573974132537842, + "loss": 0.05928546190261841, + "loss/core": 0.05928546190261841, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.3013224601745605, + "rewards/margins": 0.635219931602478, + "rewards/rejected": 0.6661025285720825, + "step": 295 + }, + { + "epoch": 0.28663561447509855, + "grad_norm": 142.0, + "learning_rate": 4.22264398708477e-07, + "logits/chosen": -1.9641075134277344, + "logits/rejected": -1.9324430227279663, + "logps/chosen": -0.3132562041282654, + "logps/rejected": -0.3241512179374695, + "loss": 0.9757488369941711, + "loss/core": 0.9757488369941711, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.715285539627075, + "rewards/margins": 2.727508306503296, + "rewards/rejected": 0.9877773523330688, + "step": 300 + }, + { + "epoch": 0.2914128747163502, + "grad_norm": 12.0, + "learning_rate": 4.187187514652819e-07, + "logits/chosen": -2.102832794189453, + "logits/rejected": -2.1649816036224365, + "logps/chosen": -0.269456148147583, + "logps/rejected": -0.26115578413009644, + "loss": 0.22697356343269348, + "loss/core": 0.22697356343269348, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.1590380668640137, + "rewards/margins": 1.2657159566879272, + "rewards/rejected": 0.8933222889900208, + "step": 305 + }, + { + "epoch": 0.2961901349576018, + "grad_norm": 132.0, + "learning_rate": 4.151096559997519e-07, + "logits/chosen": -2.111849308013916, + "logits/rejected": -2.0225253105163574, + "logps/chosen": -0.2739498019218445, + "logps/rejected": -0.28690317273139954, + "loss": 0.9037171602249146, + "loss/core": 0.9037171602249146, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.6031503677368164, + "rewards/margins": 0.6503016352653503, + "rewards/rejected": 1.9528486728668213, + "step": 310 + }, + { + "epoch": 0.30096739519885346, + "grad_norm": 5.84375, + "learning_rate": 4.114384695450905e-07, + "logits/chosen": -1.9431352615356445, + "logits/rejected": -2.0480642318725586, + "logps/chosen": -0.2702833116054535, + "logps/rejected": -0.27680981159210205, + "loss": 0.45981135964393616, + "loss/core": 0.45981135964393616, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.901813268661499, + "rewards/margins": 1.5673255920410156, + "rewards/rejected": 1.3344876766204834, + "step": 315 + }, + { + "epoch": 0.3057446554401051, + "grad_norm": 3.578125, + "learning_rate": 4.077065726843828e-07, + "logits/chosen": -1.9837052822113037, + "logits/rejected": -2.000582456588745, + "logps/chosen": -0.2918964922428131, + "logps/rejected": -0.29292041063308716, + "loss": 0.19344636797904968, + "loss/core": 0.19344636797904968, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.5632867813110352, + "rewards/margins": 0.3376685380935669, + "rewards/rejected": 1.2256181240081787, + "step": 320 + }, + { + "epoch": 0.31052191568135673, + "grad_norm": 556.0, + "learning_rate": 4.039153688314145e-07, + "logits/chosen": -2.2357137203216553, + "logits/rejected": -2.1825029850006104, + "logps/chosen": -0.28656476736068726, + "logps/rejected": -0.28232595324516296, + "loss": 1.8139476776123047, + "loss/core": 1.8139476776123047, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.6962344646453857, + "rewards/margins": 2.1483452320098877, + "rewards/rejected": 1.5478893518447876, + "step": 325 + }, + { + "epoch": 0.31529917592260837, + "grad_norm": 2.640625, + "learning_rate": 4.0006628370290613e-07, + "logits/chosen": -2.2076706886291504, + "logits/rejected": -2.196662664413452, + "logps/chosen": -0.282164603471756, + "logps/rejected": -0.27122125029563904, + "loss": 0.2774432897567749, + "loss/core": 0.2774432897567749, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.995111107826233, + "rewards/margins": 0.8708856701850891, + "rewards/rejected": 1.12422513961792, + "step": 330 + }, + { + "epoch": 0.32007643616386, + "grad_norm": 344.0, + "learning_rate": 3.9616076478235826e-07, + "logits/chosen": -2.0309836864471436, + "logits/rejected": -2.014965057373047, + "logps/chosen": -0.30268004536628723, + "logps/rejected": -0.32926321029663086, + "loss": 0.4523457884788513, + "loss/core": 0.4523457884788513, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.116103410720825, + "rewards/margins": 1.5347257852554321, + "rewards/rejected": 0.5813775658607483, + "step": 335 + }, + { + "epoch": 0.3248536964051117, + "grad_norm": 132.0, + "learning_rate": 3.922002807757129e-07, + "logits/chosen": -2.0748965740203857, + "logits/rejected": -2.196530342102051, + "logps/chosen": -0.28026294708251953, + "logps/rejected": -0.2833474576473236, + "loss": 0.31017398834228516, + "loss/core": 0.31017398834228516, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.0382297039031982, + "rewards/margins": 1.1847279071807861, + "rewards/rejected": 0.8535016775131226, + "step": 340 + }, + { + "epoch": 0.3296309566463633, + "grad_norm": 8.625, + "learning_rate": 3.8818632105903315e-07, + "logits/chosen": -2.299494981765747, + "logits/rejected": -2.3579187393188477, + "logps/chosen": -0.28255495429039, + "logps/rejected": -0.2946614921092987, + "loss": 0.1961742490530014, + "loss/core": 0.1961742490530014, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.639917016029358, + "rewards/margins": 0.8168700933456421, + "rewards/rejected": 0.823046863079071, + "step": 345 + }, + { + "epoch": 0.33440821688761496, + "grad_norm": 7.625, + "learning_rate": 3.841203951184094e-07, + "logits/chosen": -2.0392203330993652, + "logits/rejected": -2.0187597274780273, + "logps/chosen": -0.261064350605011, + "logps/rejected": -0.25779426097869873, + "loss": 0.20222067832946777, + "loss/core": 0.20222067832946777, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.9913139343261719, + "rewards/margins": 1.0575062036514282, + "rewards/rejected": 0.9338076710700989, + "step": 350 + }, + { + "epoch": 0.3391854771288666, + "grad_norm": 9.625, + "learning_rate": 3.800040319823038e-07, + "logits/chosen": -2.088989019393921, + "logits/rejected": -2.2161617279052734, + "logps/chosen": -0.2810487449169159, + "logps/rejected": -0.2965686023235321, + "loss": 0.40663355588912964, + "loss/core": 0.40663355588912964, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8474032878875732, + "rewards/margins": 0.5359752774238586, + "rewards/rejected": 1.3114280700683594, + "step": 355 + }, + { + "epoch": 0.34396273737011823, + "grad_norm": 149.0, + "learning_rate": 3.75838779646545e-07, + "logits/chosen": -2.110039234161377, + "logits/rejected": -2.1716244220733643, + "logps/chosen": -0.2803703844547272, + "logps/rejected": -0.2838951647281647, + "loss": 0.621938169002533, + "loss/core": 0.621938169002533, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.6312050819396973, + "rewards/margins": 1.3762609958648682, + "rewards/rejected": 1.2549439668655396, + "step": 360 + }, + { + "epoch": 0.34873999761136987, + "grad_norm": 474.0, + "learning_rate": 3.7162620449218993e-07, + "logits/chosen": -2.048468589782715, + "logits/rejected": -2.1561145782470703, + "logps/chosen": -0.287244588136673, + "logps/rejected": -0.28558677434921265, + "loss": 0.9522292017936707, + "loss/core": 0.9522292017936707, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.8516860008239746, + "rewards/margins": 2.0592613220214844, + "rewards/rejected": 0.7924247980117798, + "step": 365 + }, + { + "epoch": 0.3535172578526215, + "grad_norm": 21.25, + "learning_rate": 3.673678906964727e-07, + "logits/chosen": -2.246171474456787, + "logits/rejected": -2.291537046432495, + "logps/chosen": -0.28519970178604126, + "logps/rejected": -0.2871125638484955, + "loss": 0.17473874986171722, + "loss/core": 0.17473874986171722, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.6407058238983154, + "rewards/margins": 0.5970925688743591, + "rewards/rejected": 1.0436131954193115, + "step": 370 + }, + { + "epoch": 0.35829451809387314, + "grad_norm": 7.9375, + "learning_rate": 3.6306543963705936e-07, + "logits/chosen": -2.027927875518799, + "logits/rejected": -2.0061910152435303, + "logps/chosen": -0.2594534456729889, + "logps/rejected": -0.2606300711631775, + "loss": 0.8009087443351746, + "loss/core": 0.8009087443351746, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.0067553520202637, + "rewards/margins": 0.8116471171379089, + "rewards/rejected": 2.195107936859131, + "step": 375 + }, + { + "epoch": 0.36307177833512483, + "grad_norm": 13.1875, + "learning_rate": 3.5872046928983623e-07, + "logits/chosen": -2.099975109100342, + "logits/rejected": -2.1087899208068848, + "logps/chosen": -0.2818046808242798, + "logps/rejected": -0.29659533500671387, + "loss": 0.7302302122116089, + "loss/core": 0.7302302122116089, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.033167839050293, + "rewards/margins": 0.5039272904396057, + "rewards/rejected": 1.529240608215332, + "step": 380 + }, + { + "epoch": 0.36784903857637646, + "grad_norm": 29.375, + "learning_rate": 3.5433461362045447e-07, + "logits/chosen": -2.0900938510894775, + "logits/rejected": -2.1108756065368652, + "logps/chosen": -0.2719820439815521, + "logps/rejected": -0.2751936912536621, + "loss": 0.3768286406993866, + "loss/core": 0.3768286406993866, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.160437822341919, + "rewards/margins": 0.6413952112197876, + "rewards/rejected": 1.5190424919128418, + "step": 385 + }, + { + "epoch": 0.3726262988176281, + "grad_norm": 1.953125, + "learning_rate": 3.4990952196986305e-07, + "logits/chosen": -1.8510644435882568, + "logits/rejected": -1.9279950857162476, + "logps/chosen": -0.3081986606121063, + "logps/rejected": -0.30626800656318665, + "loss": 0.1976659595966339, + "loss/core": 0.1976659595966339, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.7517478466033936, + "rewards/margins": 0.8936412930488586, + "rewards/rejected": 0.8581066131591797, + "step": 390 + }, + { + "epoch": 0.37740355905887973, + "grad_norm": 19.75, + "learning_rate": 3.4544685843405875e-07, + "logits/chosen": -2.1983537673950195, + "logits/rejected": -2.3012776374816895, + "logps/chosen": -0.2957797050476074, + "logps/rejected": -0.2903483510017395, + "loss": 0.1597086489200592, + "loss/core": 0.1597086489200592, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 1.6743053197860718, + "rewards/margins": 0.8747150301933289, + "rewards/rejected": 0.7995903491973877, + "step": 395 + }, + { + "epoch": 0.38218081930013137, + "grad_norm": 50.5, + "learning_rate": 3.4094830123828786e-07, + "logits/chosen": -2.0785083770751953, + "logits/rejected": -2.1138789653778076, + "logps/chosen": -0.2923468351364136, + "logps/rejected": -0.29322344064712524, + "loss": 1.0465948581695557, + "loss/core": 1.0465948581695557, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.9298787117004395, + "rewards/margins": 2.0894033908843994, + "rewards/rejected": 0.8404749631881714, + "step": 400 + }, + { + "epoch": 0.386958079541383, + "grad_norm": 5.90625, + "learning_rate": 3.3641554210593414e-07, + "logits/chosen": -2.1093640327453613, + "logits/rejected": -2.1642727851867676, + "logps/chosen": -0.31529492139816284, + "logps/rejected": -0.30353349447250366, + "loss": 0.6650071740150452, + "loss/core": 0.6650071740150452, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.3120980262756348, + "rewards/margins": 1.074296236038208, + "rewards/rejected": 1.2378015518188477, + "step": 405 + }, + { + "epoch": 0.39173533978263464, + "grad_norm": 20.125, + "learning_rate": 3.3185028562233107e-07, + "logits/chosen": -2.227759838104248, + "logits/rejected": -2.2588248252868652, + "logps/chosen": -0.277637243270874, + "logps/rejected": -0.2886066436767578, + "loss": 0.27694252133369446, + "loss/core": 0.27694252133369446, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.081727981567383, + "rewards/margins": 0.8987264633178711, + "rewards/rejected": 1.1830016374588013, + "step": 410 + }, + { + "epoch": 0.3965126000238863, + "grad_norm": 12.625, + "learning_rate": 3.272542485937368e-07, + "logits/chosen": -2.0978333950042725, + "logits/rejected": -2.175910234451294, + "logps/chosen": -0.3309980034828186, + "logps/rejected": -0.32166236639022827, + "loss": 0.05876865237951279, + "loss/core": 0.05876865237951279, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 0.988106906414032, + "rewards/margins": 0.6192494630813599, + "rewards/rejected": 0.36885741353034973, + "step": 415 + }, + { + "epoch": 0.40128986026513797, + "grad_norm": 88.5, + "learning_rate": 3.226291594017137e-07, + "logits/chosen": -2.326503038406372, + "logits/rejected": -2.2930359840393066, + "logps/chosen": -0.28376084566116333, + "logps/rejected": -0.3000229001045227, + "loss": 0.3358566462993622, + "loss/core": 0.3358566462993622, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.0413100719451904, + "rewards/margins": 1.3324414491653442, + "rewards/rejected": 0.7088686227798462, + "step": 420 + }, + { + "epoch": 0.4060671205063896, + "grad_norm": 196.0, + "learning_rate": 3.1797675735315454e-07, + "logits/chosen": -1.9907684326171875, + "logits/rejected": -2.05031156539917, + "logps/chosen": -0.2916226387023926, + "logps/rejected": -0.2697221636772156, + "loss": 0.768527626991272, + "loss/core": 0.768527626991272, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.2350106239318848, + "rewards/margins": 0.6351408958435059, + "rewards/rejected": 1.5998692512512207, + "step": 425 + }, + { + "epoch": 0.41084438074764124, + "grad_norm": 732.0, + "learning_rate": 3.1329879202620047e-07, + "logits/chosen": -2.0472471714019775, + "logits/rejected": -2.0714943408966064, + "logps/chosen": -0.28120359778404236, + "logps/rejected": -0.2685726583003998, + "loss": 1.8000812530517578, + "loss/core": 1.8000812530517578, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 4.386286735534668, + "rewards/margins": 2.987924814224243, + "rewards/rejected": 1.3983619213104248, + "step": 430 + }, + { + "epoch": 0.41562164098889287, + "grad_norm": 8.8125, + "learning_rate": 3.0859702261229613e-07, + "logits/chosen": -2.324946165084839, + "logits/rejected": -2.3079473972320557, + "logps/chosen": -0.27902793884277344, + "logps/rejected": -0.29347580671310425, + "loss": 0.2374797761440277, + "loss/core": 0.2374797761440277, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.617841362953186, + "rewards/margins": 0.6150254011154175, + "rewards/rejected": 1.0028159618377686, + "step": 435 + }, + { + "epoch": 0.4203989012301445, + "grad_norm": 484.0, + "learning_rate": 3.0387321725463e-07, + "logits/chosen": -2.0346179008483887, + "logits/rejected": -2.105949878692627, + "logps/chosen": -0.3050372004508972, + "logps/rejected": -0.304879367351532, + "loss": 0.9780192375183105, + "loss/core": 0.9780192375183105, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.988939046859741, + "rewards/margins": 1.6511949300765991, + "rewards/rejected": 1.3377439975738525, + "step": 440 + }, + { + "epoch": 0.42517616147139614, + "grad_norm": 41.0, + "learning_rate": 2.991291523832075e-07, + "logits/chosen": -1.9136848449707031, + "logits/rejected": -1.8891462087631226, + "logps/chosen": -0.29649606347084045, + "logps/rejected": -0.2987465262413025, + "loss": 0.5171946287155151, + "loss/core": 0.5171946287155151, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.9516830444335938, + "rewards/margins": 1.4048423767089844, + "rewards/rejected": 1.5468409061431885, + "step": 445 + }, + { + "epoch": 0.4299534217126478, + "grad_norm": 418.0, + "learning_rate": 2.943666120468088e-07, + "logits/chosen": -1.9518722295761108, + "logits/rejected": -1.9711889028549194, + "logps/chosen": -0.3000376522541046, + "logps/rejected": -0.29597753286361694, + "loss": 0.8665231466293335, + "loss/core": 0.8665231466293335, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.440600872039795, + "rewards/margins": 1.8995399475097656, + "rewards/rejected": 1.5410608053207397, + "step": 450 + }, + { + "epoch": 0.4347306819538994, + "grad_norm": 183.0, + "learning_rate": 2.8958738724208073e-07, + "logits/chosen": -1.9178097248077393, + "logits/rejected": -2.1076836585998535, + "logps/chosen": -0.3262283205986023, + "logps/rejected": -0.3135696053504944, + "loss": 1.426609754562378, + "loss/core": 1.426609754562378, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.4305222034454346, + "rewards/margins": 1.9480667114257812, + "rewards/rejected": 1.4824554920196533, + "step": 455 + }, + { + "epoch": 0.4395079421951511, + "grad_norm": 30.75, + "learning_rate": 2.8479327524001633e-07, + "logits/chosen": -2.036893606185913, + "logits/rejected": -2.0251917839050293, + "logps/chosen": -0.2996273636817932, + "logps/rejected": -0.30763858556747437, + "loss": 1.1361322402954102, + "loss/core": 1.1361322402954102, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.628493547439575, + "rewards/margins": 1.7698605060577393, + "rewards/rejected": 1.858633041381836, + "step": 460 + }, + { + "epoch": 0.44428520243640274, + "grad_norm": 10.3125, + "learning_rate": 2.7998607891007493e-07, + "logits/chosen": -1.8706251382827759, + "logits/rejected": -1.9697706699371338, + "logps/chosen": -0.2975318431854248, + "logps/rejected": -0.2920708954334259, + "loss": 0.23389527201652527, + "loss/core": 0.23389527201652527, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.297299861907959, + "rewards/margins": 1.371745228767395, + "rewards/rejected": 0.9255548715591431, + "step": 465 + }, + { + "epoch": 0.4490624626776544, + "grad_norm": 1048.0, + "learning_rate": 2.7516760604219616e-07, + "logits/chosen": -2.105297803878784, + "logits/rejected": -2.173494815826416, + "logps/chosen": -0.28127214312553406, + "logps/rejected": -0.2611326277256012, + "loss": 1.1431621313095093, + "loss/core": 1.1431621313095093, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.987865447998047, + "rewards/margins": 1.8296188116073608, + "rewards/rejected": 1.1582467555999756, + "step": 470 + }, + { + "epoch": 0.453839722918906, + "grad_norm": 90.5, + "learning_rate": 2.703396686669646e-07, + "logits/chosen": -2.0695085525512695, + "logits/rejected": -2.0106379985809326, + "logps/chosen": -0.2961866855621338, + "logps/rejected": -0.32303735613822937, + "loss": 0.46794575452804565, + "loss/core": 0.46794575452804565, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.823664665222168, + "rewards/margins": 0.37910977005958557, + "rewards/rejected": 1.4445549249649048, + "step": 475 + }, + { + "epoch": 0.45861698316015764, + "grad_norm": 6.0, + "learning_rate": 2.6550408237417884e-07, + "logits/chosen": -2.2378296852111816, + "logits/rejected": -2.2934558391571045, + "logps/chosen": -0.2757987380027771, + "logps/rejected": -0.2735208570957184, + "loss": 0.5451094508171082, + "loss/core": 0.5451094508171082, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.354505777359009, + "rewards/margins": 1.340338945388794, + "rewards/rejected": 1.0141668319702148, + "step": 480 + }, + { + "epoch": 0.4633942434014093, + "grad_norm": 13.25, + "learning_rate": 2.6066266563008265e-07, + "logits/chosen": -2.183929920196533, + "logits/rejected": -2.191530704498291, + "logps/chosen": -0.2924697995185852, + "logps/rejected": -0.29630663990974426, + "loss": 0.15576469898223877, + "loss/core": 0.15576469898223877, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.7340059280395508, + "rewards/margins": 0.8523953557014465, + "rewards/rejected": 0.8816105127334595, + "step": 485 + }, + { + "epoch": 0.4681715036426609, + "grad_norm": 20.0, + "learning_rate": 2.5581723909351404e-07, + "logits/chosen": -2.13179087638855, + "logits/rejected": -2.1726982593536377, + "logps/chosen": -0.3071516156196594, + "logps/rejected": -0.29533272981643677, + "loss": 0.29640907049179077, + "loss/core": 0.29640907049179077, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.0775198936462402, + "rewards/margins": 1.033736228942871, + "rewards/rejected": 1.0437836647033691, + "step": 490 + }, + { + "epoch": 0.47294876388391255, + "grad_norm": 26.125, + "learning_rate": 2.509696249312301e-07, + "logits/chosen": -2.2027294635772705, + "logits/rejected": -2.2974157333374023, + "logps/chosen": -0.2999098598957062, + "logps/rejected": -0.2914625406265259, + "loss": 0.7030426263809204, + "loss/core": 0.7030426263809204, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.590477228164673, + "rewards/margins": 1.704890251159668, + "rewards/rejected": 0.8855869174003601, + "step": 495 + }, + { + "epoch": 0.47772602412516424, + "grad_norm": 133.0, + "learning_rate": 2.461216461326642e-07, + "logits/chosen": -1.9994663000106812, + "logits/rejected": -1.9483753442764282, + "logps/chosen": -0.28346696496009827, + "logps/rejected": -0.30574333667755127, + "loss": 2.2656641006469727, + "loss/core": 2.2656641006469727, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.139856815338135, + "rewards/margins": 2.1449146270751953, + "rewards/rejected": 1.994942307472229, + "step": 500 + }, + { + "epoch": 0.4825032843664159, + "grad_norm": 348.0, + "learning_rate": 2.412751258243748e-07, + "logits/chosen": -2.1276748180389404, + "logits/rejected": -2.1600797176361084, + "logps/chosen": -0.2883078157901764, + "logps/rejected": -0.29086965322494507, + "loss": 1.4473992586135864, + "loss/core": 1.4473992586135864, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.5834662914276123, + "rewards/margins": 1.8612127304077148, + "rewards/rejected": 1.7222535610198975, + "step": 505 + }, + { + "epoch": 0.4872805446076675, + "grad_norm": 0.2890625, + "learning_rate": 2.3643188658444158e-07, + "logits/chosen": -2.127025604248047, + "logits/rejected": -2.259706735610962, + "logps/chosen": -0.31721481680870056, + "logps/rejected": -0.29182881116867065, + "loss": 0.41495251655578613, + "loss/core": 0.41495251655578613, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9977633953094482, + "rewards/margins": 1.2025401592254639, + "rewards/rejected": 0.7952233552932739, + "step": 510 + }, + { + "epoch": 0.49205780484891914, + "grad_norm": 35.75, + "learning_rate": 2.315937497570688e-07, + "logits/chosen": -2.010084629058838, + "logits/rejected": -2.0125226974487305, + "logps/chosen": -0.2837277054786682, + "logps/rejected": -0.3070703148841858, + "loss": 0.43133026361465454, + "loss/core": 0.43133026361465454, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.5341956615448, + "rewards/margins": 0.8866618275642395, + "rewards/rejected": 1.6475337743759155, + "step": 515 + }, + { + "epoch": 0.4968350650901708, + "grad_norm": 40.5, + "learning_rate": 2.2676253476765194e-07, + "logits/chosen": -2.039220094680786, + "logits/rejected": -2.2076566219329834, + "logps/chosen": -0.32395556569099426, + "logps/rejected": -0.33448874950408936, + "loss": 0.802078366279602, + "loss/core": 0.802078366279602, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.054922580718994, + "rewards/margins": 2.0955018997192383, + "rewards/rejected": 0.9594209790229797, + "step": 520 + }, + { + "epoch": 0.5016123253314224, + "grad_norm": 1288.0, + "learning_rate": 2.2194005843856633e-07, + "logits/chosen": -2.1711580753326416, + "logits/rejected": -2.1403403282165527, + "logps/chosen": -0.3041841983795166, + "logps/rejected": -0.3024327754974365, + "loss": 1.989890456199646, + "loss/core": 1.989890456199646, + "rewards/accuracies": 0.875, + "rewards/chosen": 4.269538402557373, + "rewards/margins": 2.8283638954162598, + "rewards/rejected": 1.441174864768982, + "step": 525 + }, + { + "epoch": 0.5063895855726741, + "grad_norm": 0.408203125, + "learning_rate": 2.1712813430593434e-07, + "logits/chosen": -2.016860246658325, + "logits/rejected": -1.9806495904922485, + "logps/chosen": -0.2869611382484436, + "logps/rejected": -0.2894926369190216, + "loss": 0.7920265793800354, + "loss/core": 0.7920265793800354, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.041126251220703, + "rewards/margins": 1.5607634782791138, + "rewards/rejected": 1.4803630113601685, + "step": 530 + }, + { + "epoch": 0.5111668458139257, + "grad_norm": 644.0, + "learning_rate": 2.123285719376292e-07, + "logits/chosen": -2.1647980213165283, + "logits/rejected": -2.086789846420288, + "logps/chosen": -0.2868487536907196, + "logps/rejected": -0.28134408593177795, + "loss": 0.9000055193901062, + "loss/core": 0.9000055193901062, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.0810141563415527, + "rewards/margins": 2.0247347354888916, + "rewards/rejected": 1.056279182434082, + "step": 535 + }, + { + "epoch": 0.5159441060551774, + "grad_norm": 1.453125, + "learning_rate": 2.0754317625276982e-07, + "logits/chosen": -2.085092067718506, + "logits/rejected": -2.064635992050171, + "logps/chosen": -0.2793554961681366, + "logps/rejected": -0.2893218398094177, + "loss": 0.3135538399219513, + "loss/core": 0.3135538399219513, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.5822449922561646, + "rewards/margins": 0.09157203137874603, + "rewards/rejected": 1.4906729459762573, + "step": 540 + }, + { + "epoch": 0.520721366296429, + "grad_norm": 1128.0, + "learning_rate": 2.0277374684296498e-07, + "logits/chosen": -2.306612730026245, + "logits/rejected": -2.1498613357543945, + "logps/chosen": -0.2859051823616028, + "logps/rejected": -0.29374048113822937, + "loss": 0.9199239611625671, + "loss/core": 0.9199239611625671, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.6144795417785645, + "rewards/margins": 1.7432453632354736, + "rewards/rejected": 0.8712340593338013, + "step": 545 + }, + { + "epoch": 0.5254986265376806, + "grad_norm": 12.8125, + "learning_rate": 1.980220772955602e-07, + "logits/chosen": -1.8739219903945923, + "logits/rejected": -1.8925116062164307, + "logps/chosen": -0.27485448122024536, + "logps/rejected": -0.3017066717147827, + "loss": 0.7916240692138672, + "loss/core": 0.7916240692138672, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.78112530708313, + "rewards/margins": 1.196991205215454, + "rewards/rejected": 1.5841339826583862, + "step": 550 + }, + { + "epoch": 0.5302758867789323, + "grad_norm": 5.34375, + "learning_rate": 1.932899545191433e-07, + "logits/chosen": -2.0721099376678467, + "logits/rejected": -2.1279807090759277, + "logps/chosen": -0.311161607503891, + "logps/rejected": -0.27461129426956177, + "loss": 0.827431857585907, + "loss/core": 0.827431857585907, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.3001389503479004, + "rewards/margins": 2.3012726306915283, + "rewards/rejected": 0.998866081237793, + "step": 555 + }, + { + "epoch": 0.5350531470201839, + "grad_norm": 54.25, + "learning_rate": 1.885791580715609e-07, + "logits/chosen": -2.157796859741211, + "logits/rejected": -2.144073009490967, + "logps/chosen": -0.27682000398635864, + "logps/rejected": -0.28132444620132446, + "loss": 0.6787029504776001, + "loss/core": 0.6787029504776001, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.8043270111083984, + "rewards/margins": 1.2791929244995117, + "rewards/rejected": 1.5251343250274658, + "step": 560 + }, + { + "epoch": 0.5398304072614356, + "grad_norm": 21.0, + "learning_rate": 1.8389145949069951e-07, + "logits/chosen": -2.05513596534729, + "logits/rejected": -2.0813965797424316, + "logps/chosen": -0.28800255060195923, + "logps/rejected": -0.3045352101325989, + "loss": 1.3088706731796265, + "loss/core": 1.3088706731796265, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.775949478149414, + "rewards/margins": 1.1087366342544556, + "rewards/rejected": 1.6672130823135376, + "step": 565 + }, + { + "epoch": 0.5446076675026872, + "grad_norm": 66.5, + "learning_rate": 1.792286216282824e-07, + "logits/chosen": -2.0531630516052246, + "logits/rejected": -2.1032092571258545, + "logps/chosen": -0.2883487045764923, + "logps/rejected": -0.29088732600212097, + "loss": 0.902314305305481, + "loss/core": 0.902314305305481, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.3471977710723877, + "rewards/margins": 1.6828256845474243, + "rewards/rejected": 1.6643718481063843, + "step": 570 + }, + { + "epoch": 0.5493849277439389, + "grad_norm": 8.6875, + "learning_rate": 1.745923979869336e-07, + "logits/chosen": -2.1334240436553955, + "logits/rejected": -2.164499044418335, + "logps/chosen": -0.3011549413204193, + "logps/rejected": -0.32307350635528564, + "loss": 0.3328213393688202, + "loss/core": 0.3328213393688202, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.8378055095672607, + "rewards/margins": 1.2759101390838623, + "rewards/rejected": 0.5618952512741089, + "step": 575 + }, + { + "epoch": 0.5541621879851905, + "grad_norm": 0.490234375, + "learning_rate": 1.6998453206075708e-07, + "logits/chosen": -2.4017269611358643, + "logits/rejected": -2.458293914794922, + "logps/chosen": -0.26788607239723206, + "logps/rejected": -0.28343814611434937, + "loss": 0.1603700816631317, + "loss/core": 0.1603700816631317, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.406388282775879, + "rewards/margins": 0.4723166823387146, + "rewards/rejected": 0.9340717196464539, + "step": 580 + }, + { + "epoch": 0.5589394482264421, + "grad_norm": 32.0, + "learning_rate": 1.6540675667967973e-07, + "logits/chosen": -2.1147303581237793, + "logits/rejected": -2.0781502723693848, + "logps/chosen": -0.2706436514854431, + "logps/rejected": -0.2854623794555664, + "loss": 1.4029546976089478, + "loss/core": 1.4029546976089478, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.21844220161438, + "rewards/margins": 0.8768823742866516, + "rewards/rejected": 1.341559648513794, + "step": 585 + }, + { + "epoch": 0.5637167084676937, + "grad_norm": 4.40625, + "learning_rate": 1.60860793357805e-07, + "logits/chosen": -2.001035690307617, + "logits/rejected": -2.096832275390625, + "logps/chosen": -0.29010075330734253, + "logps/rejected": -0.2887147068977356, + "loss": 0.8843881487846375, + "loss/core": 0.8843881487846375, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.5953457355499268, + "rewards/margins": 2.580474376678467, + "rewards/rejected": 1.01487135887146, + "step": 590 + }, + { + "epoch": 0.5684939687089454, + "grad_norm": 3.734375, + "learning_rate": 1.5634835164602196e-07, + "logits/chosen": -2.020965099334717, + "logits/rejected": -2.015770673751831, + "logps/chosen": -0.27582883834838867, + "logps/rejected": -0.2730068266391754, + "loss": 0.1141238808631897, + "loss/core": 0.1141238808631897, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.7046051025390625, + "rewards/margins": 0.826572060585022, + "rewards/rejected": 0.8780330419540405, + "step": 595 + }, + { + "epoch": 0.5732712289501971, + "grad_norm": 24.375, + "learning_rate": 1.518711284891132e-07, + "logits/chosen": -2.250802755355835, + "logits/rejected": -2.2414631843566895, + "logps/chosen": -0.26670363545417786, + "logps/rejected": -0.286176860332489, + "loss": 0.2968599498271942, + "loss/core": 0.2968599498271942, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.8693134784698486, + "rewards/margins": 0.8250168561935425, + "rewards/rejected": 1.0442965030670166, + "step": 600 + }, + { + "epoch": 0.5780484891914487, + "grad_norm": 129.0, + "learning_rate": 1.47430807587603e-07, + "logits/chosen": -2.0972466468811035, + "logits/rejected": -2.1712050437927246, + "logps/chosen": -0.2813746929168701, + "logps/rejected": -0.2767219841480255, + "loss": 1.7541176080703735, + "loss/core": 1.7541176080703735, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.171405792236328, + "rewards/margins": 2.975759267807007, + "rewards/rejected": 1.1956462860107422, + "step": 605 + }, + { + "epoch": 0.5828257494327004, + "grad_norm": 104.0, + "learning_rate": 1.430290587645865e-07, + "logits/chosen": -2.0927460193634033, + "logits/rejected": -2.1099424362182617, + "logps/chosen": -0.27945631742477417, + "logps/rejected": -0.27728086709976196, + "loss": 1.8355001211166382, + "loss/core": 1.8355001211166382, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 4.239993095397949, + "rewards/margins": 2.323775291442871, + "rewards/rejected": 1.9162174463272095, + "step": 610 + }, + { + "epoch": 0.587603009673952, + "grad_norm": 13.0, + "learning_rate": 1.3866753733777765e-07, + "logits/chosen": -1.9469194412231445, + "logits/rejected": -2.012991189956665, + "logps/chosen": -0.3159197270870209, + "logps/rejected": -0.3063480257987976, + "loss": 0.5646291971206665, + "loss/core": 0.5646291971206665, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.7208328247070312, + "rewards/margins": 1.2584408521652222, + "rewards/rejected": 1.4623918533325195, + "step": 615 + }, + { + "epoch": 0.5923802699152036, + "grad_norm": 6.78125, + "learning_rate": 1.343478834970121e-07, + "logits/chosen": -2.1200366020202637, + "logits/rejected": -2.1472256183624268, + "logps/chosen": -0.286575585603714, + "logps/rejected": -0.29443609714508057, + "loss": 0.37433984875679016, + "loss/core": 0.37433984875679016, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.6850149631500244, + "rewards/margins": 0.25270089507102966, + "rewards/rejected": 1.432314157485962, + "step": 620 + }, + { + "epoch": 0.5971575301564552, + "grad_norm": 1520.0, + "learning_rate": 1.3007172168743852e-07, + "logits/chosen": -1.9102928638458252, + "logits/rejected": -1.938646912574768, + "logps/chosen": -0.3089125454425812, + "logps/rejected": -0.3455188274383545, + "loss": 2.0537664890289307, + "loss/core": 2.0537664890289307, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.8455417156219482, + "rewards/margins": 2.4277970790863037, + "rewards/rejected": 1.4177448749542236, + "step": 625 + }, + { + "epoch": 0.6019347903977069, + "grad_norm": 178.0, + "learning_rate": 1.25840659998631e-07, + "logits/chosen": -1.9074939489364624, + "logits/rejected": -2.022514820098877, + "logps/chosen": -0.27223581075668335, + "logps/rejected": -0.28289422392845154, + "loss": 0.8955941200256348, + "loss/core": 0.8955941200256348, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.9453296661376953, + "rewards/margins": 1.6964915990829468, + "rewards/rejected": 1.2488377094268799, + "step": 630 + }, + { + "epoch": 0.6067120506389586, + "grad_norm": 286.0, + "learning_rate": 1.2165628955985313e-07, + "logits/chosen": -1.9614832401275635, + "logits/rejected": -2.0468192100524902, + "logps/chosen": -0.30598872900009155, + "logps/rejected": -0.30102452635765076, + "loss": 0.4347690939903259, + "loss/core": 0.4347690939903259, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.2950398921966553, + "rewards/margins": 1.2899543046951294, + "rewards/rejected": 1.0050859451293945, + "step": 635 + }, + { + "epoch": 0.6114893108802102, + "grad_norm": 2.59375, + "learning_rate": 1.175201839416988e-07, + "logits/chosen": -2.1878843307495117, + "logits/rejected": -2.2639148235321045, + "logps/chosen": -0.3026634752750397, + "logps/rejected": -0.3000384271144867, + "loss": 0.3829612135887146, + "loss/core": 0.3829612135887146, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.9458287954330444, + "rewards/margins": 1.161188006401062, + "rewards/rejected": 0.7846406102180481, + "step": 640 + }, + { + "epoch": 0.6162665711214619, + "grad_norm": 15.0, + "learning_rate": 1.1343389856433658e-07, + "logits/chosen": -2.133284091949463, + "logits/rejected": -2.1109793186187744, + "logps/chosen": -0.2913122773170471, + "logps/rejected": -0.3065160810947418, + "loss": 0.20538568496704102, + "loss/core": 0.20538568496704102, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.6671216487884521, + "rewards/margins": 0.5487877726554871, + "rewards/rejected": 1.1183340549468994, + "step": 645 + }, + { + "epoch": 0.6210438313627135, + "grad_norm": 3.78125, + "learning_rate": 1.0939897011258e-07, + "logits/chosen": -2.198296546936035, + "logits/rejected": -2.151305913925171, + "logps/chosen": -0.2899485230445862, + "logps/rejected": -0.30108729004859924, + "loss": 0.1469523161649704, + "loss/core": 0.1469523161649704, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.3288962841033936, + "rewards/margins": 0.5087905526161194, + "rewards/rejected": 0.8201059103012085, + "step": 650 + }, + { + "epoch": 0.6258210916039652, + "grad_norm": 53.0, + "learning_rate": 1.0541691595800336e-07, + "logits/chosen": -2.037231922149658, + "logits/rejected": -2.0994153022766113, + "logps/chosen": -0.28296002745628357, + "logps/rejected": -0.29361408948898315, + "loss": 0.24921326339244843, + "loss/core": 0.24921326339244843, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.997532844543457, + "rewards/margins": 1.048076868057251, + "rewards/rejected": 0.9494560360908508, + "step": 655 + }, + { + "epoch": 0.6305983518452167, + "grad_norm": 2.390625, + "learning_rate": 1.0148923358832021e-07, + "logits/chosen": -2.0072593688964844, + "logits/rejected": -2.000403642654419, + "logps/chosen": -0.29590821266174316, + "logps/rejected": -0.3091500699520111, + "loss": 1.2735823392868042, + "loss/core": 1.2735823392868042, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.7570533752441406, + "rewards/margins": 1.4806814193725586, + "rewards/rejected": 1.2763724327087402, + "step": 660 + }, + { + "epoch": 0.6353756120864684, + "grad_norm": 134.0, + "learning_rate": 9.761740004423926e-08, + "logits/chosen": -2.169792652130127, + "logits/rejected": -2.1452460289001465, + "logps/chosen": -0.28445929288864136, + "logps/rejected": -0.2821694612503052, + "loss": 0.505510151386261, + "loss/core": 0.505510151386261, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.346339702606201, + "rewards/margins": 0.8737912178039551, + "rewards/rejected": 1.472548484802246, + "step": 665 + }, + { + "epoch": 0.64015287232772, + "grad_norm": 1.421875, + "learning_rate": 9.380287136400999e-08, + "logits/chosen": -1.958132028579712, + "logits/rejected": -2.0062148571014404, + "logps/chosen": -0.2736981213092804, + "logps/rejected": -0.32142865657806396, + "loss": 0.9827208518981934, + "loss/core": 0.9827208518981934, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.6511759757995605, + "rewards/margins": 2.406550168991089, + "rewards/rejected": 1.2446256875991821, + "step": 670 + }, + { + "epoch": 0.6449301325689717, + "grad_norm": 15.375, + "learning_rate": 9.004708203586628e-08, + "logits/chosen": -2.010897159576416, + "logits/rejected": -2.05232834815979, + "logps/chosen": -0.28901082277297974, + "logps/rejected": -0.2891398072242737, + "loss": 0.12457512319087982, + "loss/core": 0.12457512319087982, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.6163723468780518, + "rewards/margins": 0.7447583079338074, + "rewards/rejected": 0.8716139793395996, + "step": 675 + }, + { + "epoch": 0.6497073928102234, + "grad_norm": 1.7265625, + "learning_rate": 8.635144445857407e-08, + "logits/chosen": -2.133448362350464, + "logits/rejected": -2.0533690452575684, + "logps/chosen": -0.28220346570014954, + "logps/rejected": -0.2999503016471863, + "loss": 0.41169777512550354, + "loss/core": 0.41169777512550354, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.1444594860076904, + "rewards/margins": 0.8426515460014343, + "rewards/rejected": 1.3018076419830322, + "step": 680 + }, + { + "epoch": 0.654484653051475, + "grad_norm": 4.28125, + "learning_rate": 8.271734841028552e-08, + "logits/chosen": -2.0622639656066895, + "logits/rejected": -2.0482723712921143, + "logps/chosen": -0.2543559670448303, + "logps/rejected": -0.2783207893371582, + "loss": 0.9395847320556641, + "loss/core": 0.9395847320556641, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.8786520957946777, + "rewards/margins": 1.356764554977417, + "rewards/rejected": 1.5218875408172607, + "step": 685 + }, + { + "epoch": 0.6592619132927267, + "grad_norm": 3.203125, + "learning_rate": 7.91461605259007e-08, + "logits/chosen": -1.8647575378417969, + "logits/rejected": -1.887168526649475, + "logps/chosen": -0.3094645142555237, + "logps/rejected": -0.32861852645874023, + "loss": 0.20163805782794952, + "loss/core": 0.20163805782794952, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.7553918361663818, + "rewards/margins": 0.6693966388702393, + "rewards/rejected": 1.0859953165054321, + "step": 690 + }, + { + "epoch": 0.6640391735339782, + "grad_norm": 9.375, + "learning_rate": 7.563922378313217e-08, + "logits/chosen": -2.2469770908355713, + "logits/rejected": -2.256824016571045, + "logps/chosen": -0.3077448308467865, + "logps/rejected": -0.2938057780265808, + "loss": 0.3713051676750183, + "loss/core": 0.3713051676750183, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9559170007705688, + "rewards/margins": 1.282508134841919, + "rewards/rejected": 0.6734089851379395, + "step": 695 + }, + { + "epoch": 0.6688164337752299, + "grad_norm": 388.0, + "learning_rate": 7.219785699746572e-08, + "logits/chosen": -2.08168888092041, + "logits/rejected": -2.1877474784851074, + "logps/chosen": -0.2970959544181824, + "logps/rejected": -0.3301224112510681, + "loss": 0.46775469183921814, + "loss/core": 0.46775469183921814, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.227623224258423, + "rewards/margins": 0.7473882436752319, + "rewards/rejected": 1.4802348613739014, + "step": 700 + }, + { + "epoch": 0.6735936940164815, + "grad_norm": 84.5, + "learning_rate": 6.882335432620779e-08, + "logits/chosen": -2.1753158569335938, + "logits/rejected": -2.1324844360351562, + "logps/chosen": -0.28070613741874695, + "logps/rejected": -0.2675221264362335, + "loss": 0.8060401082038879, + "loss/core": 0.8060401082038879, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.4011025428771973, + "rewards/margins": 1.8703250885009766, + "rewards/rejected": 1.5307778120040894, + "step": 705 + }, + { + "epoch": 0.6783709542577332, + "grad_norm": 370.0, + "learning_rate": 6.551698478180589e-08, + "logits/chosen": -1.862591028213501, + "logits/rejected": -1.8508355617523193, + "logps/chosen": -0.27737393975257874, + "logps/rejected": -0.3012697696685791, + "loss": 1.3350226879119873, + "loss/core": 1.3350226879119873, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.5951266288757324, + "rewards/margins": 1.7325704097747803, + "rewards/rejected": 1.8625566959381104, + "step": 710 + }, + { + "epoch": 0.6831482144989849, + "grad_norm": 3.78125, + "learning_rate": 6.22799917546252e-08, + "logits/chosen": -2.230132579803467, + "logits/rejected": -2.2110886573791504, + "logps/chosen": -0.2842472493648529, + "logps/rejected": -0.3385520279407501, + "loss": 0.5101768374443054, + "loss/core": 0.5101768374443054, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.8642584085464478, + "rewards/margins": 1.1836979389190674, + "rewards/rejected": 0.6805603504180908, + "step": 715 + }, + { + "epoch": 0.6879254747402365, + "grad_norm": 23.625, + "learning_rate": 5.9113592545359944e-08, + "logits/chosen": -1.9779284000396729, + "logits/rejected": -1.9853112697601318, + "logps/chosen": -0.2979108393192291, + "logps/rejected": -0.30363336205482483, + "loss": 0.4835893511772156, + "loss/core": 0.4835893511772156, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.0863685607910156, + "rewards/margins": 0.6202920079231262, + "rewards/rejected": 1.4660766124725342, + "step": 720 + }, + { + "epoch": 0.6927027349814882, + "grad_norm": 3.953125, + "learning_rate": 5.601897790725643e-08, + "logits/chosen": -2.102151393890381, + "logits/rejected": -2.2167160511016846, + "logps/chosen": -0.28644677996635437, + "logps/rejected": -0.27808719873428345, + "loss": 0.17492833733558655, + "loss/core": 0.17492833733558655, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.5772634744644165, + "rewards/margins": 0.8539497256278992, + "rewards/rejected": 0.7233136892318726, + "step": 725 + }, + { + "epoch": 0.6974799952227397, + "grad_norm": 398.0, + "learning_rate": 5.299731159831952e-08, + "logits/chosen": -2.1961236000061035, + "logits/rejected": -2.246107339859009, + "logps/chosen": -0.287611722946167, + "logps/rejected": -0.2948874533176422, + "loss": 0.19452638924121857, + "loss/core": 0.19452638924121857, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.0076961517333984, + "rewards/margins": 0.14150258898735046, + "rewards/rejected": 0.8661934733390808, + "step": 730 + }, + { + "epoch": 0.7022572554639914, + "grad_norm": 58.0, + "learning_rate": 5.0049729943671013e-08, + "logits/chosen": -2.0291497707366943, + "logits/rejected": -2.0758984088897705, + "logps/chosen": -0.32085052132606506, + "logps/rejected": -0.31806251406669617, + "loss": 0.14255869388580322, + "loss/core": 0.14255869388580322, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.4092836380004883, + "rewards/margins": 0.7487727403640747, + "rewards/rejected": 0.6605110168457031, + "step": 735 + }, + { + "epoch": 0.707034515705243, + "grad_norm": 5.46875, + "learning_rate": 4.7177341408223994e-08, + "logits/chosen": -2.077390193939209, + "logits/rejected": -2.095350742340088, + "logps/chosen": -0.3161494731903076, + "logps/rejected": -0.3177900016307831, + "loss": 1.4744199514389038, + "loss/core": 1.4744199514389038, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.533155918121338, + "rewards/margins": 0.720278799533844, + "rewards/rejected": 1.8128772974014282, + "step": 740 + }, + { + "epoch": 0.7118117759464947, + "grad_norm": 384.0, + "learning_rate": 4.438122617983442e-08, + "logits/chosen": -2.031482696533203, + "logits/rejected": -2.0756478309631348, + "logps/chosen": -0.29046431183815, + "logps/rejected": -0.2840021848678589, + "loss": 0.611270010471344, + "loss/core": 0.611270010471344, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.1211209297180176, + "rewards/margins": 1.6258805990219116, + "rewards/rejected": 1.4952404499053955, + "step": 745 + }, + { + "epoch": 0.7165890361877463, + "grad_norm": 318.0, + "learning_rate": 4.1662435763087114e-08, + "logits/chosen": -2.16795015335083, + "logits/rejected": -2.160215377807617, + "logps/chosen": -0.3122718930244446, + "logps/rejected": -0.3045649826526642, + "loss": 0.5550835728645325, + "loss/core": 0.5550835728645325, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.2003018856048584, + "rewards/margins": -0.04670139029622078, + "rewards/rejected": 1.2470033168792725, + "step": 750 + }, + { + "epoch": 0.721366296428998, + "grad_norm": 23.125, + "learning_rate": 3.902199258386732e-08, + "logits/chosen": -1.905664086341858, + "logits/rejected": -1.815201997756958, + "logps/chosen": -0.3100651502609253, + "logps/rejected": -0.2970584034919739, + "loss": 0.3808279037475586, + "loss/core": 0.3808279037475586, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.371333122253418, + "rewards/margins": 0.689149796962738, + "rewards/rejected": 1.6821835041046143, + "step": 755 + }, + { + "epoch": 0.7261435566702497, + "grad_norm": 20.0, + "learning_rate": 3.646088960486862e-08, + "logits/chosen": -2.1599106788635254, + "logits/rejected": -2.178335189819336, + "logps/chosen": -0.3025417923927307, + "logps/rejected": -0.2942703664302826, + "loss": 0.27844661474227905, + "loss/core": 0.27844661474227905, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.9924026727676392, + "rewards/margins": 1.1353952884674072, + "rewards/rejected": 0.8570073843002319, + "step": 760 + }, + { + "epoch": 0.7309208169115012, + "grad_norm": 41.25, + "learning_rate": 3.398008995217988e-08, + "logits/chosen": -2.05729603767395, + "logits/rejected": -2.128960132598877, + "logps/chosen": -0.2781466543674469, + "logps/rejected": -0.2730472981929779, + "loss": 0.23510298132896423, + "loss/core": 0.23510298132896423, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.773431420326233, + "rewards/margins": 0.7215307354927063, + "rewards/rejected": 1.0519007444381714, + "step": 765 + }, + { + "epoch": 0.7356980771527529, + "grad_norm": 50.25, + "learning_rate": 3.1580526553093315e-08, + "logits/chosen": -1.9602434635162354, + "logits/rejected": -1.927735686302185, + "logps/chosen": -0.31485968828201294, + "logps/rejected": -0.28551191091537476, + "loss": 2.1671090126037598, + "loss/core": 2.1671090126037598, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.6825318336486816, + "rewards/margins": 1.338300108909607, + "rewards/rejected": 2.3442318439483643, + "step": 770 + }, + { + "epoch": 0.7404753373940045, + "grad_norm": 1.8984375, + "learning_rate": 2.9263101785268252e-08, + "logits/chosen": -2.0744194984436035, + "logits/rejected": -2.105100154876709, + "logps/chosen": -0.3076358139514923, + "logps/rejected": -0.3005218207836151, + "loss": 0.14362755417823792, + "loss/core": 0.14362755417823792, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.5125480890274048, + "rewards/margins": 0.5728277564048767, + "rewards/rejected": 0.9397203326225281, + "step": 775 + }, + { + "epoch": 0.7452525976352562, + "grad_norm": 19.125, + "learning_rate": 2.7028687137384264e-08, + "logits/chosen": -2.156768798828125, + "logits/rejected": -2.170663356781006, + "logps/chosen": -0.3037753999233246, + "logps/rejected": -0.3085966110229492, + "loss": 0.22171640396118164, + "loss/core": 0.22171640396118164, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.8746483325958252, + "rewards/margins": 0.6551548838615417, + "rewards/rejected": 1.2194935083389282, + "step": 780 + }, + { + "epoch": 0.7500298578765078, + "grad_norm": 1.6328125, + "learning_rate": 2.4878122881409447e-08, + "logits/chosen": -1.9704217910766602, + "logits/rejected": -1.92739999294281, + "logps/chosen": -0.3119404911994934, + "logps/rejected": -0.2957368493080139, + "loss": 1.2243354320526123, + "loss/core": 1.2243354320526123, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 3.421541690826416, + "rewards/margins": 2.0960593223571777, + "rewards/rejected": 1.3254823684692383, + "step": 785 + }, + { + "epoch": 0.7548071181177595, + "grad_norm": 244.0, + "learning_rate": 2.2812217756608937e-08, + "logits/chosen": -2.1468310356140137, + "logits/rejected": -1.9746036529541016, + "logps/chosen": -0.28714293241500854, + "logps/rejected": -0.3162146210670471, + "loss": 1.1724170446395874, + "loss/core": 1.1724170446395874, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.608785390853882, + "rewards/margins": 0.6583928465843201, + "rewards/rejected": 1.9503923654556274, + "step": 790 + }, + { + "epoch": 0.7595843783590112, + "grad_norm": 24.125, + "learning_rate": 2.0831748665410763e-08, + "logits/chosen": -1.9510393142700195, + "logits/rejected": -1.968506097793579, + "logps/chosen": -0.30975866317749023, + "logps/rejected": -0.2976404130458832, + "loss": 1.044623851776123, + "loss/core": 1.044623851776123, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.357090473175049, + "rewards/margins": 1.1862857341766357, + "rewards/rejected": 1.1708046197891235, + "step": 795 + }, + { + "epoch": 0.7643616386002627, + "grad_norm": 4.53125, + "learning_rate": 1.8937460381244967e-08, + "logits/chosen": -2.131096363067627, + "logits/rejected": -2.1090807914733887, + "logps/chosen": -0.2900455892086029, + "logps/rejected": -0.28949636220932007, + "loss": 0.3053150773048401, + "loss/core": 0.3053150773048401, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.288881778717041, + "rewards/margins": 1.1297084093093872, + "rewards/rejected": 1.1591734886169434, + "step": 800 + }, + { + "epoch": 0.7691388988415144, + "grad_norm": 9.5625, + "learning_rate": 1.713006526846439e-08, + "logits/chosen": -2.0039420127868652, + "logits/rejected": -1.9931786060333252, + "logps/chosen": -0.27616554498672485, + "logps/rejected": -0.27139753103256226, + "loss": 0.5071675181388855, + "loss/core": 0.5071675181388855, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.244047164916992, + "rewards/margins": 0.266120582818985, + "rewards/rejected": 1.9779266119003296, + "step": 805 + }, + { + "epoch": 0.773916159082766, + "grad_norm": 28.75, + "learning_rate": 1.541024301445404e-08, + "logits/chosen": -2.2073111534118652, + "logits/rejected": -2.1122589111328125, + "logps/chosen": -0.3092059791088104, + "logps/rejected": -0.32349738478660583, + "loss": 0.23964479565620422, + "loss/core": 0.23964479565620422, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.5212091207504272, + "rewards/margins": 0.11138463020324707, + "rewards/rejected": 1.409824252128601, + "step": 810 + }, + { + "epoch": 0.7786934193240177, + "grad_norm": 35.0, + "learning_rate": 1.3778640374027983e-08, + "logits/chosen": -2.0743167400360107, + "logits/rejected": -2.162884473800659, + "logps/chosen": -0.2836160361766815, + "logps/rejected": -0.28452104330062866, + "loss": 0.24607639014720917, + "loss/core": 0.24607639014720917, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.1251344680786133, + "rewards/margins": 1.0335019826889038, + "rewards/rejected": 1.09163236618042, + "step": 815 + }, + { + "epoch": 0.7834706795652693, + "grad_norm": 3.296875, + "learning_rate": 1.2235870926211616e-08, + "logits/chosen": -2.079317569732666, + "logits/rejected": -2.112313747406006, + "logps/chosen": -0.288349449634552, + "logps/rejected": -0.31876736879348755, + "loss": 0.2414264678955078, + "loss/core": 0.2414264678955078, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9595178365707397, + "rewards/margins": 1.2891201972961426, + "rewards/rejected": 0.6703976988792419, + "step": 820 + }, + { + "epoch": 0.788247939806521, + "grad_norm": 60.5, + "learning_rate": 1.0782514843499652e-08, + "logits/chosen": -2.1222786903381348, + "logits/rejected": -2.2262613773345947, + "logps/chosen": -0.30225175619125366, + "logps/rejected": -0.2850930094718933, + "loss": 0.308658629655838, + "loss/core": 0.308658629655838, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.461728096008301, + "rewards/margins": 1.250454068183899, + "rewards/rejected": 1.2112740278244019, + "step": 825 + }, + { + "epoch": 0.7930252000477725, + "grad_norm": 8.625, + "learning_rate": 9.419118673676924e-09, + "logits/chosen": -2.156327962875366, + "logits/rejected": -2.1050965785980225, + "logps/chosen": -0.2788023054599762, + "logps/rejected": -0.3164371848106384, + "loss": 0.30850037932395935, + "loss/core": 0.30850037932395935, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7236570119857788, + "rewards/margins": 0.15539640188217163, + "rewards/rejected": 1.5682607889175415, + "step": 830 + }, + { + "epoch": 0.7978024602890242, + "grad_norm": 173.0, + "learning_rate": 8.14619513428405e-09, + "logits/chosen": -2.0721898078918457, + "logits/rejected": -2.071643590927124, + "logps/chosen": -0.2557886242866516, + "logps/rejected": -0.30318066477775574, + "loss": 0.3867768943309784, + "loss/core": 0.3867768943309784, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.3934996128082275, + "rewards/margins": 1.5836869478225708, + "rewards/rejected": 0.8098125457763672, + "step": 835 + }, + { + "epoch": 0.8025797205302759, + "grad_norm": 2672.0, + "learning_rate": 6.96422291980539e-09, + "logits/chosen": -2.0129635334014893, + "logits/rejected": -2.0625948905944824, + "logps/chosen": -0.32838812470436096, + "logps/rejected": -0.3022240698337555, + "loss": 0.8802303075790405, + "loss/core": 0.8802303075790405, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.7439942359924316, + "rewards/margins": 1.7150169610977173, + "rewards/rejected": 1.0289772748947144, + "step": 840 + }, + { + "epoch": 0.8073569807715275, + "grad_norm": 3.296875, + "learning_rate": 5.8736465216517594e-09, + "logits/chosen": -2.1509671211242676, + "logits/rejected": -2.22468638420105, + "logps/chosen": -0.29492026567459106, + "logps/rejected": -0.3140087425708771, + "loss": 0.15912021696567535, + "loss/core": 0.15912021696567535, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.4049441814422607, + "rewards/margins": 0.977611243724823, + "rewards/rejected": 0.4273327887058258, + "step": 845 + }, + { + "epoch": 0.8121342410127792, + "grad_norm": 37.0, + "learning_rate": 4.874876061005173e-09, + "logits/chosen": -2.2683005332946777, + "logits/rejected": -2.3160834312438965, + "logps/chosen": -0.25317704677581787, + "logps/rejected": -0.2489752322435379, + "loss": 0.0958312377333641, + "loss/core": 0.0958312377333641, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.54629647731781, + "rewards/margins": 0.6603418588638306, + "rewards/rejected": 0.8859546780586243, + "step": 850 + }, + { + "epoch": 0.8169115012540308, + "grad_norm": 59.5, + "learning_rate": 3.968287134589188e-09, + "logits/chosen": -1.9828602075576782, + "logits/rejected": -2.0203537940979004, + "logps/chosen": -0.30375003814697266, + "logps/rejected": -0.2930416464805603, + "loss": 0.8324758410453796, + "loss/core": 0.8324758410453796, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.6595799922943115, + "rewards/margins": 1.4085386991500854, + "rewards/rejected": 1.251041293144226, + "step": 855 + }, + { + "epoch": 0.8216887614952825, + "grad_norm": 2.515625, + "learning_rate": 3.154220673422192e-09, + "logits/chosen": -1.9286237955093384, + "logits/rejected": -2.016119956970215, + "logps/chosen": -0.3028337359428406, + "logps/rejected": -0.30019161105155945, + "loss": 1.6009376049041748, + "loss/core": 1.6009376049041748, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.053446292877197, + "rewards/margins": 2.1368956565856934, + "rewards/rejected": 1.916550874710083, + "step": 860 + }, + { + "epoch": 0.826466021736534, + "grad_norm": 51.25, + "learning_rate": 2.4329828146074096e-09, + "logits/chosen": -1.9394134283065796, + "logits/rejected": -2.0103347301483154, + "logps/chosen": -0.2917220890522003, + "logps/rejected": -0.2960386276245117, + "loss": 0.590233325958252, + "loss/core": 0.590233325958252, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.443800926208496, + "rewards/margins": 0.8796428442001343, + "rewards/rejected": 1.5641582012176514, + "step": 865 + }, + { + "epoch": 0.8312432819777857, + "grad_norm": 13.125, + "learning_rate": 1.8048447862070714e-09, + "logits/chosen": -2.090069532394409, + "logits/rejected": -2.098034381866455, + "logps/chosen": -0.3098277449607849, + "logps/rejected": -0.29889780282974243, + "loss": 1.0631767511367798, + "loss/core": 1.0631767511367798, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.1978888511657715, + "rewards/margins": 2.2793126106262207, + "rewards/rejected": 0.9185762405395508, + "step": 870 + }, + { + "epoch": 0.8360205422190374, + "grad_norm": 7.15625, + "learning_rate": 1.2700428052447033e-09, + "logits/chosen": -2.1596736907958984, + "logits/rejected": -2.207904100418091, + "logps/chosen": -0.30997827649116516, + "logps/rejected": -0.3309651017189026, + "loss": 0.7209604382514954, + "loss/core": 0.7209604382514954, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8416748046875, + "rewards/margins": 0.8194574117660522, + "rewards/rejected": 1.0222175121307373, + "step": 875 + }, + { + "epoch": 0.840797802460289, + "grad_norm": 62.25, + "learning_rate": 8.287779888734858e-10, + "logits/chosen": -2.0957977771759033, + "logits/rejected": -2.079482078552246, + "logps/chosen": -0.291581928730011, + "logps/rejected": -0.3347880244255066, + "loss": 0.9310186505317688, + "loss/core": 0.9310186505317688, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.448652982711792, + "rewards/margins": 0.716099739074707, + "rewards/rejected": 1.7325531244277954, + "step": 880 + }, + { + "epoch": 0.8455750627015407, + "grad_norm": 23.75, + "learning_rate": 4.812162787445062e-10, + "logits/chosen": -2.0210766792297363, + "logits/rejected": -2.097437858581543, + "logps/chosen": -0.28354987502098083, + "logps/rejected": -0.27728238701820374, + "loss": 0.42107605934143066, + "loss/core": 0.42107605934143066, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.7150750160217285, + "rewards/margins": 1.5944372415542603, + "rewards/rejected": 1.1206376552581787, + "step": 885 + }, + { + "epoch": 0.8503523229427923, + "grad_norm": 296.0, + "learning_rate": 2.2748837860270265e-10, + "logits/chosen": -1.9509280920028687, + "logits/rejected": -2.072624921798706, + "logps/chosen": -0.3031921684741974, + "logps/rejected": -0.2749203145503998, + "loss": 0.6249431371688843, + "loss/core": 0.6249431371688843, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.4241182804107666, + "rewards/margins": 0.9915217161178589, + "rewards/rejected": 1.4325960874557495, + "step": 890 + }, + { + "epoch": 0.855129583184044, + "grad_norm": 52.0, + "learning_rate": 6.768970513457151e-11, + "logits/chosen": -2.0221574306488037, + "logits/rejected": -2.0542521476745605, + "logps/chosen": -0.27334579825401306, + "logps/rejected": -0.291655033826828, + "loss": 0.3214346766471863, + "loss/core": 0.3214346766471863, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.2918426990509033, + "rewards/margins": 0.789661705493927, + "rewards/rejected": 1.502180814743042, + "step": 895 + }, + { + "epoch": 0.8599068434252956, + "grad_norm": 17.125, + "learning_rate": 1.8803520859811406e-12, + "logits/chosen": -2.0327274799346924, + "logits/rejected": -2.152374744415283, + "logps/chosen": -0.271484911441803, + "logps/rejected": -0.2696631848812103, + "loss": 0.9997545480728149, + "loss/core": 0.9997545480728149, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 3.091799259185791, + "rewards/margins": 2.2562103271484375, + "rewards/rejected": 0.8355890512466431, + "step": 900 + }, + { + "epoch": 0.8599068434252956, + "step": 900, + "total_flos": 0.0, + "train_loss": 0.7102717447943158, + "train_runtime": 7093.1544, + "train_samples_per_second": 2.03, + "train_steps_per_second": 0.127 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..6d9faa5 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a7d16156a0245e4d2c7242237fe50bc0ff7de4975bd60b29150e76c65884ca9 +size 7057