From 55dd5d77068e0e14684bafb272a81dd2e7cdccdb Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Thu, 30 Jul 2026 20:35:18 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: promotion/qwen3-8b-aaai27-flagship-simpo-s44 Source: Original Platform --- .gitattributes | 36 + README.md | 24 + all_results.json | 9 + chat_template.jinja | 89 ++ config.json | 71 + config.yaml | 58 + generation_config.json | 12 + job_status.json | 23 + model.safetensors | 3 + objective_protocol.json | 69 + pair_manifest.json | 34 + provenance.json | 13 + resource_profile.json | 21 + run_status.json | 14 + split_manifest.json | 23 + stability_gate.json | 38 + tokenizer.json | 3 + tokenizer_config.json | 30 + train_results.json | 9 + trainer_state.json | 2923 +++++++++++++++++++++++++++++++++++++++ training_args.bin | 3 + 21 files changed, 3505 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 config.yaml create mode 100644 generation_config.json create mode 100644 job_status.json create mode 100644 model.safetensors create mode 100644 objective_protocol.json create mode 100644 pair_manifest.json create mode 100644 provenance.json create mode 100644 resource_profile.json create mode 100644 run_status.json create mode 100644 split_manifest.json create mode 100644 stability_gate.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..78989e0 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +--- +library_name: transformers +base_model: Qwen/Qwen3-8B +tags: +- ronpo +- mnpo +- preference-optimization +--- + +# qwen3-8b-aaai27-flagship-simpo-s44 + +Research checkpoint for the RONPO AAAI revision experiments. + +- Method: simpo +- Base model: `Qwen/Qwen3-8B` +- Seed: 44 +- Local source at upload time: `/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/simpo/seed44/attempt1` +- Uploaded at UTC: 2026-07-12T22:15:14Z +- Run status metadata: `{"attempt": 1, "effective_batch_size": 16, "method": "simpo", "objective_protocol": "objective_protocol.json", "optimizer_steps": 900, "resource_profile": "resource_profile.json", "seed": 44, "split_manifest": "split_manifest.json", "stability_gate": "stability_gate.json", "status": "completed", "wandb_run_id": "f2836fce0671", "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/f2836fce0671"}` + +AAAI-27 P1 matched budget: 900 optimizer steps, effective batch 16; passed non-thinking and collapse stability gates; sealed-test metrics are not used for checkpoint selection. + +Intended use: reproducibility and evaluation for the RONPO paper. This +checkpoint is not intended as a production assistant. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..6774539 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 1.0422146828969319, + "train_runtime": 8158.2235, + "train_samples": 16746, + "train_samples_per_second": 1.765, + "train_steps_per_second": 0.11 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..601f619 --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +model_name_or_path: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218 +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/avg: 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: simpo +eta: 0.0075 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.0 +preference_sft_weight: 0.0 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 5.0e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 44 +gradient_accumulation_steps: 8 +gradient_checkpointing: false +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 2 +per_device_eval_batch_size: 2 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/simpo/seed44/attempt1 +run_name: aaai27-flagship-full-simpo-s44-a1 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/job_status.json b/job_status.json new file mode 100644 index 0000000..337f49d --- /dev/null +++ b/job_status.json @@ -0,0 +1,23 @@ +{ + "status": "completed", + "stage": "full", + "method": "simpo", + "seed": 44, + "attempt": 1, + "gpu": 6, + "gpus": [ + 6 + ], + "returncode": 0, + "finite_metrics": true, + "oom": false, + "traceback": false, + "reason": "passed", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "f2836fce0671", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/f2836fce0671", + "output_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/simpo/seed44/attempt1", + "log": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/logs/full_simpo_s44_a1.log", + "completed_at": "2026-07-12T22:12:55Z" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..07a9473 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b22bbfe4c50e6cf7d97d1c9ba24ab1600f98d7bdcb041f275ad4c4a225651265 +size 16381517208 diff --git a/objective_protocol.json b/objective_protocol.json new file mode 100644 index 0000000..35b3b44 --- /dev/null +++ b/objective_protocol.json @@ -0,0 +1,69 @@ +{ + "schema_version": 1, + "frozen_at_kst": "2026-07-12T21:40:00+09:00", + "base_model": "Qwen/Qwen3-8B", + "primary_triple": { + "helpfulness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "ultrafeedback-helpfulness", + "transform": "identity" + }, + "safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "conciseness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "helpsteer-verbosity", + "transform": "negate" + } + }, + "gate_thresholds": { + "raw_score_std_strictly_greater_than": 0.01, + "aggregate_prompt_pair_spearman_median_strictly_less_than": 0.0, + "aggregate_pairwise_top1_mismatch_at_least": 0.5 + }, + "gate_pool": "source-train Qwen3-8B response pool split into deterministic train/validation; sealed source-test excluded", + "training_seeds": [42, 43, 44], + "optimizer_steps": 900, + "effective_batch_size": 16, + "decode": { + "enable_thinking": false, + "temperature": 0.7, + "top_p": 0.9, + "max_new_tokens": 2048, + "decode_seed": 42 + }, + "methods": [ + "ronpo_full_expect", + "ronpo_k_only", + "dpo", + "ipo", + "simpo", + "kto", + "sppo_avg", + "inpo_avg", + "ht_mnpo_helpfulness", + "ht_mnpo_safety", + "ht_mnpo_conciseness" + ], + "selection_policy": "No sealed-test selection. One preregistered configuration per method; failures use only the declared stability retry recipe, at most three attempts.", + "stability_gate": { + "validation_prompts": 128, + "min_candidate_base_mean_word_ratio": 0.33, + "max_candidate_base_mean_word_ratio": 2.0, + "max_consecutive_identical_word_run": 20, + "think_tag_count": 0, + "empty_response_count": 0, + "retry_recipe": "attempt 2 halves learning rate, enables gradient checkpointing, and adds reference=0.02/SFT=0.002 anchoring to unanchored unified losses; attempt 3 quarters learning rate and uses reference=0.05/SFT=0.005; RONPO uses reference=0.05/SFT=0.005 from attempt 1" + }, + "primary_metric": "mean prompt-level minimum of per-prompt min-max-normalized helpfulness, safety, conciseness", + "bootstrap": { + "unit": "prompt", + "paired": true, + "resamples": 2000, + "interval": "percentile_95", + "seed": 42 + } +} diff --git a/pair_manifest.json b/pair_manifest.json new file mode 100644 index 0000000..be034b0 --- /dev/null +++ b/pair_manifest.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "objectives": [ + "helpfulness", + "safety", + "conciseness" + ], + "normalization": "per_prompt_minmax", + "adversary": { + "steps": 25, + "alpha": 1.0, + "kappa": 0.05, + "preference_scale": 8.0 + }, + "ronpo_estimator": "Rao-Blackwellized full sigma expectation with three common response pairs per prompt", + "counts": { + "train": { + "avg": 16746, + "ronpo_full_expect": 50340, + "ronpo_k_only": 50340, + "ht_mnpo_helpfulness": 16743, + "ht_mnpo_safety": 16764, + "ht_mnpo_conciseness": 16755 + }, + "validation": { + "avg": 1859, + "ronpo_full_expect": 5592, + "ronpo_k_only": 5592, + "ht_mnpo_helpfulness": 1864, + "ht_mnpo_safety": 1862, + "ht_mnpo_conciseness": 1858 + } + } +} diff --git a/provenance.json b/provenance.json new file mode 100644 index 0000000..83f93bd --- /dev/null +++ b/provenance.json @@ -0,0 +1,13 @@ +{ + "method": "simpo", + "seed": 44, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "f2836fce0671", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/f2836fce0671", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json" +} diff --git a/resource_profile.json b/resource_profile.json new file mode 100644 index 0000000..18eec47 --- /dev/null +++ b/resource_profile.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "measured_at_kst": "2026-07-12T22:38:00+09:00", + "hardware": "NVIDIA B200 183359 MiB reported total; 178.35 GiB PyTorch-visible capacity", + "smoke_evidence": { + "attempted_profile": {"per_device_batch_size": 2, "gradient_accumulation_steps": 8, "gradient_checkpointing": false, "effective_batch_size": 16}, + "outcome": "OOM before completing the 20-step smoke across every launched unified baseline; representative process used 176.87 GiB and failed a 1.62 GiB allocation. KTO also OOMed at the optimizer step.", + "paper_metric": false + }, + "selected_full_training_profile": { + "per_device_batch_size": 2, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": false, + "attention_implementation": "PyTorch SDPA with cuDNN SDPA disabled", + "effective_batch_size": 16, + "applies_symmetrically_to_all_unified_pairwise_methods": true, + "kto_implementation_constraint": "TRL KTO requires actual batch size > 1. KTO uses batch 2 / accumulation 8 and identical AdamW with foreach=False. PyTorch SDPA is used with cuDNN SDPA disabled. The fixed base reference model is placed on a second B200 and its exact logits are returned to the policy device; the policy/optimizer remain on the first B200. This avoids a measured ~6-hour reference-precompute path while leaving the KTO loss, actual/effective batch, optimizer, and 900-step budget unchanged." + }, + "retry_semantics": "Resource selection is separated from method-stability attempts. Full attempt 1 uses the smoke-validated SDPA batch-2 profile and retains the preregistered method LR/anchor; attempts 2 and 3 fall back to batch 1 / accumulation 16 / gradient checkpointing and apply the declared LR/anchor stabilization recipe.", + "provenance_note": "No reward or sealed-evaluation result was available when this hardware profile was selected." +} diff --git a/run_status.json b/run_status.json new file mode 100644 index 0000000..78ca8be --- /dev/null +++ b/run_status.json @@ -0,0 +1,14 @@ +{ + "method": "simpo", + "seed": 44, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "f2836fce0671", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/f2836fce0671", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json", + "status": "completed" +} diff --git a/split_manifest.json b/split_manifest.json new file mode 100644 index 0000000..4fb96bd --- /dev/null +++ b/split_manifest.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "split_rule": "sort source-train prompts by sha256('flagship-v1||'+prompt_id); first round(10%) validation", + "validation_fraction": 0.1, + "source_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/avg_oracle", + "base_model": "Qwen/Qwen3-8B", + "counts": { + "train": 16780, + "validation": 1864, + "sealed_test": 604 + }, + "prompt_id_sha256": { + "train": "fcf6f2ea5a0b7bcd41179e6f5288f4d850b2d04bdb729da91907bd18bf733a69", + "validation": "a87ba6f519192164daa487b109a6547dd8cbde46a27a373d6da201ca2d061b8f", + "sealed_test": "b398ced6af6cf73d731e1a5f32a0631d8060323d51eda1e375695566bf5f1867" + }, + "overlaps": { + "train_validation": 0, + "train_sealed_test": 0, + "validation_sealed_test": 0 + }, + "sealed_test_policy": "prompts only until S4a; no objective scores or model generations used in S0-S3" +} diff --git a/stability_gate.json b/stability_gate.json new file mode 100644 index 0000000..c91f657 --- /dev/null +++ b/stability_gate.json @@ -0,0 +1,38 @@ +{ + "status": "passed", + "passed": true, + "base": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 388.1484375, + "max_words": 1379, + "max_repeat_run": 4 + }, + "candidate": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 392.9765625, + "max_words": 1402, + "max_repeat_run": 3 + }, + "candidate_base_mean_word_ratio": 1.0124388623875369, + "thresholds": { + "min_length_ratio": 0.33, + "max_length_ratio": 2.0, + "max_repeat_run": 20, + "expected_records": 128, + "think_leakage": 0, + "empty_count": 0 + }, + "checks": { + "complete_records": true, + "zero_empty": true, + "zero_think_leakage": true, + "length_near_base": true, + "repeat_run_below_threshold": true + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ffd19d2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..6774539 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 1.0422146828969319, + "train_runtime": 8158.2235, + "train_samples": 16746, + "train_samples_per_second": 1.765, + "train_steps_per_second": 0.11 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..3f28a37 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,2923 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8599068434252956, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.004777260241251642, + "grad_norm": 47.0, + "learning_rate": 2.222222222222222e-08, + "logits/chosen": -1.8765161037445068, + "logits/rejected": -1.8362525701522827, + "logps/chosen": -0.3087114691734314, + "logps/rejected": -0.3054303526878357, + "loss": 1.0474976301193237, + "loss/core": 1.0474976301193237, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.4050393104553223, + "rewards/margins": 1.9889495372772217, + "rewards/rejected": 1.416089653968811, + "step": 5 + }, + { + "epoch": 0.009554520482503284, + "grad_norm": 15.125, + "learning_rate": 5e-08, + "logits/chosen": -1.8804371356964111, + "logits/rejected": -1.8574333190917969, + "logps/chosen": -0.29082149267196655, + "logps/rejected": -0.27977004647254944, + "loss": 1.056221842765808, + "loss/core": 1.056221842765808, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.9182629585266113, + "rewards/margins": 1.5795632600784302, + "rewards/rejected": 1.3386995792388916, + "step": 10 + }, + { + "epoch": 0.014331780723754926, + "grad_norm": 7.03125, + "learning_rate": 7.777777777777778e-08, + "logits/chosen": -1.6232883930206299, + "logits/rejected": -1.6565492153167725, + "logps/chosen": -0.2895023226737976, + "logps/rejected": -0.28725099563598633, + "loss": 1.042079210281372, + "loss/core": 1.042079210281372, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.9274177551269531, + "rewards/margins": 1.0600453615188599, + "rewards/rejected": 0.8673725128173828, + "step": 15 + }, + { + "epoch": 0.01910904096500657, + "grad_norm": 7.65625, + "learning_rate": 1.0555555555555555e-07, + "logits/chosen": -1.892533302307129, + "logits/rejected": -1.9024165868759155, + "logps/chosen": -0.28663748502731323, + "logps/rejected": -0.2839517891407013, + "loss": 1.0445116758346558, + "loss/core": 1.0445116758346558, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.6367988586425781, + "rewards/margins": 1.0847914218902588, + "rewards/rejected": 0.5520075559616089, + "step": 20 + }, + { + "epoch": 0.02388630120625821, + "grad_norm": 17.25, + "learning_rate": 1.3333333333333334e-07, + "logits/chosen": -1.929903268814087, + "logits/rejected": -1.9212831258773804, + "logps/chosen": -0.27033165097236633, + "logps/rejected": -0.2790217995643616, + "loss": 1.028629183769226, + "loss/core": 1.028629183769226, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.8104407787322998, + "rewards/margins": 0.6213824152946472, + "rewards/rejected": 1.1890584230422974, + "step": 25 + }, + { + "epoch": 0.028663561447509853, + "grad_norm": 46.5, + "learning_rate": 1.611111111111111e-07, + "logits/chosen": -1.8559471368789673, + "logits/rejected": -1.8298406600952148, + "logps/chosen": -0.31229084730148315, + "logps/rejected": -0.27183204889297485, + "loss": 1.1012113094329834, + "loss/core": 1.1012113094329834, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.737351179122925, + "rewards/margins": 0.9300357103347778, + "rewards/rejected": 1.8073152303695679, + "step": 30 + }, + { + "epoch": 0.033440821688761495, + "grad_norm": 9.3125, + "learning_rate": 1.8888888888888888e-07, + "logits/chosen": -1.848472237586975, + "logits/rejected": -1.9320148229599, + "logps/chosen": -0.2827574610710144, + "logps/rejected": -0.26435643434524536, + "loss": 1.06340754032135, + "loss/core": 1.06340754032135, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8763984441757202, + "rewards/margins": 1.0876319408416748, + "rewards/rejected": 0.7887665629386902, + "step": 35 + }, + { + "epoch": 0.03821808193001314, + "grad_norm": 10.8125, + "learning_rate": 2.1666666666666667e-07, + "logits/chosen": -1.5899121761322021, + "logits/rejected": -1.6173330545425415, + "logps/chosen": -0.2993370592594147, + "logps/rejected": -0.30007094144821167, + "loss": 1.0462119579315186, + "loss/core": 1.0462119579315186, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.3351786136627197, + "rewards/margins": 1.363264799118042, + "rewards/rejected": 0.9719139933586121, + "step": 40 + }, + { + "epoch": 0.04299534217126478, + "grad_norm": 34.75, + "learning_rate": 2.4444444444444445e-07, + "logits/chosen": -1.8352829217910767, + "logits/rejected": -1.8310245275497437, + "logps/chosen": -0.2917270064353943, + "logps/rejected": -0.2925964891910553, + "loss": 1.0427584648132324, + "loss/core": 1.0427584648132324, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.797705888748169, + "rewards/margins": 0.967245876789093, + "rewards/rejected": 1.8304599523544312, + "step": 45 + }, + { + "epoch": 0.04777260241251642, + "grad_norm": 4.25, + "learning_rate": 2.7222222222222216e-07, + "logits/chosen": -1.7508865594863892, + "logits/rejected": -1.819305419921875, + "logps/chosen": -0.2858908176422119, + "logps/rejected": -0.290945827960968, + "loss": 1.0335471630096436, + "loss/core": 1.0335471630096436, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.8175321817398071, + "rewards/margins": 0.4548220634460449, + "rewards/rejected": 1.3627102375030518, + "step": 50 + }, + { + "epoch": 0.05254986265376806, + "grad_norm": 6.375, + "learning_rate": 3e-07, + "logits/chosen": -1.7931705713272095, + "logits/rejected": -1.942427396774292, + "logps/chosen": -0.29617246985435486, + "logps/rejected": -0.2928510010242462, + "loss": 1.0443193912506104, + "loss/core": 1.0443193912506104, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.6830313205718994, + "rewards/margins": 0.7598537802696228, + "rewards/rejected": 0.9231774210929871, + "step": 55 + }, + { + "epoch": 0.057327122895019705, + "grad_norm": 23.75, + "learning_rate": 3.2777777777777776e-07, + "logits/chosen": -1.921984314918518, + "logits/rejected": -1.9806658029556274, + "logps/chosen": -0.2908121645450592, + "logps/rejected": -0.28627273440361023, + "loss": 1.0474377870559692, + "loss/core": 1.0474377870559692, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.452622652053833, + "rewards/margins": 0.369498610496521, + "rewards/rejected": 1.0831242799758911, + "step": 60 + }, + { + "epoch": 0.06210438313627135, + "grad_norm": 35.75, + "learning_rate": 3.5555555555555553e-07, + "logits/chosen": -1.824541449546814, + "logits/rejected": -1.8805049657821655, + "logps/chosen": -0.3165293037891388, + "logps/rejected": -0.3103017210960388, + "loss": 1.0472676753997803, + "loss/core": 1.0472676753997803, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.4452695846557617, + "rewards/margins": 1.5458446741104126, + "rewards/rejected": 0.8994248509407043, + "step": 65 + }, + { + "epoch": 0.06688164337752299, + "grad_norm": 62.25, + "learning_rate": 3.8333333333333335e-07, + "logits/chosen": -1.7961727380752563, + "logits/rejected": -1.8802435398101807, + "logps/chosen": -0.3014804720878601, + "logps/rejected": -0.3035959005355835, + "loss": 1.0426177978515625, + "loss/core": 1.0426177978515625, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.323197841644287, + "rewards/margins": 1.3843536376953125, + "rewards/rejected": 0.9388440847396851, + "step": 70 + }, + { + "epoch": 0.07165890361877464, + "grad_norm": 7.71875, + "learning_rate": 4.1111111111111107e-07, + "logits/chosen": -1.7457157373428345, + "logits/rejected": -1.7732973098754883, + "logps/chosen": -0.29039743542671204, + "logps/rejected": -0.29679733514785767, + "loss": 1.031226396560669, + "loss/core": 1.031226396560669, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.5424000024795532, + "rewards/margins": 0.5238652229309082, + "rewards/rejected": 1.018534779548645, + "step": 75 + }, + { + "epoch": 0.07643616386002627, + "grad_norm": 8.9375, + "learning_rate": 4.3888888888888884e-07, + "logits/chosen": -1.8642539978027344, + "logits/rejected": -1.9320249557495117, + "logps/chosen": -0.3098042905330658, + "logps/rejected": -0.3067478537559509, + "loss": 1.0435469150543213, + "loss/core": 1.0435469150543213, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.2842977046966553, + "rewards/margins": 0.5746160745620728, + "rewards/rejected": 0.7096816301345825, + "step": 80 + }, + { + "epoch": 0.08121342410127792, + "grad_norm": 5.34375, + "learning_rate": 4.6666666666666666e-07, + "logits/chosen": -1.7921772003173828, + "logits/rejected": -1.7714446783065796, + "logps/chosen": -0.2924823760986328, + "logps/rejected": -0.2849774658679962, + "loss": 1.0487070083618164, + "loss/core": 1.0487070083618164, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.4415981769561768, + "rewards/margins": 0.7337201833724976, + "rewards/rejected": 1.7078783512115479, + "step": 85 + }, + { + "epoch": 0.08599068434252956, + "grad_norm": 9.8125, + "learning_rate": 4.944444444444445e-07, + "logits/chosen": -1.7024269104003906, + "logits/rejected": -1.8251234292984009, + "logps/chosen": -0.30760306119918823, + "logps/rejected": -0.30106034874916077, + "loss": 1.0480482578277588, + "loss/core": 1.0480482578277588, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.881970167160034, + "rewards/margins": 2.2146878242492676, + "rewards/rejected": 0.6672819256782532, + "step": 90 + }, + { + "epoch": 0.09076794458378121, + "grad_norm": 29.875, + "learning_rate": 4.999699149323369e-07, + "logits/chosen": -1.787375807762146, + "logits/rejected": -1.7270876169204712, + "logps/chosen": -0.28113657236099243, + "logps/rejected": -0.3039243221282959, + "loss": 1.0137240886688232, + "loss/core": 1.0137240886688232, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.1713666915893555, + "rewards/margins": 0.8056703805923462, + "rewards/rejected": 1.3656964302062988, + "step": 95 + }, + { + "epoch": 0.09554520482503284, + "grad_norm": 5.6875, + "learning_rate": 4.998477067547739e-07, + "logits/chosen": -1.668386697769165, + "logits/rejected": -1.639217734336853, + "logps/chosen": -0.2826381325721741, + "logps/rejected": -0.30979663133621216, + "loss": 1.0171899795532227, + "loss/core": 1.0171899795532227, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.973217725753784, + "rewards/margins": 1.4058278799057007, + "rewards/rejected": 1.567389726638794, + "step": 100 + }, + { + "epoch": 0.10032246506628449, + "grad_norm": 7.84375, + "learning_rate": 4.996315410727229e-07, + "logits/chosen": -1.54190993309021, + "logits/rejected": -1.6111608743667603, + "logps/chosen": -0.27607348561286926, + "logps/rejected": -0.2650708854198456, + "loss": 1.0601215362548828, + "loss/core": 1.0601215362548828, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.1852011680603027, + "rewards/margins": 1.8053677082061768, + "rewards/rejected": 1.3798331022262573, + "step": 105 + }, + { + "epoch": 0.10509972530753613, + "grad_norm": 4.90625, + "learning_rate": 4.993214991772562e-07, + "logits/chosen": -1.7653499841690063, + "logits/rejected": -1.7720153331756592, + "logps/chosen": -0.2952477037906647, + "logps/rejected": -0.3086181879043579, + "loss": 1.0259872674942017, + "loss/core": 1.0259872674942017, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.194749355316162, + "rewards/margins": 1.190367579460144, + "rewards/rejected": 1.0043818950653076, + "step": 110 + }, + { + "epoch": 0.10987698554878778, + "grad_norm": 78.0, + "learning_rate": 4.989176976624511e-07, + "logits/chosen": -1.7536379098892212, + "logits/rejected": -1.7781728506088257, + "logps/chosen": -0.29186949133872986, + "logps/rejected": -0.2834053635597229, + "loss": 1.0515763759613037, + "loss/core": 1.0515763759613037, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.4836719036102295, + "rewards/margins": 2.2081823348999023, + "rewards/rejected": 1.2754894495010376, + "step": 115 + }, + { + "epoch": 0.11465424579003941, + "grad_norm": 34.0, + "learning_rate": 4.984202883815428e-07, + "logits/chosen": -1.9431312084197998, + "logits/rejected": -1.8143190145492554, + "logps/chosen": -0.26071271300315857, + "logps/rejected": -0.2756553292274475, + "loss": 1.0258220434188843, + "loss/core": 1.0258220434188843, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8046910762786865, + "rewards/margins": 0.35107293725013733, + "rewards/rejected": 1.4536181688308716, + "step": 120 + }, + { + "epoch": 0.11943150603129106, + "grad_norm": 4.03125, + "learning_rate": 4.978294583898195e-07, + "logits/chosen": -1.7581638097763062, + "logits/rejected": -1.8171669244766235, + "logps/chosen": -0.2824077606201172, + "logps/rejected": -0.27793461084365845, + "loss": 1.0521032810211182, + "loss/core": 1.0521032810211182, + "rewards/accuracies": 0.875, + "rewards/chosen": 4.071238040924072, + "rewards/margins": 3.047459125518799, + "rewards/rejected": 1.0237786769866943, + "step": 125 + }, + { + "epoch": 0.1242087662725427, + "grad_norm": 91.5, + "learning_rate": 4.971454298742779e-07, + "logits/chosen": -1.738824486732483, + "logits/rejected": -1.773324966430664, + "logps/chosen": -0.29637548327445984, + "logps/rejected": -0.292829692363739, + "loss": 1.0435963869094849, + "loss/core": 1.0435963869094849, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.691030979156494, + "rewards/margins": 0.9570609331130981, + "rewards/rejected": 1.733970284461975, + "step": 130 + }, + { + "epoch": 0.12898602651379434, + "grad_norm": 11.9375, + "learning_rate": 4.963684600700678e-07, + "logits/chosen": -1.7135483026504517, + "logits/rejected": -1.7729370594024658, + "logps/chosen": -0.27682191133499146, + "logps/rejected": -0.26342201232910156, + "loss": 1.0573183298110962, + "loss/core": 1.0573183298110962, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.7214828729629517, + "rewards/margins": 0.8246873617172241, + "rewards/rejected": 0.8967955708503723, + "step": 135 + }, + { + "epoch": 0.13376328675504598, + "grad_norm": 4.9375, + "learning_rate": 4.954988411637571e-07, + "logits/chosen": -1.7595752477645874, + "logits/rejected": -1.805037498474121, + "logps/chosen": -0.2989395260810852, + "logps/rejected": -0.29219749569892883, + "loss": 1.0499845743179321, + "loss/core": 1.0499845743179321, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.22503924369812, + "rewards/margins": 1.3095721006393433, + "rewards/rejected": 0.9154669046401978, + "step": 140 + }, + { + "epoch": 0.13854054699629761, + "grad_norm": 17.25, + "learning_rate": 4.945369001834514e-07, + "logits/chosen": -1.6619952917099, + "logits/rejected": -1.6887853145599365, + "logps/chosen": -0.3317473828792572, + "logps/rejected": -0.31527823209762573, + "loss": 1.0616098642349243, + "loss/core": 1.0616098642349243, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8598458766937256, + "rewards/margins": 0.7766186594963074, + "rewards/rejected": 1.0832273960113525, + "step": 145 + }, + { + "epoch": 0.14331780723754928, + "grad_norm": 6.21875, + "learning_rate": 4.93482998875813e-07, + "logits/chosen": -1.6465572118759155, + "logits/rejected": -1.7042934894561768, + "logps/chosen": -0.310824453830719, + "logps/rejected": -0.3171466886997223, + "loss": 1.0322322845458984, + "loss/core": 1.0322322845458984, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.5014384984970093, + "rewards/margins": 0.929470419883728, + "rewards/rejected": 0.571968138217926, + "step": 150 + }, + { + "epoch": 0.1480950674788009, + "grad_norm": 14.75, + "learning_rate": 4.923375335700223e-07, + "logits/chosen": -1.9649051427841187, + "logits/rejected": -1.9704341888427734, + "logps/chosen": -0.3165343403816223, + "logps/rejected": -0.303557425737381, + "loss": 1.056438684463501, + "loss/core": 1.056438684463501, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.594120740890503, + "rewards/margins": 0.7040458917617798, + "rewards/rejected": 0.8900747299194336, + "step": 155 + }, + { + "epoch": 0.15287232772005255, + "grad_norm": 146.0, + "learning_rate": 4.911009350287347e-07, + "logits/chosen": -1.7473423480987549, + "logits/rejected": -1.7553294897079468, + "logps/chosen": -0.29741325974464417, + "logps/rejected": -0.2803717255592346, + "loss": 1.0651330947875977, + "loss/core": 1.0651330947875977, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.707282543182373, + "rewards/margins": 2.542534351348877, + "rewards/rejected": 1.164747953414917, + "step": 160 + }, + { + "epoch": 0.15764958796130418, + "grad_norm": 9.4375, + "learning_rate": 4.897736682860885e-07, + "logits/chosen": -1.839644193649292, + "logits/rejected": -1.9340137243270874, + "logps/chosen": -0.3177702724933624, + "logps/rejected": -0.3130636215209961, + "loss": 1.049741268157959, + "loss/core": 1.049741268157959, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8438761234283447, + "rewards/margins": 0.8960213661193848, + "rewards/rejected": 0.9478548765182495, + "step": 165 + }, + { + "epoch": 0.16242684820255585, + "grad_norm": 14.375, + "learning_rate": 4.883562324728241e-07, + "logits/chosen": -1.9273979663848877, + "logits/rejected": -2.001987934112549, + "logps/chosen": -0.2734295725822449, + "logps/rejected": -0.2835696339607239, + "loss": 1.0265181064605713, + "loss/core": 1.0265181064605713, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.5538994073867798, + "rewards/margins": 0.5848657488822937, + "rewards/rejected": 0.9690335988998413, + "step": 170 + }, + { + "epoch": 0.16720410844380748, + "grad_norm": 17.5, + "learning_rate": 4.868491606285823e-07, + "logits/chosen": -1.6086976528167725, + "logits/rejected": -1.6097809076309204, + "logps/chosen": -0.2869703769683838, + "logps/rejected": -0.2960742712020874, + "loss": 1.0377720594406128, + "loss/core": 1.0377720594406128, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 5.263535499572754, + "rewards/margins": 3.293330669403076, + "rewards/rejected": 1.9702045917510986, + "step": 175 + }, + { + "epoch": 0.17198136868505912, + "grad_norm": 4.375, + "learning_rate": 4.852530195014489e-07, + "logits/chosen": -1.8328279256820679, + "logits/rejected": -1.9173707962036133, + "logps/chosen": -0.28484848141670227, + "logps/rejected": -0.2925505042076111, + "loss": 1.0417985916137695, + "loss/core": 1.0417985916137695, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 3.295905590057373, + "rewards/margins": 1.7848495244979858, + "rewards/rejected": 1.5110561847686768, + "step": 180 + }, + { + "epoch": 0.17675862892631075, + "grad_norm": 6.59375, + "learning_rate": 4.835684093348244e-07, + "logits/chosen": -1.7860227823257446, + "logits/rejected": -1.84549880027771, + "logps/chosen": -0.25924015045166016, + "logps/rejected": -0.26765456795692444, + "loss": 1.0282881259918213, + "loss/core": 1.0282881259918213, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9171737432479858, + "rewards/margins": 1.1408088207244873, + "rewards/rejected": 0.7763648629188538, + "step": 185 + }, + { + "epoch": 0.18153588916756241, + "grad_norm": 5.3125, + "learning_rate": 4.817959636416969e-07, + "logits/chosen": -1.6373240947723389, + "logits/rejected": -1.6969833374023438, + "logps/chosen": -0.3073750138282776, + "logps/rejected": -0.30846190452575684, + "loss": 1.0419862270355225, + "loss/core": 1.0419862270355225, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.9135539531707764, + "rewards/margins": 1.0026248693466187, + "rewards/rejected": 0.9109293222427368, + "step": 190 + }, + { + "epoch": 0.18631314940881405, + "grad_norm": 6.9375, + "learning_rate": 4.799363489664039e-07, + "logits/chosen": -1.808831810951233, + "logits/rejected": -1.7658703327178955, + "logps/chosen": -0.2862377464771271, + "logps/rejected": -0.3009548783302307, + "loss": 1.0329413414001465, + "loss/core": 1.0329413414001465, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.1667561531066895, + "rewards/margins": 0.9283055067062378, + "rewards/rejected": 1.2384504079818726, + "step": 195 + }, + { + "epoch": 0.19109040965006568, + "grad_norm": 24.875, + "learning_rate": 4.779902646339721e-07, + "logits/chosen": -1.722137451171875, + "logits/rejected": -1.6766093969345093, + "logps/chosen": -0.29227524995803833, + "logps/rejected": -0.289249062538147, + "loss": 1.0445283651351929, + "loss/core": 1.0445283651351929, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.8198704719543457, + "rewards/margins": 1.2550045251846313, + "rewards/rejected": 1.5648664236068726, + "step": 200 + }, + { + "epoch": 0.19586766989131732, + "grad_norm": 18.625, + "learning_rate": 4.759584424871301e-07, + "logits/chosen": -1.7560946941375732, + "logits/rejected": -1.7568002939224243, + "logps/chosen": -0.3940495252609253, + "logps/rejected": -0.3073170483112335, + "loss": 1.2059910297393799, + "loss/core": 1.2059910297393799, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.591503620147705, + "rewards/margins": 1.2615159749984741, + "rewards/rejected": 1.329987645149231, + "step": 205 + }, + { + "epoch": 0.20064493013256898, + "grad_norm": 5.125, + "learning_rate": 4.738416466110917e-07, + "logits/chosen": -1.9233392477035522, + "logits/rejected": -2.00313663482666, + "logps/chosen": -0.25894150137901306, + "logps/rejected": -0.2547500729560852, + "loss": 1.0453473329544067, + "loss/core": 1.0453473329544067, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.7126867771148682, + "rewards/margins": 0.995030403137207, + "rewards/rejected": 0.7176561951637268, + "step": 210 + }, + { + "epoch": 0.20542219037382062, + "grad_norm": 51.75, + "learning_rate": 4.716406730462153e-07, + "logits/chosen": -1.8582137823104858, + "logits/rejected": -1.8355731964111328, + "logps/chosen": -0.28673282265663147, + "logps/rejected": -0.2773427367210388, + "loss": 1.0536106824874878, + "loss/core": 1.0536106824874878, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.7520334720611572, + "rewards/margins": 1.0252889394760132, + "rewards/rejected": 1.7267444133758545, + "step": 215 + }, + { + "epoch": 0.21019945061507225, + "grad_norm": 5.46875, + "learning_rate": 4.693563494886454e-07, + "logits/chosen": -1.6601266860961914, + "logits/rejected": -1.832810640335083, + "logps/chosen": -0.2959999144077301, + "logps/rejected": -0.28008145093917847, + "loss": 1.061637043952942, + "loss/core": 1.061637043952942, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.475043535232544, + "rewards/margins": 2.701432466506958, + "rewards/rejected": 0.7736114263534546, + "step": 220 + }, + { + "epoch": 0.2149767108563239, + "grad_norm": 35.0, + "learning_rate": 4.6698953497905016e-07, + "logits/chosen": -1.826503038406372, + "logits/rejected": -1.9233640432357788, + "logps/chosen": -0.27310609817504883, + "logps/rejected": -0.27189555764198303, + "loss": 1.040917158126831, + "loss/core": 1.040917158126831, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.2421226501464844, + "rewards/margins": 2.2241339683532715, + "rewards/rejected": 1.017989158630371, + "step": 225 + }, + { + "epoch": 0.21975397109757555, + "grad_norm": 124.0, + "learning_rate": 4.645411195795709e-07, + "logits/chosen": -1.8059947490692139, + "logits/rejected": -1.8729541301727295, + "logps/chosen": -0.25209683179855347, + "logps/rejected": -0.2518470287322998, + "loss": 1.0411286354064941, + "loss/core": 1.0411286354064941, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.6358604431152344, + "rewards/margins": 2.28558087348938, + "rewards/rejected": 1.3502795696258545, + "step": 230 + }, + { + "epoch": 0.2245312313388272, + "grad_norm": 5.90625, + "learning_rate": 4.6201202403910643e-07, + "logits/chosen": -2.0312843322753906, + "logits/rejected": -2.0044198036193848, + "logps/chosen": -0.2595166563987732, + "logps/rejected": -0.2711992859840393, + "loss": 1.0278807878494263, + "loss/core": 1.0278807878494263, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.072014331817627, + "rewards/margins": 0.8968809247016907, + "rewards/rejected": 1.1751338243484497, + "step": 235 + }, + { + "epoch": 0.22930849158007882, + "grad_norm": 14.6875, + "learning_rate": 4.594031994470573e-07, + "logits/chosen": -1.8727977275848389, + "logits/rejected": -1.8875232934951782, + "logps/chosen": -0.27784401178359985, + "logps/rejected": -0.29739147424697876, + "loss": 1.0186278820037842, + "loss/core": 1.0186278820037842, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.4038894176483154, + "rewards/margins": 1.5082296133041382, + "rewards/rejected": 1.8956598043441772, + "step": 240 + }, + { + "epoch": 0.23408575182133046, + "grad_norm": 71.5, + "learning_rate": 4.567156268756593e-07, + "logits/chosen": -1.5276188850402832, + "logits/rejected": -1.57925283908844, + "logps/chosen": -0.2935658097267151, + "logps/rejected": -0.3003990352153778, + "loss": 1.0357589721679688, + "loss/core": 1.0357589721679688, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.293689250946045, + "rewards/margins": 1.7028945684432983, + "rewards/rejected": 1.5907951593399048, + "step": 245 + }, + { + "epoch": 0.23886301206258212, + "grad_norm": 4.125, + "learning_rate": 4.5395031701104303e-07, + "logits/chosen": -1.9313945770263672, + "logits/rejected": -1.8916925191879272, + "logps/chosen": -0.2886388301849365, + "logps/rejected": -0.28957533836364746, + "loss": 1.037282943725586, + "loss/core": 1.037282943725586, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.5945872068405151, + "rewards/margins": 0.4880189895629883, + "rewards/rejected": 1.1065683364868164, + "step": 250 + }, + { + "epoch": 0.24364027230383375, + "grad_norm": 3.671875, + "learning_rate": 4.511083097731555e-07, + "logits/chosen": -1.8803619146347046, + "logits/rejected": -1.8885911703109741, + "logps/chosen": -0.30258265137672424, + "logps/rejected": -0.30648648738861084, + "loss": 1.0346689224243164, + "loss/core": 1.0346689224243164, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 1.496037483215332, + "rewards/margins": 0.9408340454101562, + "rewards/rejected": 0.555203378200531, + "step": 255 + }, + { + "epoch": 0.2484175325450854, + "grad_norm": 4.59375, + "learning_rate": 4.481906739246894e-07, + "logits/chosen": -1.963661551475525, + "logits/rejected": -1.9137052297592163, + "logps/chosen": -0.27444905042648315, + "logps/rejected": -0.27626100182533264, + "loss": 1.0373988151550293, + "loss/core": 1.0373988151550293, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.3847882747650146, + "rewards/margins": 1.301588773727417, + "rewards/rejected": 1.0831997394561768, + "step": 260 + }, + { + "epoch": 0.25319479278633705, + "grad_norm": 5.75, + "learning_rate": 4.451985066691648e-07, + "logits/chosen": -2.0017013549804688, + "logits/rejected": -2.0215392112731934, + "logps/chosen": -0.2834641933441162, + "logps/rejected": -0.2866167724132538, + "loss": 1.0352154970169067, + "loss/core": 1.0352154970169067, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.4251301288604736, + "rewards/margins": 0.7156028151512146, + "rewards/rejected": 0.709527313709259, + "step": 265 + }, + { + "epoch": 0.2579720530275887, + "grad_norm": 12.0, + "learning_rate": 4.421329332383158e-07, + "logits/chosen": -1.7916221618652344, + "logits/rejected": -1.9190409183502197, + "logps/chosen": -0.3161014914512634, + "logps/rejected": -0.293036550283432, + "loss": 1.0780491828918457, + "loss/core": 1.0780491828918457, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8342010974884033, + "rewards/margins": 1.156964898109436, + "rewards/rejected": 0.677236020565033, + "step": 270 + }, + { + "epoch": 0.2627493132688403, + "grad_norm": 10.6875, + "learning_rate": 4.3899510646893696e-07, + "logits/chosen": -1.8899133205413818, + "logits/rejected": -1.8713350296020508, + "logps/chosen": -0.2559998631477356, + "logps/rejected": -0.2608237862586975, + "loss": 1.0367724895477295, + "loss/core": 1.0367724895477295, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.4705395698547363, + "rewards/margins": 1.2723934650421143, + "rewards/rejected": 1.1981456279754639, + "step": 275 + }, + { + "epoch": 0.26752657351009196, + "grad_norm": 22.75, + "learning_rate": 4.357862063693485e-07, + "logits/chosen": -1.6495206356048584, + "logits/rejected": -1.69220769405365, + "logps/chosen": -0.3172406554222107, + "logps/rejected": -0.32189950346946716, + "loss": 1.0348529815673828, + "loss/core": 1.0348529815673828, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.4244751930236816, + "rewards/margins": 1.04317307472229, + "rewards/rejected": 1.3813021183013916, + "step": 280 + }, + { + "epoch": 0.2723038337513436, + "grad_norm": 18.625, + "learning_rate": 4.3250743967564364e-07, + "logits/chosen": -1.7316668033599854, + "logits/rejected": -1.6830451488494873, + "logps/chosen": -0.26700717210769653, + "logps/rejected": -0.2771323621273041, + "loss": 1.0284992456436157, + "loss/core": 1.0284992456436157, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.8682186603546143, + "rewards/margins": 1.2967889308929443, + "rewards/rejected": 1.5714294910430908, + "step": 285 + }, + { + "epoch": 0.27708109399259523, + "grad_norm": 10.9375, + "learning_rate": 4.29160039397884e-07, + "logits/chosen": -1.8204319477081299, + "logits/rejected": -1.9146995544433594, + "logps/chosen": -0.29401618242263794, + "logps/rejected": -0.2924644351005554, + "loss": 1.0413658618927002, + "loss/core": 1.0413658618927002, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.0374536514282227, + "rewards/margins": 1.1444836854934692, + "rewards/rejected": 0.8929699063301086, + "step": 290 + }, + { + "epoch": 0.28185835423384686, + "grad_norm": 68.0, + "learning_rate": 4.2574526435641546e-07, + "logits/chosen": -1.6919562816619873, + "logits/rejected": -1.6904808282852173, + "logps/chosen": -0.27512839436531067, + "logps/rejected": -0.2824132442474365, + "loss": 1.033488154411316, + "loss/core": 1.033488154411316, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.2550134658813477, + "rewards/margins": 0.5635195970535278, + "rewards/rejected": 1.6914937496185303, + "step": 295 + }, + { + "epoch": 0.28663561447509855, + "grad_norm": 14.5, + "learning_rate": 4.22264398708477e-07, + "logits/chosen": -1.6543853282928467, + "logits/rejected": -1.6278355121612549, + "logps/chosen": -0.2689480185508728, + "logps/rejected": -0.28801828622817993, + "loss": 1.0174682140350342, + "loss/core": 1.0174682140350342, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.770953416824341, + "rewards/margins": 1.4299379587173462, + "rewards/rejected": 1.3410152196884155, + "step": 300 + }, + { + "epoch": 0.2914128747163502, + "grad_norm": 19.0, + "learning_rate": 4.187187514652819e-07, + "logits/chosen": -1.756967544555664, + "logits/rejected": -1.7581822872161865, + "logps/chosen": -0.2890106439590454, + "logps/rejected": -0.30179038643836975, + "loss": 1.028267502784729, + "loss/core": 1.028267502784729, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.826168417930603, + "rewards/margins": 0.5155857801437378, + "rewards/rejected": 1.3105827569961548, + "step": 305 + }, + { + "epoch": 0.2961901349576018, + "grad_norm": 31.5, + "learning_rate": 4.151096559997519e-07, + "logits/chosen": -1.8081289529800415, + "logits/rejected": -1.7069460153579712, + "logps/chosen": -0.2982190251350403, + "logps/rejected": -0.3032774031162262, + "loss": 1.0330473184585571, + "loss/core": 1.0330473184585571, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.1015119552612305, + "rewards/margins": 0.7455414533615112, + "rewards/rejected": 1.3559703826904297, + "step": 310 + }, + { + "epoch": 0.30096739519885346, + "grad_norm": 6.1875, + "learning_rate": 4.114384695450905e-07, + "logits/chosen": -1.9606285095214844, + "logits/rejected": -1.9415584802627563, + "logps/chosen": -0.2907028794288635, + "logps/rejected": -0.28985944390296936, + "loss": 1.0457550287246704, + "loss/core": 1.0457550287246704, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.5825884342193604, + "rewards/margins": 0.5970422029495239, + "rewards/rejected": 0.9855462312698364, + "step": 315 + }, + { + "epoch": 0.3057446554401051, + "grad_norm": 6.625, + "learning_rate": 4.077065726843828e-07, + "logits/chosen": -1.654712438583374, + "logits/rejected": -1.5861515998840332, + "logps/chosen": -0.28932780027389526, + "logps/rejected": -0.276142954826355, + "loss": 1.0589449405670166, + "loss/core": 1.0589449405670166, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.8658580780029297, + "rewards/margins": 0.9593901634216309, + "rewards/rejected": 1.9064680337905884, + "step": 320 + }, + { + "epoch": 0.31052191568135673, + "grad_norm": 8.9375, + "learning_rate": 4.039153688314145e-07, + "logits/chosen": -1.6774059534072876, + "logits/rejected": -1.7040441036224365, + "logps/chosen": -0.29178744554519653, + "logps/rejected": -0.2866431474685669, + "loss": 1.0456210374832153, + "loss/core": 1.0456210374832153, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.5071741342544556, + "rewards/margins": -0.009084177203476429, + "rewards/rejected": 1.5162582397460938, + "step": 325 + }, + { + "epoch": 0.31529917592260837, + "grad_norm": 12.6875, + "learning_rate": 4.0006628370290613e-07, + "logits/chosen": -1.8318054676055908, + "logits/rejected": -1.81647527217865, + "logps/chosen": -0.2811618745326996, + "logps/rejected": -0.28358012437820435, + "loss": 1.03645920753479, + "loss/core": 1.03645920753479, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.583585023880005, + "rewards/margins": 1.4332873821258545, + "rewards/rejected": 1.15029776096344, + "step": 330 + }, + { + "epoch": 0.32007643616386, + "grad_norm": 40.5, + "learning_rate": 3.9616076478235826e-07, + "logits/chosen": -1.6614739894866943, + "logits/rejected": -1.6278941631317139, + "logps/chosen": -0.28862547874450684, + "logps/rejected": -0.2881496548652649, + "loss": 1.0447041988372803, + "loss/core": 1.0447041988372803, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.6951169967651367, + "rewards/margins": 1.3128235340118408, + "rewards/rejected": 1.382293462753296, + "step": 335 + }, + { + "epoch": 0.3248536964051117, + "grad_norm": 11.125, + "learning_rate": 3.922002807757129e-07, + "logits/chosen": -1.8974721431732178, + "logits/rejected": -1.9225714206695557, + "logps/chosen": -0.29584866762161255, + "logps/rejected": -0.3060780167579651, + "loss": 1.0265626907348633, + "loss/core": 1.0265626907348633, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.664690613746643, + "rewards/margins": 0.8293659090995789, + "rewards/rejected": 0.8353245854377747, + "step": 340 + }, + { + "epoch": 0.3296309566463633, + "grad_norm": 13.875, + "learning_rate": 3.8818632105903315e-07, + "logits/chosen": -1.673317551612854, + "logits/rejected": -1.7775061130523682, + "logps/chosen": -0.2768518328666687, + "logps/rejected": -0.2922056317329407, + "loss": 1.0234686136245728, + "loss/core": 1.0234686136245728, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.881852626800537, + "rewards/margins": 0.5690313577651978, + "rewards/rejected": 2.31282114982605, + "step": 345 + }, + { + "epoch": 0.33440821688761496, + "grad_norm": 24.5, + "learning_rate": 3.841203951184094e-07, + "logits/chosen": -1.9435956478118896, + "logits/rejected": -1.8745100498199463, + "logps/chosen": -0.28658443689346313, + "logps/rejected": -0.2910448908805847, + "loss": 1.0343317985534668, + "loss/core": 1.0343317985534668, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.736780881881714, + "rewards/margins": 2.516613483428955, + "rewards/rejected": 1.220167636871338, + "step": 350 + }, + { + "epoch": 0.3391854771288666, + "grad_norm": 4.6875, + "learning_rate": 3.800040319823038e-07, + "logits/chosen": -1.682003378868103, + "logits/rejected": -1.7151581048965454, + "logps/chosen": -0.3114383816719055, + "logps/rejected": -0.3123466372489929, + "loss": 1.043797492980957, + "loss/core": 1.043797492980957, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 1.962998390197754, + "rewards/margins": 1.1944884061813354, + "rewards/rejected": 0.7685100436210632, + "step": 355 + }, + { + "epoch": 0.34396273737011823, + "grad_norm": 26.875, + "learning_rate": 3.75838779646545e-07, + "logits/chosen": -1.719026803970337, + "logits/rejected": -1.6863863468170166, + "logps/chosen": -0.31425201892852783, + "logps/rejected": -0.32097968459129333, + "loss": 1.0321420431137085, + "loss/core": 1.0321420431137085, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.7521181106567383, + "rewards/margins": -0.033901017159223557, + "rewards/rejected": 1.7860190868377686, + "step": 360 + }, + { + "epoch": 0.34873999761136987, + "grad_norm": 27.75, + "learning_rate": 3.7162620449218993e-07, + "logits/chosen": -1.8344494104385376, + "logits/rejected": -1.8459513187408447, + "logps/chosen": -0.29098817706108093, + "logps/rejected": -0.28455573320388794, + "loss": 1.0480910539627075, + "loss/core": 1.0480910539627075, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.697824478149414, + "rewards/margins": 2.8228657245635986, + "rewards/rejected": 0.8749583959579468, + "step": 365 + }, + { + "epoch": 0.3535172578526215, + "grad_norm": 10.75, + "learning_rate": 3.673678906964727e-07, + "logits/chosen": -1.622668981552124, + "logits/rejected": -1.6612154245376587, + "logps/chosen": -0.2679634094238281, + "logps/rejected": -0.2821475863456726, + "loss": 1.0219076871871948, + "loss/core": 1.0219076871871948, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.9509494304656982, + "rewards/margins": 1.733298659324646, + "rewards/rejected": 1.2176507711410522, + "step": 370 + }, + { + "epoch": 0.35829451809387314, + "grad_norm": 14.25, + "learning_rate": 3.6306543963705936e-07, + "logits/chosen": -1.685569405555725, + "logits/rejected": -1.6597143411636353, + "logps/chosen": -0.30852535367012024, + "logps/rejected": -0.2950063943862915, + "loss": 1.0666342973709106, + "loss/core": 1.0666342973709106, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 4.885121822357178, + "rewards/margins": 2.607081174850464, + "rewards/rejected": 2.278040647506714, + "step": 375 + }, + { + "epoch": 0.36307177833512483, + "grad_norm": 24.375, + "learning_rate": 3.5872046928983623e-07, + "logits/chosen": -1.7393958568572998, + "logits/rejected": -1.839385747909546, + "logps/chosen": -0.2777525782585144, + "logps/rejected": -0.28187328577041626, + "loss": 1.037045955657959, + "loss/core": 1.037045955657959, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.303683042526245, + "rewards/margins": 1.0037339925765991, + "rewards/rejected": 1.299949049949646, + "step": 380 + }, + { + "epoch": 0.36784903857637646, + "grad_norm": 17.875, + "learning_rate": 3.5433461362045447e-07, + "logits/chosen": -1.6332000494003296, + "logits/rejected": -1.6447935104370117, + "logps/chosen": -0.3114931285381317, + "logps/rejected": -0.28601643443107605, + "loss": 1.073043704032898, + "loss/core": 1.073043704032898, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.6627743244171143, + "rewards/margins": 0.6046234369277954, + "rewards/rejected": 1.0581510066986084, + "step": 385 + }, + { + "epoch": 0.3726262988176281, + "grad_norm": 5.6875, + "learning_rate": 3.4990952196986305e-07, + "logits/chosen": -1.9358383417129517, + "logits/rejected": -1.9621446132659912, + "logps/chosen": -0.27814939618110657, + "logps/rejected": -0.28488942980766296, + "loss": 1.0306986570358276, + "loss/core": 1.0306986570358276, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.2591309547424316, + "rewards/margins": 1.023651361465454, + "rewards/rejected": 1.2354795932769775, + "step": 390 + }, + { + "epoch": 0.37740355905887973, + "grad_norm": 6.875, + "learning_rate": 3.4544685843405875e-07, + "logits/chosen": -1.9065141677856445, + "logits/rejected": -1.8645994663238525, + "logps/chosen": -0.3078649640083313, + "logps/rejected": -0.302230566740036, + "loss": 1.0477346181869507, + "loss/core": 1.0477346181869507, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.9888968467712402, + "rewards/margins": 1.2866131067276, + "rewards/rejected": 1.7022842168807983, + "step": 395 + }, + { + "epoch": 0.38218081930013137, + "grad_norm": 4.15625, + "learning_rate": 3.4094830123828786e-07, + "logits/chosen": -1.7724406719207764, + "logits/rejected": -1.9576416015625, + "logps/chosen": -0.2747136950492859, + "logps/rejected": -0.2948540151119232, + "loss": 1.0195709466934204, + "loss/core": 1.0195709466934204, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.7523436546325684, + "rewards/margins": 0.6447616815567017, + "rewards/rejected": 2.1075825691223145, + "step": 400 + }, + { + "epoch": 0.386958079541383, + "grad_norm": 42.0, + "learning_rate": 3.3641554210593414e-07, + "logits/chosen": -1.838844895362854, + "logits/rejected": -1.8915517330169678, + "logps/chosen": -0.32020050287246704, + "logps/rejected": -0.28831055760383606, + "loss": 1.0958722829818726, + "loss/core": 1.0958722829818726, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.624875545501709, + "rewards/margins": 1.3493173122406006, + "rewards/rejected": 1.2755582332611084, + "step": 405 + }, + { + "epoch": 0.39173533978263464, + "grad_norm": 5.5, + "learning_rate": 3.3185028562233107e-07, + "logits/chosen": -1.8786590099334717, + "logits/rejected": -1.8944612741470337, + "logps/chosen": -0.26763150095939636, + "logps/rejected": -0.2702597975730896, + "loss": 1.038240671157837, + "loss/core": 1.038240671157837, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.777128219604492, + "rewards/margins": 1.4700839519500732, + "rewards/rejected": 1.3070439100265503, + "step": 410 + }, + { + "epoch": 0.3965126000238863, + "grad_norm": 50.0, + "learning_rate": 3.272542485937368e-07, + "logits/chosen": -1.6480621099472046, + "logits/rejected": -1.7010635137557983, + "logps/chosen": -0.2993305027484894, + "logps/rejected": -0.2903633117675781, + "loss": 1.056257963180542, + "loss/core": 1.056257963180542, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.3192853927612305, + "rewards/margins": 1.6964190006256104, + "rewards/rejected": 0.6228662729263306, + "step": 415 + }, + { + "epoch": 0.40128986026513797, + "grad_norm": 8.625, + "learning_rate": 3.226291594017137e-07, + "logits/chosen": -1.556668996810913, + "logits/rejected": -1.5511045455932617, + "logps/chosen": -0.2851598262786865, + "logps/rejected": -0.288934588432312, + "loss": 1.0358564853668213, + "loss/core": 1.0358564853668213, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.2491157054901123, + "rewards/margins": 1.3990309238433838, + "rewards/rejected": 1.850084900856018, + "step": 420 + }, + { + "epoch": 0.4060671205063896, + "grad_norm": 18.625, + "learning_rate": 3.1797675735315454e-07, + "logits/chosen": -1.6744778156280518, + "logits/rejected": -1.7476119995117188, + "logps/chosen": -0.2897992730140686, + "logps/rejected": -0.2777308523654938, + "loss": 1.0568740367889404, + "loss/core": 1.0568740367889404, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.9222044944763184, + "rewards/margins": 0.9556252360343933, + "rewards/rejected": 1.9665788412094116, + "step": 425 + }, + { + "epoch": 0.41084438074764124, + "grad_norm": 6.375, + "learning_rate": 3.1329879202620047e-07, + "logits/chosen": -1.8092721700668335, + "logits/rejected": -1.8535629510879517, + "logps/chosen": -0.3078886866569519, + "logps/rejected": -0.30679917335510254, + "loss": 1.04254150390625, + "loss/core": 1.04254150390625, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.51043438911438, + "rewards/margins": 1.6923158168792725, + "rewards/rejected": 0.818118691444397, + "step": 430 + }, + { + "epoch": 0.41562164098889287, + "grad_norm": 15.1875, + "learning_rate": 3.0859702261229613e-07, + "logits/chosen": -1.84933602809906, + "logits/rejected": -1.77957284450531, + "logps/chosen": -0.28597092628479004, + "logps/rejected": -0.2642970383167267, + "loss": 1.0709216594696045, + "loss/core": 1.0709216594696045, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.4492688179016113, + "rewards/margins": 1.2216014862060547, + "rewards/rejected": 1.2276675701141357, + "step": 435 + }, + { + "epoch": 0.4203989012301445, + "grad_norm": 44.5, + "learning_rate": 3.0387321725463e-07, + "logits/chosen": -1.6569089889526367, + "logits/rejected": -1.6702871322631836, + "logps/chosen": -0.31207168102264404, + "logps/rejected": -0.3157859444618225, + "loss": 1.0342601537704468, + "loss/core": 1.0342601537704468, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.2018513679504395, + "rewards/margins": 1.3161648511886597, + "rewards/rejected": 0.8856865167617798, + "step": 440 + }, + { + "epoch": 0.42517616147139614, + "grad_norm": 79.0, + "learning_rate": 2.991291523832075e-07, + "logits/chosen": -1.7703174352645874, + "logits/rejected": -1.7698230743408203, + "logps/chosen": -0.2817544937133789, + "logps/rejected": -0.2641424536705017, + "loss": 1.0657778978347778, + "loss/core": 1.0657778978347778, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.6899399757385254, + "rewards/margins": 1.9449743032455444, + "rewards/rejected": 1.7449655532836914, + "step": 445 + }, + { + "epoch": 0.4299534217126478, + "grad_norm": 7.625, + "learning_rate": 2.943666120468088e-07, + "logits/chosen": -1.723353624343872, + "logits/rejected": -1.7594738006591797, + "logps/chosen": -0.30141833424568176, + "logps/rejected": -0.2993377447128296, + "loss": 1.0418169498443604, + "loss/core": 1.0418169498443604, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9154335260391235, + "rewards/margins": 1.0606777667999268, + "rewards/rejected": 0.8547555804252625, + "step": 450 + }, + { + "epoch": 0.4347306819538994, + "grad_norm": 14.125, + "learning_rate": 2.8958738724208073e-07, + "logits/chosen": -1.8625282049179077, + "logits/rejected": -1.9700651168823242, + "logps/chosen": -0.26571571826934814, + "logps/rejected": -0.2809098958969116, + "loss": 1.0212137699127197, + "loss/core": 1.0212137699127197, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.1734156608581543, + "rewards/margins": 0.8710582852363586, + "rewards/rejected": 1.30235755443573, + "step": 455 + }, + { + "epoch": 0.4395079421951511, + "grad_norm": 7.1875, + "learning_rate": 2.8479327524001633e-07, + "logits/chosen": -1.8570115566253662, + "logits/rejected": -1.7697608470916748, + "logps/chosen": -0.2919974625110626, + "logps/rejected": -0.29211685061454773, + "loss": 1.039764165878296, + "loss/core": 1.039764165878296, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.0760490894317627, + "rewards/margins": 1.1579499244689941, + "rewards/rejected": 0.9180992245674133, + "step": 460 + }, + { + "epoch": 0.44428520243640274, + "grad_norm": 18.375, + "learning_rate": 2.7998607891007493e-07, + "logits/chosen": -2.035067081451416, + "logits/rejected": -2.0927367210388184, + "logps/chosen": -0.3042372763156891, + "logps/rejected": -0.3184296488761902, + "loss": 1.069641351699829, + "loss/core": 1.069641351699829, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.6732687950134277, + "rewards/margins": 1.3037307262420654, + "rewards/rejected": 1.3695380687713623, + "step": 465 + }, + { + "epoch": 0.4490624626776544, + "grad_norm": 8.875, + "learning_rate": 2.7516760604219616e-07, + "logits/chosen": -1.872999906539917, + "logits/rejected": -1.875903844833374, + "logps/chosen": -0.296085923910141, + "logps/rejected": -0.2649732828140259, + "loss": 1.0810202360153198, + "loss/core": 1.0810202360153198, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.5348351001739502, + "rewards/margins": 0.5164709091186523, + "rewards/rejected": 1.0183641910552979, + "step": 470 + }, + { + "epoch": 0.453839722918906, + "grad_norm": 5.25, + "learning_rate": 2.703396686669646e-07, + "logits/chosen": -1.8748611211776733, + "logits/rejected": -1.8687400817871094, + "logps/chosen": -0.3014170825481415, + "logps/rejected": -0.2969133257865906, + "loss": 1.044512391090393, + "loss/core": 1.044512391090393, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.9237457513809204, + "rewards/margins": 1.39292573928833, + "rewards/rejected": 0.5308200120925903, + "step": 475 + }, + { + "epoch": 0.45861698316015764, + "grad_norm": 44.5, + "learning_rate": 2.6550408237417884e-07, + "logits/chosen": -1.7492471933364868, + "logits/rejected": -1.7116022109985352, + "logps/chosen": -0.3013203740119934, + "logps/rejected": -0.31782329082489014, + "loss": 1.023486852645874, + "loss/core": 1.023486852645874, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.455150604248047, + "rewards/margins": 1.1551944017410278, + "rewards/rejected": 1.2999564409255981, + "step": 480 + }, + { + "epoch": 0.4633942434014093, + "grad_norm": 30.75, + "learning_rate": 2.6066266563008265e-07, + "logits/chosen": -1.691613793373108, + "logits/rejected": -1.6590354442596436, + "logps/chosen": -0.2971816658973694, + "logps/rejected": -0.29611891508102417, + "loss": 1.040845513343811, + "loss/core": 1.040845513343811, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.0947299003601074, + "rewards/margins": 1.0051168203353882, + "rewards/rejected": 1.0896131992340088, + "step": 485 + }, + { + "epoch": 0.4681715036426609, + "grad_norm": 9.125, + "learning_rate": 2.5581723909351404e-07, + "logits/chosen": -1.6561720371246338, + "logits/rejected": -1.6545698642730713, + "logps/chosen": -0.3090967535972595, + "logps/rejected": -0.30728867650032043, + "loss": 1.0426571369171143, + "loss/core": 1.0426571369171143, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.3853068351745605, + "rewards/margins": 1.2928699254989624, + "rewards/rejected": 1.0924367904663086, + "step": 490 + }, + { + "epoch": 0.47294876388391255, + "grad_norm": 12.1875, + "learning_rate": 2.509696249312301e-07, + "logits/chosen": -1.7215149402618408, + "logits/rejected": -1.759275197982788, + "logps/chosen": -0.28365054726600647, + "logps/rejected": -0.2850833535194397, + "loss": 1.0367029905319214, + "loss/core": 1.0367029905319214, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.926408052444458, + "rewards/margins": 0.9314010739326477, + "rewards/rejected": 0.9950069189071655, + "step": 495 + }, + { + "epoch": 0.47772602412516424, + "grad_norm": 63.25, + "learning_rate": 2.461216461326642e-07, + "logits/chosen": -1.6816461086273193, + "logits/rejected": -1.7013599872589111, + "logps/chosen": -0.2813529670238495, + "logps/rejected": -0.28367528319358826, + "loss": 1.0371912717819214, + "loss/core": 1.0371912717819214, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.650829315185547, + "rewards/margins": 1.9984947443008423, + "rewards/rejected": 1.6523345708847046, + "step": 500 + }, + { + "epoch": 0.4825032843664159, + "grad_norm": 8.1875, + "learning_rate": 2.412751258243748e-07, + "logits/chosen": -1.6414563655853271, + "logits/rejected": -1.585834264755249, + "logps/chosen": -0.3117072582244873, + "logps/rejected": -0.3133184313774109, + "loss": 1.0436599254608154, + "loss/core": 1.0436599254608154, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.4120712280273438, + "rewards/margins": 2.0417308807373047, + "rewards/rejected": 1.3703404664993286, + "step": 505 + }, + { + "epoch": 0.4872805446076675, + "grad_norm": 10.1875, + "learning_rate": 2.3643188658444158e-07, + "logits/chosen": -1.901231050491333, + "logits/rejected": -1.9024569988250732, + "logps/chosen": -0.29130491614341736, + "logps/rejected": -0.28371134400367737, + "loss": 1.05287504196167, + "loss/core": 1.05287504196167, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.3334412574768066, + "rewards/margins": 1.1235178709030151, + "rewards/rejected": 1.209923505783081, + "step": 510 + }, + { + "epoch": 0.49205780484891914, + "grad_norm": 28.375, + "learning_rate": 2.315937497570688e-07, + "logits/chosen": -1.5855246782302856, + "logits/rejected": -1.6642907857894897, + "logps/chosen": -0.2806105613708496, + "logps/rejected": -0.2813439965248108, + "loss": 1.0392553806304932, + "loss/core": 1.0392553806304932, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.8368934392929077, + "rewards/margins": 0.04681912809610367, + "rewards/rejected": 1.7900745868682861, + "step": 515 + }, + { + "epoch": 0.4968350650901708, + "grad_norm": 19.875, + "learning_rate": 2.2676253476765194e-07, + "logits/chosen": -1.7813889980316162, + "logits/rejected": -1.7595758438110352, + "logps/chosen": -0.29206109046936035, + "logps/rejected": -0.28826069831848145, + "loss": 1.0510475635528564, + "loss/core": 1.0510475635528564, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.1681056022644043, + "rewards/margins": 1.8101484775543213, + "rewards/rejected": 1.357957363128662, + "step": 520 + }, + { + "epoch": 0.5016123253314224, + "grad_norm": 6.0625, + "learning_rate": 2.2194005843856633e-07, + "logits/chosen": -1.7787355184555054, + "logits/rejected": -1.8631689548492432, + "logps/chosen": -0.2766266465187073, + "logps/rejected": -0.27404943108558655, + "loss": 1.042309045791626, + "loss/core": 1.042309045791626, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.1334118843078613, + "rewards/margins": 1.3973788022994995, + "rewards/rejected": 0.7360331416130066, + "step": 525 + }, + { + "epoch": 0.5063895855726741, + "grad_norm": 5.28125, + "learning_rate": 2.1712813430593434e-07, + "logits/chosen": -1.885209321975708, + "logits/rejected": -1.9635217189788818, + "logps/chosen": -0.28333884477615356, + "logps/rejected": -0.29169341921806335, + "loss": 1.0290471315383911, + "loss/core": 1.0290471315383911, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.6382904052734375, + "rewards/margins": 0.8100762367248535, + "rewards/rejected": 0.8282139897346497, + "step": 530 + }, + { + "epoch": 0.5111668458139257, + "grad_norm": 5.5625, + "learning_rate": 2.123285719376292e-07, + "logits/chosen": -1.7140337228775024, + "logits/rejected": -1.700222373008728, + "logps/chosen": -0.2663342356681824, + "logps/rejected": -0.2759917378425598, + "loss": 1.0282187461853027, + "loss/core": 1.0282187461853027, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.3432140350341797, + "rewards/margins": 2.151822566986084, + "rewards/rejected": 1.1913913488388062, + "step": 535 + }, + { + "epoch": 0.5159441060551774, + "grad_norm": 39.25, + "learning_rate": 2.0754317625276982e-07, + "logits/chosen": -1.79038405418396, + "logits/rejected": -1.9031089544296265, + "logps/chosen": -0.29101166129112244, + "logps/rejected": -0.2809355556964874, + "loss": 1.0550512075424194, + "loss/core": 1.0550512075424194, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.919968366622925, + "rewards/margins": 2.102735996246338, + "rewards/rejected": 0.8172324299812317, + "step": 540 + }, + { + "epoch": 0.520721366296429, + "grad_norm": 35.5, + "learning_rate": 2.0277374684296498e-07, + "logits/chosen": -1.715163230895996, + "logits/rejected": -1.8029130697250366, + "logps/chosen": -0.30172231793403625, + "logps/rejected": -0.32220780849456787, + "loss": 1.0138914585113525, + "loss/core": 1.0138914585113525, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.903212070465088, + "rewards/margins": 1.434679627418518, + "rewards/rejected": 1.4685324430465698, + "step": 545 + }, + { + "epoch": 0.5254986265376806, + "grad_norm": 5.90625, + "learning_rate": 1.980220772955602e-07, + "logits/chosen": -1.949233055114746, + "logits/rejected": -2.019287347793579, + "logps/chosen": -0.2586565315723419, + "logps/rejected": -0.2541942596435547, + "loss": 1.0455553531646729, + "loss/core": 1.0455553531646729, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.489222288131714, + "rewards/margins": 0.4238598942756653, + "rewards/rejected": 2.0653622150421143, + "step": 550 + }, + { + "epoch": 0.5302758867789323, + "grad_norm": 19.125, + "learning_rate": 1.932899545191433e-07, + "logits/chosen": -1.9635541439056396, + "logits/rejected": -1.9265022277832031, + "logps/chosen": -0.290753573179245, + "logps/rejected": -0.27892938256263733, + "loss": 1.0557947158813477, + "loss/core": 1.0557947158813477, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.1324610710144043, + "rewards/margins": 0.5951575040817261, + "rewards/rejected": 1.5373035669326782, + "step": 555 + }, + { + "epoch": 0.5350531470201839, + "grad_norm": 77.0, + "learning_rate": 1.885791580715609e-07, + "logits/chosen": -1.6048539876937866, + "logits/rejected": -1.6119636297225952, + "logps/chosen": -0.28494495153427124, + "logps/rejected": -0.286663681268692, + "loss": 1.038069725036621, + "loss/core": 1.038069725036621, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.65592098236084, + "rewards/margins": 0.7274275422096252, + "rewards/rejected": 1.9284934997558594, + "step": 560 + }, + { + "epoch": 0.5398304072614356, + "grad_norm": 6.84375, + "learning_rate": 1.8389145949069951e-07, + "logits/chosen": -1.6353938579559326, + "logits/rejected": -1.6788675785064697, + "logps/chosen": -0.2736624479293823, + "logps/rejected": -0.26380524039268494, + "loss": 1.052668809890747, + "loss/core": 1.052668809890747, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.4656434059143066, + "rewards/margins": 1.5169572830200195, + "rewards/rejected": 1.9486862421035767, + "step": 565 + }, + { + "epoch": 0.5446076675026872, + "grad_norm": 45.25, + "learning_rate": 1.792286216282824e-07, + "logits/chosen": -1.558429479598999, + "logits/rejected": -1.5371347665786743, + "logps/chosen": -0.2772481441497803, + "logps/rejected": -0.2760164737701416, + "loss": 1.0409849882125854, + "loss/core": 1.0409849882125854, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.2676281929016113, + "rewards/margins": 0.6158636212348938, + "rewards/rejected": 1.6517642736434937, + "step": 570 + }, + { + "epoch": 0.5493849277439389, + "grad_norm": 3.640625, + "learning_rate": 1.745923979869336e-07, + "logits/chosen": -1.8342043161392212, + "logits/rejected": -1.8309091329574585, + "logps/chosen": -0.2796408534049988, + "logps/rejected": -0.2815830409526825, + "loss": 1.0364043712615967, + "loss/core": 1.0364043712615967, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.265101194381714, + "rewards/margins": 1.3904941082000732, + "rewards/rejected": 0.8746069669723511, + "step": 575 + }, + { + "epoch": 0.5541621879851905, + "grad_norm": 7.4375, + "learning_rate": 1.6998453206075708e-07, + "logits/chosen": -2.014143466949463, + "logits/rejected": -2.0165810585021973, + "logps/chosen": -0.2994392216205597, + "logps/rejected": -0.30162468552589417, + "loss": 1.037367582321167, + "loss/core": 1.037367582321167, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.404824137687683, + "rewards/margins": 0.4138920307159424, + "rewards/rejected": 0.990932285785675, + "step": 580 + }, + { + "epoch": 0.5589394482264421, + "grad_norm": 5.90625, + "learning_rate": 1.6540675667967973e-07, + "logits/chosen": -1.724384069442749, + "logits/rejected": -1.7589279413223267, + "logps/chosen": -0.30380764603614807, + "logps/rejected": -0.2969801127910614, + "loss": 1.0485934019088745, + "loss/core": 1.0485934019088745, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.2653725147247314, + "rewards/margins": 1.247117042541504, + "rewards/rejected": 1.0182554721832275, + "step": 585 + }, + { + "epoch": 0.5637167084676937, + "grad_norm": 11.5625, + "learning_rate": 1.60860793357805e-07, + "logits/chosen": -1.806414008140564, + "logits/rejected": -1.8376471996307373, + "logps/chosen": -0.2740582823753357, + "logps/rejected": -0.27855148911476135, + "loss": 1.033927083015442, + "loss/core": 1.033927083015442, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.1178157329559326, + "rewards/margins": 0.7777595520019531, + "rewards/rejected": 1.3400561809539795, + "step": 590 + }, + { + "epoch": 0.5684939687089454, + "grad_norm": 10.5, + "learning_rate": 1.5634835164602196e-07, + "logits/chosen": -1.6470069885253906, + "logits/rejected": -1.700169563293457, + "logps/chosen": -0.2837096154689789, + "logps/rejected": -0.28879398107528687, + "loss": 1.033031940460205, + "loss/core": 1.033031940460205, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.160616636276245, + "rewards/margins": 0.9769798517227173, + "rewards/rejected": 1.1836367845535278, + "step": 595 + }, + { + "epoch": 0.5732712289501971, + "grad_norm": 13.625, + "learning_rate": 1.518711284891132e-07, + "logits/chosen": -1.9625580310821533, + "logits/rejected": -1.928464651107788, + "logps/chosen": -0.2618710696697235, + "logps/rejected": -0.27233943343162537, + "loss": 1.0305383205413818, + "loss/core": 1.0305383205413818, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9427595138549805, + "rewards/margins": 0.7403771877288818, + "rewards/rejected": 1.2023823261260986, + "step": 600 + }, + { + "epoch": 0.5780484891914487, + "grad_norm": 79.5, + "learning_rate": 1.47430807587603e-07, + "logits/chosen": -1.7923481464385986, + "logits/rejected": -1.9285895824432373, + "logps/chosen": -0.3259024918079376, + "logps/rejected": -0.33158808946609497, + "loss": 1.0331567525863647, + "loss/core": 1.0331567525863647, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.143375873565674, + "rewards/margins": 1.2113240957260132, + "rewards/rejected": 0.9320517778396606, + "step": 605 + }, + { + "epoch": 0.5828257494327004, + "grad_norm": 18.125, + "learning_rate": 1.430290587645865e-07, + "logits/chosen": -1.8447580337524414, + "logits/rejected": -1.826443076133728, + "logps/chosen": -0.2840345799922943, + "logps/rejected": -0.29238995909690857, + "loss": 1.0294818878173828, + "loss/core": 1.0294818878173828, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.1355230808258057, + "rewards/margins": 0.9689822196960449, + "rewards/rejected": 1.1665408611297607, + "step": 610 + }, + { + "epoch": 0.587603009673952, + "grad_norm": 10.75, + "learning_rate": 1.3866753733777765e-07, + "logits/chosen": -1.8555551767349243, + "logits/rejected": -1.9416917562484741, + "logps/chosen": -0.27575385570526123, + "logps/rejected": -0.3149864971637726, + "loss": 1.0164090394973755, + "loss/core": 1.0164090394973755, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.061546802520752, + "rewards/margins": 1.4506876468658447, + "rewards/rejected": 1.6108589172363281, + "step": 615 + }, + { + "epoch": 0.5923802699152036, + "grad_norm": 46.25, + "learning_rate": 1.343478834970121e-07, + "logits/chosen": -1.8785676956176758, + "logits/rejected": -1.8584808111190796, + "logps/chosen": -0.2844165861606598, + "logps/rejected": -0.29962268471717834, + "loss": 1.0305485725402832, + "loss/core": 1.0305485725402832, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.5849804878234863, + "rewards/margins": 1.1563302278518677, + "rewards/rejected": 1.4286500215530396, + "step": 620 + }, + { + "epoch": 0.5971575301564552, + "grad_norm": 11.375, + "learning_rate": 1.3007172168743852e-07, + "logits/chosen": -1.8624359369277954, + "logits/rejected": -1.852315902709961, + "logps/chosen": -0.2829623222351074, + "logps/rejected": -0.30127206444740295, + "loss": 1.0275830030441284, + "loss/core": 1.0275830030441284, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9424110651016235, + "rewards/margins": 0.5080059170722961, + "rewards/rejected": 1.4344053268432617, + "step": 625 + }, + { + "epoch": 0.6019347903977069, + "grad_norm": 26.25, + "learning_rate": 1.25840659998631e-07, + "logits/chosen": -1.7425400018692017, + "logits/rejected": -1.77998948097229, + "logps/chosen": -0.29986807703971863, + "logps/rejected": -0.3102700114250183, + "loss": 1.0461167097091675, + "loss/core": 1.0461167097091675, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.9104256629943848, + "rewards/margins": 1.9246628284454346, + "rewards/rejected": 0.9857630729675293, + "step": 630 + }, + { + "epoch": 0.6067120506389586, + "grad_norm": 18.75, + "learning_rate": 1.2165628955985313e-07, + "logits/chosen": -1.8899660110473633, + "logits/rejected": -1.788923978805542, + "logps/chosen": -0.30364158749580383, + "logps/rejected": -0.30772846937179565, + "loss": 1.0387496948242188, + "loss/core": 1.0387496948242188, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.7164779901504517, + "rewards/margins": 0.41386109590530396, + "rewards/rejected": 1.302616834640503, + "step": 635 + }, + { + "epoch": 0.6114893108802102, + "grad_norm": 11.3125, + "learning_rate": 1.175201839416988e-07, + "logits/chosen": -1.7163013219833374, + "logits/rejected": -1.7982505559921265, + "logps/chosen": -0.2873172163963318, + "logps/rejected": -0.29000353813171387, + "loss": 1.036372184753418, + "loss/core": 1.036372184753418, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.6618483066558838, + "rewards/margins": 1.0304663181304932, + "rewards/rejected": 0.6313816905021667, + "step": 640 + }, + { + "epoch": 0.6162665711214619, + "grad_norm": 54.0, + "learning_rate": 1.1343389856433658e-07, + "logits/chosen": -1.9578911066055298, + "logits/rejected": -1.9668989181518555, + "logps/chosen": -0.2846335470676422, + "logps/rejected": -0.27758553624153137, + "loss": 1.0494635105133057, + "loss/core": 1.0494635105133057, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.0247325897216797, + "rewards/margins": 0.21105141937732697, + "rewards/rejected": 1.8136810064315796, + "step": 645 + }, + { + "epoch": 0.6210438313627135, + "grad_norm": 70.5, + "learning_rate": 1.0939897011258e-07, + "logits/chosen": -1.6655654907226562, + "logits/rejected": -1.7558271884918213, + "logps/chosen": -0.29102975130081177, + "logps/rejected": -0.2871955335140228, + "loss": 1.046040654182434, + "loss/core": 1.046040654182434, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.1432576179504395, + "rewards/margins": 1.0412713289260864, + "rewards/rejected": 2.1019864082336426, + "step": 650 + }, + { + "epoch": 0.6258210916039652, + "grad_norm": 65.0, + "learning_rate": 1.0541691595800336e-07, + "logits/chosen": -1.9179801940917969, + "logits/rejected": -1.831691026687622, + "logps/chosen": -0.2815897464752197, + "logps/rejected": -0.29244643449783325, + "loss": 1.026608943939209, + "loss/core": 1.026608943939209, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.7700217962265015, + "rewards/margins": 0.5304393768310547, + "rewards/rejected": 1.2395825386047363, + "step": 655 + }, + { + "epoch": 0.6305983518452167, + "grad_norm": 5.0, + "learning_rate": 1.0148923358832021e-07, + "logits/chosen": -1.7689975500106812, + "logits/rejected": -1.8483155965805054, + "logps/chosen": -0.2766217589378357, + "logps/rejected": -0.2837319076061249, + "loss": 1.0300475358963013, + "loss/core": 1.0300475358963013, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.812479019165039, + "rewards/margins": 0.43489259481430054, + "rewards/rejected": 1.3775863647460938, + "step": 660 + }, + { + "epoch": 0.6353756120864684, + "grad_norm": 8.1875, + "learning_rate": 9.761740004423926e-08, + "logits/chosen": -1.8706638813018799, + "logits/rejected": -1.864198088645935, + "logps/chosen": -0.2770681381225586, + "logps/rejected": -0.2515154182910919, + "loss": 1.073243498802185, + "loss/core": 1.073243498802185, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.2681612968444824, + "rewards/margins": 1.0078423023223877, + "rewards/rejected": 1.2603189945220947, + "step": 665 + }, + { + "epoch": 0.64015287232772, + "grad_norm": 5.125, + "learning_rate": 9.380287136400999e-08, + "logits/chosen": -1.8545506000518799, + "logits/rejected": -1.9372055530548096, + "logps/chosen": -0.2920622229576111, + "logps/rejected": -0.2942867577075958, + "loss": 1.0373417139053345, + "loss/core": 1.0373417139053345, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.3302066326141357, + "rewards/margins": 0.37538617849349976, + "rewards/rejected": 0.954820454120636, + "step": 670 + }, + { + "epoch": 0.6449301325689717, + "grad_norm": 19.375, + "learning_rate": 9.004708203586628e-08, + "logits/chosen": -2.0307302474975586, + "logits/rejected": -1.9755264520645142, + "logps/chosen": -0.28046339750289917, + "logps/rejected": -0.29223278164863586, + "loss": 1.0251643657684326, + "loss/core": 1.0251643657684326, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.621219277381897, + "rewards/margins": 0.32391253113746643, + "rewards/rejected": 1.2973066568374634, + "step": 675 + }, + { + "epoch": 0.6497073928102234, + "grad_norm": 11.6875, + "learning_rate": 8.635144445857407e-08, + "logits/chosen": -1.800567626953125, + "logits/rejected": -1.8191674947738647, + "logps/chosen": -0.2679905295372009, + "logps/rejected": -0.27527937293052673, + "loss": 1.0352510213851929, + "loss/core": 1.0352510213851929, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.176344871520996, + "rewards/margins": 1.9697097539901733, + "rewards/rejected": 1.206634759902954, + "step": 680 + }, + { + "epoch": 0.654484653051475, + "grad_norm": 35.25, + "learning_rate": 8.271734841028552e-08, + "logits/chosen": -1.8452231884002686, + "logits/rejected": -1.8123219013214111, + "logps/chosen": -0.2911657691001892, + "logps/rejected": -0.2622499167919159, + "loss": 1.0803478956222534, + "loss/core": 1.0803478956222534, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.31235933303833, + "rewards/margins": -0.02849596180021763, + "rewards/rejected": 2.340855121612549, + "step": 685 + }, + { + "epoch": 0.6592619132927267, + "grad_norm": 40.0, + "learning_rate": 7.91461605259007e-08, + "logits/chosen": -1.593219518661499, + "logits/rejected": -1.6734249591827393, + "logps/chosen": -0.31815314292907715, + "logps/rejected": -0.32347214221954346, + "loss": 1.0386189222335815, + "loss/core": 1.0386189222335815, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.939263105392456, + "rewards/margins": 0.8087685704231262, + "rewards/rejected": 2.1304945945739746, + "step": 690 + }, + { + "epoch": 0.6640391735339782, + "grad_norm": 18.625, + "learning_rate": 7.563922378313217e-08, + "logits/chosen": -1.778198480606079, + "logits/rejected": -1.7811349630355835, + "logps/chosen": -0.2683827579021454, + "logps/rejected": -0.2815011143684387, + "loss": 1.0239763259887695, + "loss/core": 1.0239763259887695, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.358581781387329, + "rewards/margins": 1.180909514427185, + "rewards/rejected": 1.1776721477508545, + "step": 695 + }, + { + "epoch": 0.6688164337752299, + "grad_norm": 29.25, + "learning_rate": 7.219785699746572e-08, + "logits/chosen": -1.9994821548461914, + "logits/rejected": -1.947238564491272, + "logps/chosen": -0.286782830953598, + "logps/rejected": -0.27130791544914246, + "loss": 1.0630942583084106, + "loss/core": 1.0630942583084106, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.2841906547546387, + "rewards/margins": 1.6503528356552124, + "rewards/rejected": 1.633837342262268, + "step": 700 + }, + { + "epoch": 0.6735936940164815, + "grad_norm": 4.53125, + "learning_rate": 6.882335432620779e-08, + "logits/chosen": -1.8093124628067017, + "logits/rejected": -1.754615068435669, + "logps/chosen": -0.3254457712173462, + "logps/rejected": -0.31252914667129517, + "loss": 1.0568554401397705, + "loss/core": 1.0568554401397705, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.3327386379241943, + "rewards/margins": 0.8064883947372437, + "rewards/rejected": 0.5262502431869507, + "step": 705 + }, + { + "epoch": 0.6783709542577332, + "grad_norm": 19.875, + "learning_rate": 6.551698478180589e-08, + "logits/chosen": -1.6687633991241455, + "logits/rejected": -1.6874847412109375, + "logps/chosen": -0.2778703570365906, + "logps/rejected": -0.28912389278411865, + "loss": 1.025848150253296, + "loss/core": 1.025848150253296, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.4708938598632812, + "rewards/margins": 1.6537853479385376, + "rewards/rejected": 0.8171082735061646, + "step": 710 + }, + { + "epoch": 0.6831482144989849, + "grad_norm": 9.0, + "learning_rate": 6.22799917546252e-08, + "logits/chosen": -1.8202791213989258, + "logits/rejected": -1.9110924005508423, + "logps/chosen": -0.2837766408920288, + "logps/rejected": -0.28179988265037537, + "loss": 1.0426539182662964, + "loss/core": 1.0426539182662964, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.3514044284820557, + "rewards/margins": 1.4219096899032593, + "rewards/rejected": 0.9294947385787964, + "step": 715 + }, + { + "epoch": 0.6879254747402365, + "grad_norm": 7.40625, + "learning_rate": 5.9113592545359944e-08, + "logits/chosen": -1.9379427433013916, + "logits/rejected": -1.941803216934204, + "logps/chosen": -0.2888217568397522, + "logps/rejected": -0.3106004297733307, + "loss": 1.0178791284561157, + "loss/core": 1.0178791284561157, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.548539638519287, + "rewards/margins": 1.4435575008392334, + "rewards/rejected": 1.1049822568893433, + "step": 720 + }, + { + "epoch": 0.6927027349814882, + "grad_norm": 3.9375, + "learning_rate": 5.601897790725643e-08, + "logits/chosen": -1.8667323589324951, + "logits/rejected": -1.8213609457015991, + "logps/chosen": -0.28370243310928345, + "logps/rejected": -0.2769761383533478, + "loss": 1.0477391481399536, + "loss/core": 1.0477391481399536, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.1843743324279785, + "rewards/margins": 0.06037312000989914, + "rewards/rejected": 1.124001145362854, + "step": 725 + }, + { + "epoch": 0.6974799952227397, + "grad_norm": 22.125, + "learning_rate": 5.299731159831952e-08, + "logits/chosen": -1.7045841217041016, + "logits/rejected": -1.7231441736221313, + "logps/chosen": -0.3042663633823395, + "logps/rejected": -0.3213575482368469, + "loss": 1.0313047170639038, + "loss/core": 1.0313047170639038, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.7923970222473145, + "rewards/margins": 1.8490235805511475, + "rewards/rejected": 0.9433733820915222, + "step": 730 + }, + { + "epoch": 0.7022572554639914, + "grad_norm": 44.75, + "learning_rate": 5.0049729943671013e-08, + "logits/chosen": -1.915514349937439, + "logits/rejected": -1.888715386390686, + "logps/chosen": -0.2653517723083496, + "logps/rejected": -0.27663201093673706, + "loss": 1.0275311470031738, + "loss/core": 1.0275311470031738, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.803576707839966, + "rewards/margins": 1.734165906906128, + "rewards/rejected": 1.0694106817245483, + "step": 735 + }, + { + "epoch": 0.707034515705243, + "grad_norm": 30.5, + "learning_rate": 4.7177341408223994e-08, + "logits/chosen": -1.6809492111206055, + "logits/rejected": -1.6404021978378296, + "logps/chosen": -0.29158157110214233, + "logps/rejected": -0.3046492338180542, + "loss": 1.0248371362686157, + "loss/core": 1.0248371362686157, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.1834161281585693, + "rewards/margins": 0.5413880348205566, + "rewards/rejected": 1.6420282125473022, + "step": 740 + }, + { + "epoch": 0.7118117759464947, + "grad_norm": 6.375, + "learning_rate": 4.438122617983442e-08, + "logits/chosen": -1.887942910194397, + "logits/rejected": -1.8876336812973022, + "logps/chosen": -0.2923053801059723, + "logps/rejected": -0.2827454209327698, + "loss": 1.0520001649856567, + "loss/core": 1.0520001649856567, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.3299145698547363, + "rewards/margins": 1.1022189855575562, + "rewards/rejected": 1.2276954650878906, + "step": 745 + }, + { + "epoch": 0.7165890361877463, + "grad_norm": 36.0, + "learning_rate": 4.1662435763087114e-08, + "logits/chosen": -1.841260552406311, + "logits/rejected": -1.7571378946304321, + "logps/chosen": -0.2794218361377716, + "logps/rejected": -0.3028506338596344, + "loss": 1.012818694114685, + "loss/core": 1.012818694114685, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.2234444618225098, + "rewards/margins": 0.845853328704834, + "rewards/rejected": 1.3775911331176758, + "step": 750 + }, + { + "epoch": 0.721366296428998, + "grad_norm": 5.46875, + "learning_rate": 3.902199258386732e-08, + "logits/chosen": -1.8437637090682983, + "logits/rejected": -1.8173859119415283, + "logps/chosen": -0.27545225620269775, + "logps/rejected": -0.2872912287712097, + "loss": 1.0292212963104248, + "loss/core": 1.0292212963104248, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.603247880935669, + "rewards/margins": 1.7060693502426147, + "rewards/rejected": 1.8971784114837646, + "step": 755 + }, + { + "epoch": 0.7261435566702497, + "grad_norm": 6.96875, + "learning_rate": 3.646088960486862e-08, + "logits/chosen": -1.6895129680633545, + "logits/rejected": -1.7431033849716187, + "logps/chosen": -0.3080654442310333, + "logps/rejected": -0.3540648818016052, + "loss": 1.0325762033462524, + "loss/core": 1.0325762033462524, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.53886079788208, + "rewards/margins": 1.5185824632644653, + "rewards/rejected": 2.020278215408325, + "step": 760 + }, + { + "epoch": 0.7309208169115012, + "grad_norm": 8.4375, + "learning_rate": 3.398008995217988e-08, + "logits/chosen": -1.964421033859253, + "logits/rejected": -1.953046441078186, + "logps/chosen": -0.2779175639152527, + "logps/rejected": -0.26899924874305725, + "loss": 1.0525156259536743, + "loss/core": 1.0525156259536743, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.6339603662490845, + "rewards/margins": 0.7983501553535461, + "rewards/rejected": 0.8356102705001831, + "step": 765 + }, + { + "epoch": 0.7356980771527529, + "grad_norm": 108.0, + "learning_rate": 3.1580526553093315e-08, + "logits/chosen": -1.877881407737732, + "logits/rejected": -1.9419214725494385, + "logps/chosen": -0.28730475902557373, + "logps/rejected": -0.2802114188671112, + "loss": 1.0498204231262207, + "loss/core": 1.0498204231262207, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.2819581031799316, + "rewards/margins": 1.5368332862854004, + "rewards/rejected": 0.7451248168945312, + "step": 770 + }, + { + "epoch": 0.7404753373940045, + "grad_norm": 5.125, + "learning_rate": 2.9263101785268252e-08, + "logits/chosen": -1.890252709388733, + "logits/rejected": -1.786828637123108, + "logps/chosen": -0.27267634868621826, + "logps/rejected": -0.2676258087158203, + "loss": 1.0456647872924805, + "loss/core": 1.0456647872924805, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.8578557968139648, + "rewards/margins": 0.4590323567390442, + "rewards/rejected": 1.3988234996795654, + "step": 775 + }, + { + "epoch": 0.7452525976352562, + "grad_norm": 268.0, + "learning_rate": 2.7028687137384264e-08, + "logits/chosen": -1.9074313640594482, + "logits/rejected": -1.8822380304336548, + "logps/chosen": -0.2915792465209961, + "logps/rejected": -0.293667197227478, + "loss": 1.037200927734375, + "loss/core": 1.037200927734375, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.0063960552215576, + "rewards/margins": 1.0704381465911865, + "rewards/rejected": 1.935957670211792, + "step": 780 + }, + { + "epoch": 0.7500298578765078, + "grad_norm": 9.6875, + "learning_rate": 2.4878122881409447e-08, + "logits/chosen": -1.8461768627166748, + "logits/rejected": -1.8658325672149658, + "logps/chosen": -0.2820442020893097, + "logps/rejected": -0.2832142412662506, + "loss": 1.0385828018188477, + "loss/core": 1.0385828018188477, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.9002225399017334, + "rewards/margins": 1.1294481754302979, + "rewards/rejected": 0.7707743644714355, + "step": 785 + }, + { + "epoch": 0.7548071181177595, + "grad_norm": 36.25, + "learning_rate": 2.2812217756608937e-08, + "logits/chosen": -1.825455904006958, + "logits/rejected": -1.8060061931610107, + "logps/chosen": -0.2775079011917114, + "logps/rejected": -0.27117595076560974, + "loss": 1.0483758449554443, + "loss/core": 1.0483758449554443, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.307039260864258, + "rewards/margins": 2.0355687141418457, + "rewards/rejected": 1.271470546722412, + "step": 790 + }, + { + "epoch": 0.7595843783590112, + "grad_norm": 61.5, + "learning_rate": 2.0831748665410763e-08, + "logits/chosen": -1.770493745803833, + "logits/rejected": -1.7957115173339844, + "logps/chosen": -0.27306801080703735, + "logps/rejected": -0.28165358304977417, + "loss": 1.0311884880065918, + "loss/core": 1.0311884880065918, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.751323699951172, + "rewards/margins": 2.812521457672119, + "rewards/rejected": 0.9388027191162109, + "step": 795 + }, + { + "epoch": 0.7643616386002627, + "grad_norm": 10.5, + "learning_rate": 1.8937460381244967e-08, + "logits/chosen": -2.075711727142334, + "logits/rejected": -2.082383394241333, + "logps/chosen": -0.2964426577091217, + "logps/rejected": -0.30977433919906616, + "loss": 1.0251314640045166, + "loss/core": 1.0251314640045166, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.5174245834350586, + "rewards/margins": 1.0785468816757202, + "rewards/rejected": 1.4388777017593384, + "step": 800 + }, + { + "epoch": 0.7691388988415144, + "grad_norm": 7.65625, + "learning_rate": 1.713006526846439e-08, + "logits/chosen": -1.8648141622543335, + "logits/rejected": -1.8252522945404053, + "logps/chosen": -0.30121514201164246, + "logps/rejected": -0.3006502687931061, + "loss": 1.0409377813339233, + "loss/core": 1.0409377813339233, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 0.9633139371871948, + "rewards/margins": 0.3292653262615204, + "rewards/rejected": 0.634048581123352, + "step": 805 + }, + { + "epoch": 0.773916159082766, + "grad_norm": 28.125, + "learning_rate": 1.541024301445404e-08, + "logits/chosen": -1.6336872577667236, + "logits/rejected": -1.643832802772522, + "logps/chosen": -0.26690006256103516, + "logps/rejected": -0.26295873522758484, + "loss": 1.0444200038909912, + "loss/core": 1.0444200038909912, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.8206228017807007, + "rewards/margins": 0.1702648401260376, + "rewards/rejected": 1.6503578424453735, + "step": 810 + }, + { + "epoch": 0.7786934193240177, + "grad_norm": 5.84375, + "learning_rate": 1.3778640374027983e-08, + "logits/chosen": -1.751861333847046, + "logits/rejected": -1.9031435251235962, + "logps/chosen": -0.3171994984149933, + "logps/rejected": -0.3077738881111145, + "loss": 1.0605608224868774, + "loss/core": 1.0605608224868774, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.4510459899902344, + "rewards/margins": 1.3974026441574097, + "rewards/rejected": 1.0536432266235352, + "step": 815 + }, + { + "epoch": 0.7834706795652693, + "grad_norm": 19.0, + "learning_rate": 1.2235870926211616e-08, + "logits/chosen": -1.6588127613067627, + "logits/rejected": -1.6749852895736694, + "logps/chosen": -0.27939319610595703, + "logps/rejected": -0.3374519348144531, + "loss": 1.0130990743637085, + "loss/core": 1.0130990743637085, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.6268069744110107, + "rewards/margins": 0.6854991912841797, + "rewards/rejected": 1.9413076639175415, + "step": 820 + }, + { + "epoch": 0.788247939806521, + "grad_norm": 13.375, + "learning_rate": 1.0782514843499652e-08, + "logits/chosen": -1.7541849613189697, + "logits/rejected": -1.7715661525726318, + "logps/chosen": -0.29441070556640625, + "logps/rejected": -0.27596911787986755, + "loss": 1.0648963451385498, + "loss/core": 1.0648963451385498, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.5321407318115234, + "rewards/margins": 1.8472797870635986, + "rewards/rejected": 1.6848608255386353, + "step": 825 + }, + { + "epoch": 0.7930252000477725, + "grad_norm": 78.5, + "learning_rate": 9.419118673676924e-09, + "logits/chosen": -1.563860535621643, + "logits/rejected": -1.5621966123580933, + "logps/chosen": -0.3032872676849365, + "logps/rejected": -0.3065972328186035, + "loss": 1.0367621183395386, + "loss/core": 1.0367621183395386, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.807102680206299, + "rewards/margins": 1.6108064651489258, + "rewards/rejected": 1.1962958574295044, + "step": 830 + }, + { + "epoch": 0.7978024602890242, + "grad_norm": 6.03125, + "learning_rate": 8.14619513428405e-09, + "logits/chosen": -1.904526948928833, + "logits/rejected": -1.9740642309188843, + "logps/chosen": -0.29926085472106934, + "logps/rejected": -0.2885436713695526, + "loss": 1.054502248764038, + "loss/core": 1.054502248764038, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.6957647800445557, + "rewards/margins": 0.6506364941596985, + "rewards/rejected": 1.045128345489502, + "step": 835 + }, + { + "epoch": 0.8025797205302759, + "grad_norm": 9.9375, + "learning_rate": 6.96422291980539e-09, + "logits/chosen": -1.8588794469833374, + "logits/rejected": -1.90227472782135, + "logps/chosen": -0.2894556224346161, + "logps/rejected": -0.3027735650539398, + "loss": 1.02797532081604, + "loss/core": 1.02797532081604, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.031310796737671, + "rewards/margins": 0.43263769149780273, + "rewards/rejected": 1.5986733436584473, + "step": 840 + }, + { + "epoch": 0.8073569807715275, + "grad_norm": 17.25, + "learning_rate": 5.8736465216517594e-09, + "logits/chosen": -1.9031187295913696, + "logits/rejected": -1.9344947338104248, + "logps/chosen": -0.2742392420768738, + "logps/rejected": -0.2774720788002014, + "loss": 1.0499933958053589, + "loss/core": 1.0499933958053589, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.593908786773682, + "rewards/margins": 2.4118456840515137, + "rewards/rejected": 2.1820626258850098, + "step": 845 + }, + { + "epoch": 0.8121342410127792, + "grad_norm": 4.09375, + "learning_rate": 4.874876061005173e-09, + "logits/chosen": -1.8513599634170532, + "logits/rejected": -1.8989410400390625, + "logps/chosen": -0.2543167173862457, + "logps/rejected": -0.25180160999298096, + "loss": 1.0425068140029907, + "loss/core": 1.0425068140029907, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7452361583709717, + "rewards/margins": 1.0755412578582764, + "rewards/rejected": 0.6696950197219849, + "step": 850 + }, + { + "epoch": 0.8169115012540308, + "grad_norm": 25.875, + "learning_rate": 3.968287134589188e-09, + "logits/chosen": -1.7955166101455688, + "logits/rejected": -1.8299312591552734, + "logps/chosen": -0.2676030695438385, + "logps/rejected": -0.2964601516723633, + "loss": 1.004606008529663, + "loss/core": 1.004606008529663, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.3115034103393555, + "rewards/margins": 0.41639384627342224, + "rewards/rejected": 1.8951095342636108, + "step": 855 + }, + { + "epoch": 0.8216887614952825, + "grad_norm": 5.375, + "learning_rate": 3.154220673422192e-09, + "logits/chosen": -1.8242244720458984, + "logits/rejected": -1.871952772140503, + "logps/chosen": -0.29963549971580505, + "logps/rejected": -0.3072584271430969, + "loss": 1.0296491384506226, + "loss/core": 1.0296491384506226, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.9225000143051147, + "rewards/margins": 0.8448880910873413, + "rewards/rejected": 1.0776121616363525, + "step": 860 + }, + { + "epoch": 0.826466021736534, + "grad_norm": 7.28125, + "learning_rate": 2.4329828146074096e-09, + "logits/chosen": -1.9124653339385986, + "logits/rejected": -2.0408012866973877, + "logps/chosen": -0.29619625210762024, + "logps/rejected": -0.2827211022377014, + "loss": 1.0616817474365234, + "loss/core": 1.0616817474365234, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.6343491077423096, + "rewards/margins": 1.424903392791748, + "rewards/rejected": 1.2094457149505615, + "step": 865 + }, + { + "epoch": 0.8312432819777857, + "grad_norm": 19.5, + "learning_rate": 1.8048447862070714e-09, + "logits/chosen": -1.5945602655410767, + "logits/rejected": -1.6320714950561523, + "logps/chosen": -0.2955728769302368, + "logps/rejected": -0.2850645184516907, + "loss": 1.061391830444336, + "loss/core": 1.061391830444336, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.463152885437012, + "rewards/margins": 2.405036449432373, + "rewards/rejected": 2.0581166744232178, + "step": 870 + }, + { + "epoch": 0.8360205422190374, + "grad_norm": 4.5, + "learning_rate": 1.2700428052447033e-09, + "logits/chosen": -1.7374908924102783, + "logits/rejected": -1.7853012084960938, + "logps/chosen": -0.3029218316078186, + "logps/rejected": -0.3070649802684784, + "loss": 1.0413432121276855, + "loss/core": 1.0413432121276855, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9537794589996338, + "rewards/margins": 0.5955628156661987, + "rewards/rejected": 1.358216643333435, + "step": 875 + }, + { + "epoch": 0.840797802460289, + "grad_norm": 3.453125, + "learning_rate": 8.287779888734858e-10, + "logits/chosen": -1.7615150213241577, + "logits/rejected": -1.707476019859314, + "logps/chosen": -0.28448665142059326, + "logps/rejected": -0.3107626140117645, + "loss": 1.0223973989486694, + "loss/core": 1.0223973989486694, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.715723752975464, + "rewards/margins": 2.2797560691833496, + "rewards/rejected": 1.4359674453735352, + "step": 880 + }, + { + "epoch": 0.8455750627015407, + "grad_norm": 18.0, + "learning_rate": 4.812162787445062e-10, + "logits/chosen": -1.8650376796722412, + "logits/rejected": -1.8496296405792236, + "logps/chosen": -0.2981775403022766, + "logps/rejected": -0.30601340532302856, + "loss": 1.029161810874939, + "loss/core": 1.029161810874939, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.5521029233932495, + "rewards/margins": 0.7568989992141724, + "rewards/rejected": 0.7952039241790771, + "step": 885 + }, + { + "epoch": 0.8503523229427923, + "grad_norm": 11.4375, + "learning_rate": 2.2748837860270265e-10, + "logits/chosen": -1.8946914672851562, + "logits/rejected": -1.9093029499053955, + "logps/chosen": -0.2697398066520691, + "logps/rejected": -0.2905231714248657, + "loss": 1.019055962562561, + "loss/core": 1.019055962562561, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.8530833721160889, + "rewards/margins": 0.24989423155784607, + "rewards/rejected": 1.6031888723373413, + "step": 890 + }, + { + "epoch": 0.855129583184044, + "grad_norm": 65.0, + "learning_rate": 6.768970513457151e-11, + "logits/chosen": -1.7693462371826172, + "logits/rejected": -1.7002862691879272, + "logps/chosen": -0.2899450659751892, + "logps/rejected": -0.2910805940628052, + "loss": 1.038063645362854, + "loss/core": 1.038063645362854, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.1362929344177246, + "rewards/margins": 1.1503578424453735, + "rewards/rejected": 0.9859352111816406, + "step": 895 + }, + { + "epoch": 0.8599068434252956, + "grad_norm": 34.0, + "learning_rate": 1.8803520859811406e-12, + "logits/chosen": -1.7765915393829346, + "logits/rejected": -1.7241567373275757, + "logps/chosen": -0.2680931091308594, + "logps/rejected": -0.2755570411682129, + "loss": 1.0350791215896606, + "loss/core": 1.0350791215896606, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.9140121936798096, + "rewards/margins": 1.5131105184555054, + "rewards/rejected": 1.4009015560150146, + "step": 900 + }, + { + "epoch": 0.8599068434252956, + "step": 900, + "total_flos": 0.0, + "train_loss": 1.0422146828969319, + "train_runtime": 8158.2235, + "train_samples_per_second": 1.765, + "train_steps_per_second": 0.11 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..ddfe9e6 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:600c099628a160fb480bf20bf843aee00bfed4d2ae14c1ddde82e699d750b32d +size 6993