commit d5eeecdd0c3f20359fd1ca214e4c7e45565599f8 Author: ModelHub XC Date: Thu Jul 23 19:11:12 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: promotion/qwen3-8b-aaai27-flagship-sppo-avg-s43 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..11fe948 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +--- +library_name: transformers +base_model: Qwen/Qwen3-8B +tags: +- ronpo +- mnpo +- preference-optimization +--- + +# qwen3-8b-aaai27-flagship-sppo-avg-s43 + +Research checkpoint for the RONPO AAAI revision experiments. + +- Method: sppo_avg +- Base model: `Qwen/Qwen3-8B` +- Seed: 43 +- Local source at upload time: `/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/sppo_avg/seed43/attempt1` +- Uploaded at UTC: 2026-07-13T04:34:41Z +- Run status metadata: `{"attempt": 1, "effective_batch_size": 16, "method": "sppo_avg", "objective_protocol": "objective_protocol.json", "optimizer_steps": 900, "resource_profile": "resource_profile.json", "seed": 43, "split_manifest": "split_manifest.json", "stability_gate": "stability_gate.json", "status": "completed", "wandb_run_id": "e6a327670d44", "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/e6a327670d44"}` + +AAAI-27 P1 matched budget: 900 optimizer steps, effective batch 16; passed non-thinking and collapse stability gates; sealed-test metrics are not used for checkpoint selection. + +Intended use: reproducibility and evaluation for the RONPO paper. This +checkpoint is not intended as a production assistant. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..5fba767 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 0.6663506550259061, + "train_runtime": 7058.8291, + "train_samples": 16746, + "train_samples_per_second": 2.04, + "train_steps_per_second": 0.127 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..c629106 --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +model_name_or_path: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218 +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/avg: 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: sppo +eta: 0.0075 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.0 +preference_sft_weight: 0.0 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 5.0e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 43 +gradient_accumulation_steps: 16 +gradient_checkpointing: true +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 1 +per_device_eval_batch_size: 1 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/sppo_avg/seed43/attempt1 +run_name: aaai27-flagship-full-sppo_avg-s43-a1 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/job_status.json b/job_status.json new file mode 100644 index 0000000..c7d319c --- /dev/null +++ b/job_status.json @@ -0,0 +1,23 @@ +{ + "status": "completed", + "stage": "full", + "method": "sppo_avg", + "seed": 43, + "attempt": 1, + "gpu": 2, + "gpus": [ + 2 + ], + "returncode": 0, + "finite_metrics": true, + "oom": false, + "traceback": false, + "reason": "passed", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "e6a327670d44", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/e6a327670d44", + "output_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/sppo_avg/seed43/attempt1", + "log": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/logs/full_sppo_avg_s43_a1.log", + "completed_at": "2026-07-13T04:30:21Z" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..67f8d00 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c16a0225e6ed043dd68f1ce4c3856e4438ec9b198bd6f053d696e23cb7097f04 +size 16381517208 diff --git a/objective_protocol.json b/objective_protocol.json new file mode 100644 index 0000000..35b3b44 --- /dev/null +++ b/objective_protocol.json @@ -0,0 +1,69 @@ +{ + "schema_version": 1, + "frozen_at_kst": "2026-07-12T21:40:00+09:00", + "base_model": "Qwen/Qwen3-8B", + "primary_triple": { + "helpfulness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "ultrafeedback-helpfulness", + "transform": "identity" + }, + "safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "conciseness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "helpsteer-verbosity", + "transform": "negate" + } + }, + "gate_thresholds": { + "raw_score_std_strictly_greater_than": 0.01, + "aggregate_prompt_pair_spearman_median_strictly_less_than": 0.0, + "aggregate_pairwise_top1_mismatch_at_least": 0.5 + }, + "gate_pool": "source-train Qwen3-8B response pool split into deterministic train/validation; sealed source-test excluded", + "training_seeds": [42, 43, 44], + "optimizer_steps": 900, + "effective_batch_size": 16, + "decode": { + "enable_thinking": false, + "temperature": 0.7, + "top_p": 0.9, + "max_new_tokens": 2048, + "decode_seed": 42 + }, + "methods": [ + "ronpo_full_expect", + "ronpo_k_only", + "dpo", + "ipo", + "simpo", + "kto", + "sppo_avg", + "inpo_avg", + "ht_mnpo_helpfulness", + "ht_mnpo_safety", + "ht_mnpo_conciseness" + ], + "selection_policy": "No sealed-test selection. One preregistered configuration per method; failures use only the declared stability retry recipe, at most three attempts.", + "stability_gate": { + "validation_prompts": 128, + "min_candidate_base_mean_word_ratio": 0.33, + "max_candidate_base_mean_word_ratio": 2.0, + "max_consecutive_identical_word_run": 20, + "think_tag_count": 0, + "empty_response_count": 0, + "retry_recipe": "attempt 2 halves learning rate, enables gradient checkpointing, and adds reference=0.02/SFT=0.002 anchoring to unanchored unified losses; attempt 3 quarters learning rate and uses reference=0.05/SFT=0.005; RONPO uses reference=0.05/SFT=0.005 from attempt 1" + }, + "primary_metric": "mean prompt-level minimum of per-prompt min-max-normalized helpfulness, safety, conciseness", + "bootstrap": { + "unit": "prompt", + "paired": true, + "resamples": 2000, + "interval": "percentile_95", + "seed": 42 + } +} diff --git a/pair_manifest.json b/pair_manifest.json new file mode 100644 index 0000000..be034b0 --- /dev/null +++ b/pair_manifest.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "objectives": [ + "helpfulness", + "safety", + "conciseness" + ], + "normalization": "per_prompt_minmax", + "adversary": { + "steps": 25, + "alpha": 1.0, + "kappa": 0.05, + "preference_scale": 8.0 + }, + "ronpo_estimator": "Rao-Blackwellized full sigma expectation with three common response pairs per prompt", + "counts": { + "train": { + "avg": 16746, + "ronpo_full_expect": 50340, + "ronpo_k_only": 50340, + "ht_mnpo_helpfulness": 16743, + "ht_mnpo_safety": 16764, + "ht_mnpo_conciseness": 16755 + }, + "validation": { + "avg": 1859, + "ronpo_full_expect": 5592, + "ronpo_k_only": 5592, + "ht_mnpo_helpfulness": 1864, + "ht_mnpo_safety": 1862, + "ht_mnpo_conciseness": 1858 + } + } +} diff --git a/provenance.json b/provenance.json new file mode 100644 index 0000000..ee8877b --- /dev/null +++ b/provenance.json @@ -0,0 +1,13 @@ +{ + "method": "sppo_avg", + "seed": 43, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "e6a327670d44", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/e6a327670d44", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json" +} diff --git a/resource_profile.json b/resource_profile.json new file mode 100644 index 0000000..18eec47 --- /dev/null +++ b/resource_profile.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "measured_at_kst": "2026-07-12T22:38:00+09:00", + "hardware": "NVIDIA B200 183359 MiB reported total; 178.35 GiB PyTorch-visible capacity", + "smoke_evidence": { + "attempted_profile": {"per_device_batch_size": 2, "gradient_accumulation_steps": 8, "gradient_checkpointing": false, "effective_batch_size": 16}, + "outcome": "OOM before completing the 20-step smoke across every launched unified baseline; representative process used 176.87 GiB and failed a 1.62 GiB allocation. KTO also OOMed at the optimizer step.", + "paper_metric": false + }, + "selected_full_training_profile": { + "per_device_batch_size": 2, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": false, + "attention_implementation": "PyTorch SDPA with cuDNN SDPA disabled", + "effective_batch_size": 16, + "applies_symmetrically_to_all_unified_pairwise_methods": true, + "kto_implementation_constraint": "TRL KTO requires actual batch size > 1. KTO uses batch 2 / accumulation 8 and identical AdamW with foreach=False. PyTorch SDPA is used with cuDNN SDPA disabled. The fixed base reference model is placed on a second B200 and its exact logits are returned to the policy device; the policy/optimizer remain on the first B200. This avoids a measured ~6-hour reference-precompute path while leaving the KTO loss, actual/effective batch, optimizer, and 900-step budget unchanged." + }, + "retry_semantics": "Resource selection is separated from method-stability attempts. Full attempt 1 uses the smoke-validated SDPA batch-2 profile and retains the preregistered method LR/anchor; attempts 2 and 3 fall back to batch 1 / accumulation 16 / gradient checkpointing and apply the declared LR/anchor stabilization recipe.", + "provenance_note": "No reward or sealed-evaluation result was available when this hardware profile was selected." +} diff --git a/run_status.json b/run_status.json new file mode 100644 index 0000000..b07a931 --- /dev/null +++ b/run_status.json @@ -0,0 +1,14 @@ +{ + "method": "sppo_avg", + "seed": 43, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "e6a327670d44", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/e6a327670d44", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json", + "status": "completed" +} diff --git a/split_manifest.json b/split_manifest.json new file mode 100644 index 0000000..4fb96bd --- /dev/null +++ b/split_manifest.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "split_rule": "sort source-train prompts by sha256('flagship-v1||'+prompt_id); first round(10%) validation", + "validation_fraction": 0.1, + "source_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/avg_oracle", + "base_model": "Qwen/Qwen3-8B", + "counts": { + "train": 16780, + "validation": 1864, + "sealed_test": 604 + }, + "prompt_id_sha256": { + "train": "fcf6f2ea5a0b7bcd41179e6f5288f4d850b2d04bdb729da91907bd18bf733a69", + "validation": "a87ba6f519192164daa487b109a6547dd8cbde46a27a373d6da201ca2d061b8f", + "sealed_test": "b398ced6af6cf73d731e1a5f32a0631d8060323d51eda1e375695566bf5f1867" + }, + "overlaps": { + "train_validation": 0, + "train_sealed_test": 0, + "validation_sealed_test": 0 + }, + "sealed_test_policy": "prompts only until S4a; no objective scores or model generations used in S0-S3" +} diff --git a/stability_gate.json b/stability_gate.json new file mode 100644 index 0000000..eab03bd --- /dev/null +++ b/stability_gate.json @@ -0,0 +1,38 @@ +{ + "status": "passed", + "passed": true, + "base": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 388.1484375, + "max_words": 1379, + "max_repeat_run": 4 + }, + "candidate": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 409.125, + "max_words": 1255, + "max_repeat_run": 4 + }, + "candidate_base_mean_word_ratio": 1.0540426302759496, + "thresholds": { + "min_length_ratio": 0.33, + "max_length_ratio": 2.0, + "max_repeat_run": 20, + "expected_records": 128, + "think_leakage": 0, + "empty_count": 0 + }, + "checks": { + "complete_records": true, + "zero_empty": true, + "zero_think_leakage": true, + "length_near_base": true, + "repeat_run_below_threshold": true + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ffd19d2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..5fba767 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 0.6663506550259061, + "train_runtime": 7058.8291, + "train_samples": 16746, + "train_samples_per_second": 2.04, + "train_steps_per_second": 0.127 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..a911c22 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,2923 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8599068434252956, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.004777260241251642, + "grad_norm": 2.296875, + "learning_rate": 2.222222222222222e-08, + "logits/chosen": -2.3184335231781006, + "logits/rejected": -2.304084300994873, + "logps/chosen": -0.31140023469924927, + "logps/rejected": -0.29028692841529846, + "loss": 0.3946344256401062, + "loss/core": 0.3946344256401062, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.209475040435791, + "rewards/margins": 1.5164874792099, + "rewards/rejected": 0.6929876208305359, + "step": 5 + }, + { + "epoch": 0.009554520482503284, + "grad_norm": 183.0, + "learning_rate": 5e-08, + "logits/chosen": -2.064580202102661, + "logits/rejected": -2.0450222492218018, + "logps/chosen": -0.28674912452697754, + "logps/rejected": -0.3005256652832031, + "loss": 1.1264296770095825, + "loss/core": 1.1264296770095825, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.9985833168029785, + "rewards/margins": 0.33723002672195435, + "rewards/rejected": 2.66135311126709, + "step": 10 + }, + { + "epoch": 0.014331780723754926, + "grad_norm": 12.625, + "learning_rate": 7.777777777777778e-08, + "logits/chosen": -2.228989601135254, + "logits/rejected": -2.2271804809570312, + "logps/chosen": -0.2594456374645233, + "logps/rejected": -0.2722489833831787, + "loss": 0.26212042570114136, + "loss/core": 0.26212042570114136, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.754608154296875, + "rewards/margins": 0.7076910734176636, + "rewards/rejected": 1.046917200088501, + "step": 15 + }, + { + "epoch": 0.01910904096500657, + "grad_norm": 4.59375, + "learning_rate": 1.0555555555555555e-07, + "logits/chosen": -2.1681175231933594, + "logits/rejected": -2.185380458831787, + "logps/chosen": -0.2745749354362488, + "logps/rejected": -0.26860952377319336, + "loss": 1.016000747680664, + "loss/core": 1.016000747680664, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.6250548362731934, + "rewards/margins": 2.429143190383911, + "rewards/rejected": 1.1959115266799927, + "step": 20 + }, + { + "epoch": 0.02388630120625821, + "grad_norm": 4.40625, + "learning_rate": 1.3333333333333334e-07, + "logits/chosen": -2.010061740875244, + "logits/rejected": -2.1256415843963623, + "logps/chosen": -0.27929872274398804, + "logps/rejected": -0.2702064514160156, + "loss": 0.17983713746070862, + "loss/core": 0.17983713746070862, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 2.1464362144470215, + "rewards/margins": 1.1600453853607178, + "rewards/rejected": 0.9863905906677246, + "step": 25 + }, + { + "epoch": 0.028663561447509853, + "grad_norm": 5.78125, + "learning_rate": 1.611111111111111e-07, + "logits/chosen": -2.1331052780151367, + "logits/rejected": -2.1351206302642822, + "logps/chosen": -0.27982744574546814, + "logps/rejected": -0.26132339239120483, + "loss": 0.5742586851119995, + "loss/core": 0.5742586851119995, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.813447952270508, + "rewards/margins": 1.750083327293396, + "rewards/rejected": 1.0633647441864014, + "step": 30 + }, + { + "epoch": 0.033440821688761495, + "grad_norm": 1.53125, + "learning_rate": 1.8888888888888888e-07, + "logits/chosen": -2.273294687271118, + "logits/rejected": -2.2828454971313477, + "logps/chosen": -0.2884656488895416, + "logps/rejected": -0.2767728865146637, + "loss": 0.8869408369064331, + "loss/core": 0.8869408369064331, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.5600028038024902, + "rewards/margins": 1.78105890750885, + "rewards/rejected": 0.7789438366889954, + "step": 35 + }, + { + "epoch": 0.03821808193001314, + "grad_norm": 27.75, + "learning_rate": 2.1666666666666667e-07, + "logits/chosen": -2.2540197372436523, + "logits/rejected": -2.351133108139038, + "logps/chosen": -0.2658698558807373, + "logps/rejected": -0.2907714247703552, + "loss": 0.6574953198432922, + "loss/core": 0.6574953198432922, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.926403522491455, + "rewards/margins": 2.233344554901123, + "rewards/rejected": 0.6930587887763977, + "step": 40 + }, + { + "epoch": 0.04299534217126478, + "grad_norm": 4.46875, + "learning_rate": 2.4444444444444445e-07, + "logits/chosen": -2.090873956680298, + "logits/rejected": -2.14512300491333, + "logps/chosen": -0.27041956782341003, + "logps/rejected": -0.27646249532699585, + "loss": 0.8163895606994629, + "loss/core": 0.8163895606994629, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.0477185249328613, + "rewards/margins": 1.7669315338134766, + "rewards/rejected": 1.2807869911193848, + "step": 45 + }, + { + "epoch": 0.04777260241251642, + "grad_norm": 66.0, + "learning_rate": 2.7222222222222216e-07, + "logits/chosen": -2.188749074935913, + "logits/rejected": -2.2616398334503174, + "logps/chosen": -0.2877681851387024, + "logps/rejected": -0.26972633600234985, + "loss": 0.21321019530296326, + "loss/core": 0.21321019530296326, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.6582177877426147, + "rewards/margins": 0.44566529989242554, + "rewards/rejected": 1.2125524282455444, + "step": 50 + }, + { + "epoch": 0.05254986265376806, + "grad_norm": 236.0, + "learning_rate": 3e-07, + "logits/chosen": -2.0755667686462402, + "logits/rejected": -2.1377971172332764, + "logps/chosen": -0.28878724575042725, + "logps/rejected": -0.28834742307662964, + "loss": 0.5084811449050903, + "loss/core": 0.5084811449050903, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.339972496032715, + "rewards/margins": 1.6370515823364258, + "rewards/rejected": 0.7029210329055786, + "step": 55 + }, + { + "epoch": 0.057327122895019705, + "grad_norm": 121.0, + "learning_rate": 3.2777777777777776e-07, + "logits/chosen": -2.1755266189575195, + "logits/rejected": -2.122792959213257, + "logps/chosen": -0.2767593562602997, + "logps/rejected": -0.2760574519634247, + "loss": 1.5027045011520386, + "loss/core": 1.5027045011520386, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.761892318725586, + "rewards/margins": 2.0544276237487793, + "rewards/rejected": 1.707464575767517, + "step": 60 + }, + { + "epoch": 0.06210438313627135, + "grad_norm": 264.0, + "learning_rate": 3.5555555555555553e-07, + "logits/chosen": -2.1052839756011963, + "logits/rejected": -2.1663551330566406, + "logps/chosen": -0.2984526455402374, + "logps/rejected": -0.28494197130203247, + "loss": 1.0875555276870728, + "loss/core": 1.0875555276870728, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.2381019592285156, + "rewards/margins": 2.0958991050720215, + "rewards/rejected": 1.1422029733657837, + "step": 65 + }, + { + "epoch": 0.06688164337752299, + "grad_norm": 16.0, + "learning_rate": 3.8333333333333335e-07, + "logits/chosen": -2.134707450866699, + "logits/rejected": -2.046159267425537, + "logps/chosen": -0.2798652648925781, + "logps/rejected": -0.27488410472869873, + "loss": 0.4238152503967285, + "loss/core": 0.4238152503967285, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.4914584159851074, + "rewards/margins": 1.004212498664856, + "rewards/rejected": 1.4872459173202515, + "step": 70 + }, + { + "epoch": 0.07165890361877464, + "grad_norm": 1.65625, + "learning_rate": 4.1111111111111107e-07, + "logits/chosen": -2.0351903438568115, + "logits/rejected": -2.068025588989258, + "logps/chosen": -0.29988136887550354, + "logps/rejected": -0.3065613806247711, + "loss": 0.5185372829437256, + "loss/core": 0.5185372829437256, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.2677860260009766, + "rewards/margins": 1.2009799480438232, + "rewards/rejected": 1.0668058395385742, + "step": 75 + }, + { + "epoch": 0.07643616386002627, + "grad_norm": 0.8359375, + "learning_rate": 4.3888888888888884e-07, + "logits/chosen": -2.2651379108428955, + "logits/rejected": -2.265864849090576, + "logps/chosen": -0.2954421639442444, + "logps/rejected": -0.2877275347709656, + "loss": 0.18425166606903076, + "loss/core": 0.18425166606903076, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.3546873331069946, + "rewards/margins": 0.4060265123844147, + "rewards/rejected": 0.9486608505249023, + "step": 80 + }, + { + "epoch": 0.08121342410127792, + "grad_norm": 744.0, + "learning_rate": 4.6666666666666666e-07, + "logits/chosen": -2.2166881561279297, + "logits/rejected": -2.2303507328033447, + "logps/chosen": -0.27470141649246216, + "logps/rejected": -0.2800918519496918, + "loss": 0.5587457418441772, + "loss/core": 0.5587457418441772, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.6758487224578857, + "rewards/margins": 0.03965097665786743, + "rewards/rejected": 1.6361976861953735, + "step": 85 + }, + { + "epoch": 0.08599068434252956, + "grad_norm": 15.6875, + "learning_rate": 4.944444444444445e-07, + "logits/chosen": -2.1294827461242676, + "logits/rejected": -2.2263193130493164, + "logps/chosen": -0.27022016048431396, + "logps/rejected": -0.31300783157348633, + "loss": 0.8455947041511536, + "loss/core": 0.8455947041511536, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.5631749629974365, + "rewards/margins": 1.0474013090133667, + "rewards/rejected": 1.515773892402649, + "step": 90 + }, + { + "epoch": 0.09076794458378121, + "grad_norm": 22.0, + "learning_rate": 4.999699149323369e-07, + "logits/chosen": -2.137500524520874, + "logits/rejected": -2.064415454864502, + "logps/chosen": -0.28990474343299866, + "logps/rejected": -0.2970162034034729, + "loss": 0.4857604503631592, + "loss/core": 0.4857604503631592, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.2846360206604004, + "rewards/margins": 1.2684682607650757, + "rewards/rejected": 1.0161678791046143, + "step": 95 + }, + { + "epoch": 0.09554520482503284, + "grad_norm": 4.4375, + "learning_rate": 4.998477067547739e-07, + "logits/chosen": -2.11228346824646, + "logits/rejected": -2.1553733348846436, + "logps/chosen": -0.26913318037986755, + "logps/rejected": -0.2833011746406555, + "loss": 1.1113026142120361, + "loss/core": 1.1113026142120361, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.711238145828247, + "rewards/margins": 2.09954833984375, + "rewards/rejected": 1.611689567565918, + "step": 100 + }, + { + "epoch": 0.10032246506628449, + "grad_norm": 24.25, + "learning_rate": 4.996315410727229e-07, + "logits/chosen": -2.0640909671783447, + "logits/rejected": -2.0353593826293945, + "logps/chosen": -0.317560613155365, + "logps/rejected": -0.304330050945282, + "loss": 0.93501216173172, + "loss/core": 0.93501216173172, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.3537240028381348, + "rewards/margins": 2.5598361492156982, + "rewards/rejected": 0.7938874959945679, + "step": 105 + }, + { + "epoch": 0.10509972530753613, + "grad_norm": 1.7890625, + "learning_rate": 4.993214991772562e-07, + "logits/chosen": -2.1243863105773926, + "logits/rejected": -2.126587390899658, + "logps/chosen": -0.2978929877281189, + "logps/rejected": -0.3149787485599518, + "loss": 0.20066983997821808, + "loss/core": 0.20066983997821808, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.6585355997085571, + "rewards/margins": 0.7344213724136353, + "rewards/rejected": 0.9241144061088562, + "step": 110 + }, + { + "epoch": 0.10987698554878778, + "grad_norm": 10.8125, + "learning_rate": 4.989176976624511e-07, + "logits/chosen": -2.0023255348205566, + "logits/rejected": -2.1182520389556885, + "logps/chosen": -0.30129915475845337, + "logps/rejected": -0.29388052225112915, + "loss": 1.1868757009506226, + "loss/core": 1.1868757009506226, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.8971168994903564, + "rewards/margins": 1.7273696660995483, + "rewards/rejected": 1.1697468757629395, + "step": 115 + }, + { + "epoch": 0.11465424579003941, + "grad_norm": 51.0, + "learning_rate": 4.984202883815428e-07, + "logits/chosen": -2.2406866550445557, + "logits/rejected": -2.252915620803833, + "logps/chosen": -0.30535244941711426, + "logps/rejected": -0.30678510665893555, + "loss": 0.48086509108543396, + "loss/core": 0.48086509108543396, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.3497025966644287, + "rewards/margins": 1.1640708446502686, + "rewards/rejected": 1.1856318712234497, + "step": 120 + }, + { + "epoch": 0.11943150603129106, + "grad_norm": 87.0, + "learning_rate": 4.978294583898195e-07, + "logits/chosen": -2.0294809341430664, + "logits/rejected": -1.9394248723983765, + "logps/chosen": -0.29434821009635925, + "logps/rejected": -0.3001564145088196, + "loss": 0.9685723185539246, + "loss/core": 0.9685723185539246, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.8707126379013062, + "rewards/margins": -0.364132821559906, + "rewards/rejected": 2.2348453998565674, + "step": 125 + }, + { + "epoch": 0.1242087662725427, + "grad_norm": 58.5, + "learning_rate": 4.971454298742779e-07, + "logits/chosen": -2.348689079284668, + "logits/rejected": -2.265820026397705, + "logps/chosen": -0.27629876136779785, + "logps/rejected": -0.302666574716568, + "loss": 0.6233381032943726, + "loss/core": 0.6233381032943726, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.2184367179870605, + "rewards/margins": 0.2606551945209503, + "rewards/rejected": 1.9577815532684326, + "step": 130 + }, + { + "epoch": 0.12898602651379434, + "grad_norm": 11.0625, + "learning_rate": 4.963684600700678e-07, + "logits/chosen": -2.0810961723327637, + "logits/rejected": -2.177532434463501, + "logps/chosen": -0.34504374861717224, + "logps/rejected": -0.3136570155620575, + "loss": 0.6652131676673889, + "loss/core": 0.6652131676673889, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.7010741233825684, + "rewards/margins": 1.2287189960479736, + "rewards/rejected": 1.4723551273345947, + "step": 135 + }, + { + "epoch": 0.13376328675504598, + "grad_norm": 57.75, + "learning_rate": 4.954988411637571e-07, + "logits/chosen": -1.9434146881103516, + "logits/rejected": -1.9310106039047241, + "logps/chosen": -0.3319775462150574, + "logps/rejected": -0.34657081961631775, + "loss": 1.2185124158859253, + "loss/core": 1.2185124158859253, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.100647449493408, + "rewards/margins": 0.7573388814926147, + "rewards/rejected": 2.343308925628662, + "step": 140 + }, + { + "epoch": 0.13854054699629761, + "grad_norm": 6.4375, + "learning_rate": 4.945369001834514e-07, + "logits/chosen": -2.0375614166259766, + "logits/rejected": -2.0471043586730957, + "logps/chosen": -0.3008027672767639, + "logps/rejected": -0.28672724962234497, + "loss": 0.20575180649757385, + "loss/core": 0.20575180649757385, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7885154485702515, + "rewards/margins": 0.952106773853302, + "rewards/rejected": 0.8364084362983704, + "step": 145 + }, + { + "epoch": 0.14331780723754928, + "grad_norm": 211.0, + "learning_rate": 4.93482998875813e-07, + "logits/chosen": -2.004568576812744, + "logits/rejected": -2.028262138366699, + "logps/chosen": -0.2854091227054596, + "logps/rejected": -0.27151355147361755, + "loss": 0.6710211038589478, + "loss/core": 0.6710211038589478, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.637166738510132, + "rewards/margins": 1.3770954608917236, + "rewards/rejected": 1.2600712776184082, + "step": 150 + }, + { + "epoch": 0.1480950674788009, + "grad_norm": 5.90625, + "learning_rate": 4.923375335700223e-07, + "logits/chosen": -1.968898057937622, + "logits/rejected": -2.025012731552124, + "logps/chosen": -0.2497386932373047, + "logps/rejected": -0.2664481997489929, + "loss": 0.38974815607070923, + "loss/core": 0.38974815607070923, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.618589401245117, + "rewards/margins": 1.195473551750183, + "rewards/rejected": 1.423115849494934, + "step": 155 + }, + { + "epoch": 0.15287232772005255, + "grad_norm": 58.5, + "learning_rate": 4.911009350287347e-07, + "logits/chosen": -2.1655495166778564, + "logits/rejected": -2.139359951019287, + "logps/chosen": -0.27988573908805847, + "logps/rejected": -0.2946188449859619, + "loss": 0.5953173041343689, + "loss/core": 0.5953173041343689, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.644726276397705, + "rewards/margins": 1.5004968643188477, + "rewards/rejected": 1.144229531288147, + "step": 160 + }, + { + "epoch": 0.15764958796130418, + "grad_norm": 5.3125, + "learning_rate": 4.897736682860885e-07, + "logits/chosen": -2.109489917755127, + "logits/rejected": -2.0843923091888428, + "logps/chosen": -0.3071993589401245, + "logps/rejected": -0.30547797679901123, + "loss": 0.1960802972316742, + "loss/core": 0.1960802972316742, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.7830066680908203, + "rewards/margins": 0.7959617376327515, + "rewards/rejected": 0.9870448112487793, + "step": 165 + }, + { + "epoch": 0.16242684820255585, + "grad_norm": 1.2578125, + "learning_rate": 4.883562324728241e-07, + "logits/chosen": -2.0669140815734863, + "logits/rejected": -1.9952644109725952, + "logps/chosen": -0.3106909692287445, + "logps/rejected": -0.3281523883342743, + "loss": 0.20468628406524658, + "loss/core": 0.20468628406524658, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.8208732604980469, + "rewards/margins": 1.240056037902832, + "rewards/rejected": 0.5808171033859253, + "step": 170 + }, + { + "epoch": 0.16720410844380748, + "grad_norm": 70.5, + "learning_rate": 4.868491606285823e-07, + "logits/chosen": -2.2352006435394287, + "logits/rejected": -2.256232261657715, + "logps/chosen": -0.27431878447532654, + "logps/rejected": -0.28181517124176025, + "loss": 0.1919003427028656, + "loss/core": 0.1919003427028656, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.784128189086914, + "rewards/margins": 1.083038568496704, + "rewards/rejected": 0.7010896801948547, + "step": 175 + }, + { + "epoch": 0.17198136868505912, + "grad_norm": 14.6875, + "learning_rate": 4.852530195014489e-07, + "logits/chosen": -2.1959755420684814, + "logits/rejected": -2.148374080657959, + "logps/chosen": -0.2624792456626892, + "logps/rejected": -0.2835710942745209, + "loss": 0.6657207608222961, + "loss/core": 0.6657207608222961, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.457181692123413, + "rewards/margins": 1.1070095300674438, + "rewards/rejected": 1.3501724004745483, + "step": 180 + }, + { + "epoch": 0.17675862892631075, + "grad_norm": 632.0, + "learning_rate": 4.835684093348244e-07, + "logits/chosen": -2.2028822898864746, + "logits/rejected": -2.2440648078918457, + "logps/chosen": -0.3046368956565857, + "logps/rejected": -0.3071895241737366, + "loss": 0.6476051807403564, + "loss/core": 0.6476051807403564, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.8896329402923584, + "rewards/margins": 0.37541091442108154, + "rewards/rejected": 1.5142220258712769, + "step": 185 + }, + { + "epoch": 0.18153588916756241, + "grad_norm": 436.0, + "learning_rate": 4.817959636416969e-07, + "logits/chosen": -2.134538412094116, + "logits/rejected": -2.1165084838867188, + "logps/chosen": -0.2866383194923401, + "logps/rejected": -0.27144110202789307, + "loss": 0.6611530780792236, + "loss/core": 0.6611530780792236, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.6436305046081543, + "rewards/margins": 1.4646612405776978, + "rewards/rejected": 1.178969144821167, + "step": 190 + }, + { + "epoch": 0.18631314940881405, + "grad_norm": 6.3125, + "learning_rate": 4.799363489664039e-07, + "logits/chosen": -1.9559043645858765, + "logits/rejected": -1.9407514333724976, + "logps/chosen": -0.28489893674850464, + "logps/rejected": -0.2946854531764984, + "loss": 0.2528533637523651, + "loss/core": 0.2528533637523651, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.681348204612732, + "rewards/margins": 0.3501024842262268, + "rewards/rejected": 1.33124577999115, + "step": 195 + }, + { + "epoch": 0.19109040965006568, + "grad_norm": 8.6875, + "learning_rate": 4.779902646339721e-07, + "logits/chosen": -2.1071603298187256, + "logits/rejected": -2.1380481719970703, + "logps/chosen": -0.2741830050945282, + "logps/rejected": -0.27496692538261414, + "loss": 0.46219339966773987, + "loss/core": 0.46219339966773987, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.5044608116149902, + "rewards/margins": 1.1724002361297607, + "rewards/rejected": 1.33206045627594, + "step": 200 + }, + { + "epoch": 0.19586766989131732, + "grad_norm": 3.203125, + "learning_rate": 4.759584424871301e-07, + "logits/chosen": -2.0774598121643066, + "logits/rejected": -2.142284631729126, + "logps/chosen": -0.2829883396625519, + "logps/rejected": -0.29093074798583984, + "loss": 0.08258922398090363, + "loss/core": 0.08258922398090363, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.3614203929901123, + "rewards/margins": 0.7309275269508362, + "rewards/rejected": 0.6304928660392761, + "step": 205 + }, + { + "epoch": 0.20064493013256898, + "grad_norm": 10.9375, + "learning_rate": 4.738416466110917e-07, + "logits/chosen": -2.0302059650421143, + "logits/rejected": -2.0086560249328613, + "logps/chosen": -0.288141131401062, + "logps/rejected": -0.2887657582759857, + "loss": 0.8564616441726685, + "loss/core": 0.8564616441726685, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.526529550552368, + "rewards/margins": 1.0289502143859863, + "rewards/rejected": 1.4975792169570923, + "step": 210 + }, + { + "epoch": 0.20542219037382062, + "grad_norm": 10.8125, + "learning_rate": 4.716406730462153e-07, + "logits/chosen": -1.9449901580810547, + "logits/rejected": -1.9634069204330444, + "logps/chosen": -0.29736560583114624, + "logps/rejected": -0.2745818793773651, + "loss": 0.24143390357494354, + "loss/core": 0.24143390357494354, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.0462050437927246, + "rewards/margins": 1.0101492404937744, + "rewards/rejected": 1.0360558032989502, + "step": 215 + }, + { + "epoch": 0.21019945061507225, + "grad_norm": 3.0, + "learning_rate": 4.693563494886454e-07, + "logits/chosen": -2.025031089782715, + "logits/rejected": -2.0499961376190186, + "logps/chosen": -0.28308767080307007, + "logps/rejected": -0.27393776178359985, + "loss": 0.16485320031642914, + "loss/core": 0.16485320031642914, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.761115312576294, + "rewards/margins": 0.9114633798599243, + "rewards/rejected": 0.849652111530304, + "step": 220 + }, + { + "epoch": 0.2149767108563239, + "grad_norm": 15.3125, + "learning_rate": 4.6698953497905016e-07, + "logits/chosen": -2.0364017486572266, + "logits/rejected": -2.0577664375305176, + "logps/chosen": -0.31471285223960876, + "logps/rejected": -0.30929452180862427, + "loss": 1.460294485092163, + "loss/core": 1.460294485092163, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.0360703468322754, + "rewards/margins": 1.0577287673950195, + "rewards/rejected": 1.9783413410186768, + "step": 225 + }, + { + "epoch": 0.21975397109757555, + "grad_norm": 54.5, + "learning_rate": 4.645411195795709e-07, + "logits/chosen": -2.118978500366211, + "logits/rejected": -2.1093850135803223, + "logps/chosen": -0.27341052889823914, + "logps/rejected": -0.26687008142471313, + "loss": 0.740382194519043, + "loss/core": 0.740382194519043, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.039541006088257, + "rewards/margins": 1.364072561264038, + "rewards/rejected": 1.6754686832427979, + "step": 230 + }, + { + "epoch": 0.2245312313388272, + "grad_norm": 2.84375, + "learning_rate": 4.6201202403910643e-07, + "logits/chosen": -2.160707950592041, + "logits/rejected": -2.255814790725708, + "logps/chosen": -0.28612154722213745, + "logps/rejected": -0.28075844049453735, + "loss": 0.19919511675834656, + "loss/core": 0.19919511675834656, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 1.6981313228607178, + "rewards/margins": 1.023926854133606, + "rewards/rejected": 0.6742045879364014, + "step": 235 + }, + { + "epoch": 0.22930849158007882, + "grad_norm": 7.90625, + "learning_rate": 4.594031994470573e-07, + "logits/chosen": -2.0647361278533936, + "logits/rejected": -2.1183581352233887, + "logps/chosen": -0.2742571234703064, + "logps/rejected": -0.2913045287132263, + "loss": 1.243248701095581, + "loss/core": 1.243248701095581, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.7567174434661865, + "rewards/margins": 0.8206822276115417, + "rewards/rejected": 1.936035394668579, + "step": 240 + }, + { + "epoch": 0.23408575182133046, + "grad_norm": 1112.0, + "learning_rate": 4.567156268756593e-07, + "logits/chosen": -2.120326519012451, + "logits/rejected": -2.2658631801605225, + "logps/chosen": -0.30770888924598694, + "logps/rejected": -0.292914479970932, + "loss": 0.6964072585105896, + "loss/core": 0.6964072585105896, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 2.1233694553375244, + "rewards/margins": 1.4673677682876587, + "rewards/rejected": 0.6560018062591553, + "step": 245 + }, + { + "epoch": 0.23886301206258212, + "grad_norm": 10.625, + "learning_rate": 4.5395031701104303e-07, + "logits/chosen": -1.9557708501815796, + "logits/rejected": -1.9600833654403687, + "logps/chosen": -0.30549654364585876, + "logps/rejected": -0.3048430383205414, + "loss": 1.1260910034179688, + "loss/core": 1.1260910034179688, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.3460769653320312, + "rewards/margins": 1.7832248210906982, + "rewards/rejected": 1.562852144241333, + "step": 250 + }, + { + "epoch": 0.24364027230383375, + "grad_norm": 4.09375, + "learning_rate": 4.511083097731555e-07, + "logits/chosen": -2.1089088916778564, + "logits/rejected": -2.0778651237487793, + "logps/chosen": -0.287789523601532, + "logps/rejected": -0.2891516089439392, + "loss": 1.2482858896255493, + "loss/core": 1.2482858896255493, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.746084213256836, + "rewards/margins": 1.564002275466919, + "rewards/rejected": 1.1820822954177856, + "step": 255 + }, + { + "epoch": 0.2484175325450854, + "grad_norm": 108.0, + "learning_rate": 4.481906739246894e-07, + "logits/chosen": -2.062340259552002, + "logits/rejected": -2.0389857292175293, + "logps/chosen": -0.3061924874782562, + "logps/rejected": -0.30993199348449707, + "loss": 0.4140895903110504, + "loss/core": 0.4140895903110504, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.9362844228744507, + "rewards/margins": 0.6997807025909424, + "rewards/rejected": 1.2365038394927979, + "step": 260 + }, + { + "epoch": 0.25319479278633705, + "grad_norm": 2.8125, + "learning_rate": 4.451985066691648e-07, + "logits/chosen": -2.0385899543762207, + "logits/rejected": -2.0162370204925537, + "logps/chosen": -0.27738499641418457, + "logps/rejected": -0.28945407271385193, + "loss": 0.23780469596385956, + "loss/core": 0.23780469596385956, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.11801815032959, + "rewards/margins": 1.098084568977356, + "rewards/rejected": 1.0199334621429443, + "step": 265 + }, + { + "epoch": 0.2579720530275887, + "grad_norm": 1.4375, + "learning_rate": 4.421329332383158e-07, + "logits/chosen": -2.0890111923217773, + "logits/rejected": -2.0384275913238525, + "logps/chosen": -0.30751222372055054, + "logps/rejected": -0.30553263425827026, + "loss": 0.3220127820968628, + "loss/core": 0.3220127820968628, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6447738409042358, + "rewards/margins": 0.24662013351917267, + "rewards/rejected": 1.3981537818908691, + "step": 270 + }, + { + "epoch": 0.2627493132688403, + "grad_norm": 3.703125, + "learning_rate": 4.3899510646893696e-07, + "logits/chosen": -2.207843780517578, + "logits/rejected": -2.2784366607666016, + "logps/chosen": -0.2971876561641693, + "logps/rejected": -0.2956683933734894, + "loss": 0.05450323969125748, + "loss/core": 0.05450323969125748, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.0277817249298096, + "rewards/margins": 0.44771966338157654, + "rewards/rejected": 0.5800620317459106, + "step": 275 + }, + { + "epoch": 0.26752657351009196, + "grad_norm": 23.0, + "learning_rate": 4.357862063693485e-07, + "logits/chosen": -2.0210609436035156, + "logits/rejected": -1.9996906518936157, + "logps/chosen": -0.3016383945941925, + "logps/rejected": -0.3017963767051697, + "loss": 0.21795885264873505, + "loss/core": 0.21795885264873505, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.8103902339935303, + "rewards/margins": 0.9846366047859192, + "rewards/rejected": 0.825753390789032, + "step": 280 + }, + { + "epoch": 0.2723038337513436, + "grad_norm": 8.5625, + "learning_rate": 4.3250743967564364e-07, + "logits/chosen": -2.2049059867858887, + "logits/rejected": -2.1598610877990723, + "logps/chosen": -0.2876811623573303, + "logps/rejected": -0.2909473478794098, + "loss": 1.0883954763412476, + "loss/core": 1.0883954763412476, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.9615416526794434, + "rewards/margins": 1.0239375829696655, + "rewards/rejected": 1.9376041889190674, + "step": 285 + }, + { + "epoch": 0.27708109399259523, + "grad_norm": 1056.0, + "learning_rate": 4.29160039397884e-07, + "logits/chosen": -2.1312131881713867, + "logits/rejected": -2.1842284202575684, + "logps/chosen": -0.2860090136528015, + "logps/rejected": -0.2702611982822418, + "loss": 2.010714292526245, + "loss/core": 2.010714292526245, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 4.503174781799316, + "rewards/margins": 3.201117992401123, + "rewards/rejected": 1.3020570278167725, + "step": 290 + }, + { + "epoch": 0.28185835423384686, + "grad_norm": 10.0625, + "learning_rate": 4.2574526435641546e-07, + "logits/chosen": -1.899706244468689, + "logits/rejected": -1.976220726966858, + "logps/chosen": -0.3136875629425049, + "logps/rejected": -0.302241712808609, + "loss": 1.1247197389602661, + "loss/core": 1.1247197389602661, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.7293989658355713, + "rewards/margins": 1.6034202575683594, + "rewards/rejected": 1.125978708267212, + "step": 295 + }, + { + "epoch": 0.28663561447509855, + "grad_norm": 5.375, + "learning_rate": 4.22264398708477e-07, + "logits/chosen": -2.0837273597717285, + "logits/rejected": -2.1485800743103027, + "logps/chosen": -0.28113582730293274, + "logps/rejected": -0.3067568838596344, + "loss": 2.029283046722412, + "loss/core": 2.029283046722412, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.4929652214050293, + "rewards/margins": 1.580517053604126, + "rewards/rejected": 1.9124481678009033, + "step": 300 + }, + { + "epoch": 0.2914128747163502, + "grad_norm": 45.25, + "learning_rate": 4.187187514652819e-07, + "logits/chosen": -1.9658424854278564, + "logits/rejected": -2.029053211212158, + "logps/chosen": -0.3088286519050598, + "logps/rejected": -0.30893000960350037, + "loss": 2.0511975288391113, + "loss/core": 2.0511975288391113, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.7278835773468018, + "rewards/margins": 1.2690778970718384, + "rewards/rejected": 2.458805561065674, + "step": 305 + }, + { + "epoch": 0.2961901349576018, + "grad_norm": 32.0, + "learning_rate": 4.151096559997519e-07, + "logits/chosen": -1.9535623788833618, + "logits/rejected": -1.9904283285140991, + "logps/chosen": -0.2780826687812805, + "logps/rejected": -0.2801373302936554, + "loss": 0.36180371046066284, + "loss/core": 0.36180371046066284, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.327801465988159, + "rewards/margins": 0.9045225977897644, + "rewards/rejected": 1.423278570175171, + "step": 310 + }, + { + "epoch": 0.30096739519885346, + "grad_norm": 29.375, + "learning_rate": 4.114384695450905e-07, + "logits/chosen": -2.104870080947876, + "logits/rejected": -2.0882060527801514, + "logps/chosen": -0.3096676766872406, + "logps/rejected": -0.303961843252182, + "loss": 0.32151490449905396, + "loss/core": 0.32151490449905396, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.1817171573638916, + "rewards/margins": -0.025614654645323753, + "rewards/rejected": 1.207331895828247, + "step": 315 + }, + { + "epoch": 0.3057446554401051, + "grad_norm": 812.0, + "learning_rate": 4.077065726843828e-07, + "logits/chosen": -1.9013550281524658, + "logits/rejected": -1.8669452667236328, + "logps/chosen": -0.3241393268108368, + "logps/rejected": -0.3211979269981384, + "loss": 1.0354502201080322, + "loss/core": 1.0354502201080322, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.091705799102783, + "rewards/margins": 1.3894522190093994, + "rewards/rejected": 1.7022536993026733, + "step": 320 + }, + { + "epoch": 0.31052191568135673, + "grad_norm": 1.5234375, + "learning_rate": 4.039153688314145e-07, + "logits/chosen": -1.993546485900879, + "logits/rejected": -2.022549867630005, + "logps/chosen": -0.28880229592323303, + "logps/rejected": -0.2948676645755768, + "loss": 0.12079276889562607, + "loss/core": 0.12079276889562607, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.4271008968353271, + "rewards/margins": 0.6435669660568237, + "rewards/rejected": 0.7835339903831482, + "step": 325 + }, + { + "epoch": 0.31529917592260837, + "grad_norm": 836.0, + "learning_rate": 4.0006628370290613e-07, + "logits/chosen": -2.167954444885254, + "logits/rejected": -2.0930395126342773, + "logps/chosen": -0.3180980980396271, + "logps/rejected": -0.3477565050125122, + "loss": 0.3206109404563904, + "loss/core": 0.3206109404563904, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.3264482021331787, + "rewards/margins": 0.24738340079784393, + "rewards/rejected": 1.0790647268295288, + "step": 330 + }, + { + "epoch": 0.32007643616386, + "grad_norm": 3.53125, + "learning_rate": 3.9616076478235826e-07, + "logits/chosen": -2.2583556175231934, + "logits/rejected": -2.293337106704712, + "logps/chosen": -0.30557915568351746, + "logps/rejected": -0.30756452679634094, + "loss": 0.0511607900261879, + "loss/core": 0.0511607900261879, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.0766465663909912, + "rewards/margins": 0.5183663964271545, + "rewards/rejected": 0.5582801699638367, + "step": 335 + }, + { + "epoch": 0.3248536964051117, + "grad_norm": 5.65625, + "learning_rate": 3.922002807757129e-07, + "logits/chosen": -2.2513046264648438, + "logits/rejected": -2.259812593460083, + "logps/chosen": -0.28524649143218994, + "logps/rejected": -0.283276230096817, + "loss": 0.631820797920227, + "loss/core": 0.631820797920227, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.1910934448242188, + "rewards/margins": 1.2970854043960571, + "rewards/rejected": 0.8940078616142273, + "step": 340 + }, + { + "epoch": 0.3296309566463633, + "grad_norm": 79.0, + "learning_rate": 3.8818632105903315e-07, + "logits/chosen": -2.1031250953674316, + "logits/rejected": -2.102226734161377, + "logps/chosen": -0.26754727959632874, + "logps/rejected": -0.28109580278396606, + "loss": 0.7250244617462158, + "loss/core": 0.7250244617462158, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.1810359954833984, + "rewards/margins": 0.5469441413879395, + "rewards/rejected": 1.6340919733047485, + "step": 345 + }, + { + "epoch": 0.33440821688761496, + "grad_norm": 11.6875, + "learning_rate": 3.841203951184094e-07, + "logits/chosen": -2.2044973373413086, + "logits/rejected": -2.2186226844787598, + "logps/chosen": -0.3000710904598236, + "logps/rejected": -0.30985429883003235, + "loss": 0.6811296939849854, + "loss/core": 0.6811296939849854, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.313596487045288, + "rewards/margins": 0.9291432499885559, + "rewards/rejected": 1.3844534158706665, + "step": 350 + }, + { + "epoch": 0.3391854771288666, + "grad_norm": 31.25, + "learning_rate": 3.800040319823038e-07, + "logits/chosen": -2.172675848007202, + "logits/rejected": -2.1876015663146973, + "logps/chosen": -0.291890025138855, + "logps/rejected": -0.2729564905166626, + "loss": 0.8517450094223022, + "loss/core": 0.8517450094223022, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.446463108062744, + "rewards/margins": 1.6567842960357666, + "rewards/rejected": 1.7896785736083984, + "step": 355 + }, + { + "epoch": 0.34396273737011823, + "grad_norm": 89.0, + "learning_rate": 3.75838779646545e-07, + "logits/chosen": -1.9489272832870483, + "logits/rejected": -1.9853594303131104, + "logps/chosen": -0.2767919600009918, + "logps/rejected": -0.279407799243927, + "loss": 0.7631909251213074, + "loss/core": 0.7631909251213074, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.232623338699341, + "rewards/margins": 1.9174954891204834, + "rewards/rejected": 1.3151280879974365, + "step": 360 + }, + { + "epoch": 0.34873999761136987, + "grad_norm": 40.5, + "learning_rate": 3.7162620449218993e-07, + "logits/chosen": -2.235380172729492, + "logits/rejected": -2.306373119354248, + "logps/chosen": -0.3116162121295929, + "logps/rejected": -0.306151807308197, + "loss": 0.5377756357192993, + "loss/core": 0.5377756357192993, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.417541265487671, + "rewards/margins": 1.4117828607559204, + "rewards/rejected": 1.0057584047317505, + "step": 365 + }, + { + "epoch": 0.3535172578526215, + "grad_norm": 22.0, + "learning_rate": 3.673678906964727e-07, + "logits/chosen": -2.117574691772461, + "logits/rejected": -2.0456650257110596, + "logps/chosen": -0.2936505675315857, + "logps/rejected": -0.30493101477622986, + "loss": 0.7472105622291565, + "loss/core": 0.7472105622291565, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.3911571502685547, + "rewards/margins": 0.7618893384933472, + "rewards/rejected": 1.629267692565918, + "step": 370 + }, + { + "epoch": 0.35829451809387314, + "grad_norm": 7.15625, + "learning_rate": 3.6306543963705936e-07, + "logits/chosen": -1.9923632144927979, + "logits/rejected": -1.9175786972045898, + "logps/chosen": -0.2919101119041443, + "logps/rejected": -0.28915977478027344, + "loss": 1.7186968326568604, + "loss/core": 1.7186968326568604, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 3.748534679412842, + "rewards/margins": 2.381950855255127, + "rewards/rejected": 1.3665834665298462, + "step": 375 + }, + { + "epoch": 0.36307177833512483, + "grad_norm": 47.0, + "learning_rate": 3.5872046928983623e-07, + "logits/chosen": -1.9712073802947998, + "logits/rejected": -2.0564160346984863, + "logps/chosen": -0.28708744049072266, + "logps/rejected": -0.308341920375824, + "loss": 1.9703826904296875, + "loss/core": 1.9703826904296875, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.5351765155792236, + "rewards/margins": 1.9188482761383057, + "rewards/rejected": 1.616328239440918, + "step": 380 + }, + { + "epoch": 0.36784903857637646, + "grad_norm": 286.0, + "learning_rate": 3.5433461362045447e-07, + "logits/chosen": -2.1837782859802246, + "logits/rejected": -2.109261989593506, + "logps/chosen": -0.3028949201107025, + "logps/rejected": -0.3316383957862854, + "loss": 1.2228304147720337, + "loss/core": 1.2228304147720337, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.1072325706481934, + "rewards/margins": 1.2266871929168701, + "rewards/rejected": 1.8805453777313232, + "step": 385 + }, + { + "epoch": 0.3726262988176281, + "grad_norm": 2.671875, + "learning_rate": 3.4990952196986305e-07, + "logits/chosen": -2.0442094802856445, + "logits/rejected": -2.072903633117676, + "logps/chosen": -0.2997892498970032, + "logps/rejected": -0.30010247230529785, + "loss": 0.475435346364975, + "loss/core": 0.475435346364975, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.0823211669921875, + "rewards/margins": 1.195578694343567, + "rewards/rejected": 0.8867424726486206, + "step": 390 + }, + { + "epoch": 0.37740355905887973, + "grad_norm": 7.5625, + "learning_rate": 3.4544685843405875e-07, + "logits/chosen": -2.156062364578247, + "logits/rejected": -2.2315170764923096, + "logps/chosen": -0.2756079435348511, + "logps/rejected": -0.2708030045032501, + "loss": 0.1924670934677124, + "loss/core": 0.1924670934677124, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.018162965774536, + "rewards/margins": 1.0384140014648438, + "rewards/rejected": 0.9797487258911133, + "step": 395 + }, + { + "epoch": 0.38218081930013137, + "grad_norm": 25.25, + "learning_rate": 3.4094830123828786e-07, + "logits/chosen": -2.1258950233459473, + "logits/rejected": -2.064730405807495, + "logps/chosen": -0.27466386556625366, + "logps/rejected": -0.28442296385765076, + "loss": 0.35562631487846375, + "loss/core": 0.35562631487846375, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.0544662475585938, + "rewards/margins": 0.5271926522254944, + "rewards/rejected": 1.5272737741470337, + "step": 400 + }, + { + "epoch": 0.386958079541383, + "grad_norm": 1208.0, + "learning_rate": 3.3641554210593414e-07, + "logits/chosen": -2.2992489337921143, + "logits/rejected": -2.3570144176483154, + "logps/chosen": -0.266804575920105, + "logps/rejected": -0.265764981508255, + "loss": 1.0862951278686523, + "loss/core": 1.0862951278686523, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.5454628467559814, + "rewards/margins": 1.311475157737732, + "rewards/rejected": 1.2339876890182495, + "step": 405 + }, + { + "epoch": 0.39173533978263464, + "grad_norm": 11.25, + "learning_rate": 3.3185028562233107e-07, + "logits/chosen": -2.073944568634033, + "logits/rejected": -2.1331162452697754, + "logps/chosen": -0.3112749457359314, + "logps/rejected": -0.3046428859233856, + "loss": 0.09161730855703354, + "loss/core": 0.09161730855703354, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.2061810493469238, + "rewards/margins": 0.551508903503418, + "rewards/rejected": 0.6546720266342163, + "step": 410 + }, + { + "epoch": 0.3965126000238863, + "grad_norm": 4.96875, + "learning_rate": 3.272542485937368e-07, + "logits/chosen": -2.2970967292785645, + "logits/rejected": -2.295583724975586, + "logps/chosen": -0.29680368304252625, + "logps/rejected": -0.28816667199134827, + "loss": 0.5273622274398804, + "loss/core": 0.5273622274398804, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.4427194595336914, + "rewards/margins": 1.134141206741333, + "rewards/rejected": 1.3085782527923584, + "step": 415 + }, + { + "epoch": 0.40128986026513797, + "grad_norm": 508.0, + "learning_rate": 3.226291594017137e-07, + "logits/chosen": -2.098283290863037, + "logits/rejected": -2.067476749420166, + "logps/chosen": -0.3195353150367737, + "logps/rejected": -0.3052719235420227, + "loss": 1.6536753177642822, + "loss/core": 1.6536753177642822, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.3768184185028076, + "rewards/margins": 2.38930344581604, + "rewards/rejected": 0.9875147938728333, + "step": 420 + }, + { + "epoch": 0.4060671205063896, + "grad_norm": 23.625, + "learning_rate": 3.1797675735315454e-07, + "logits/chosen": -2.124053478240967, + "logits/rejected": -2.2029757499694824, + "logps/chosen": -0.31783485412597656, + "logps/rejected": -0.31542786955833435, + "loss": 1.095782995223999, + "loss/core": 1.095782995223999, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.0307421684265137, + "rewards/margins": 1.9506139755249023, + "rewards/rejected": 1.0801277160644531, + "step": 425 + }, + { + "epoch": 0.41084438074764124, + "grad_norm": 66.5, + "learning_rate": 3.1329879202620047e-07, + "logits/chosen": -2.1495888233184814, + "logits/rejected": -2.263394832611084, + "logps/chosen": -0.26696711778640747, + "logps/rejected": -0.2735012173652649, + "loss": 0.14888674020767212, + "loss/core": 0.14888674020767212, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.4852890968322754, + "rewards/margins": 0.5966399908065796, + "rewards/rejected": 0.8886489868164062, + "step": 430 + }, + { + "epoch": 0.41562164098889287, + "grad_norm": 1328.0, + "learning_rate": 3.0859702261229613e-07, + "logits/chosen": -1.9643999338150024, + "logits/rejected": -1.975927710533142, + "logps/chosen": -0.2912464141845703, + "logps/rejected": -0.2692493498325348, + "loss": 1.1893305778503418, + "loss/core": 1.1893305778503418, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.8585808277130127, + "rewards/margins": 2.5125555992126465, + "rewards/rejected": 1.3460257053375244, + "step": 435 + }, + { + "epoch": 0.4203989012301445, + "grad_norm": 42.5, + "learning_rate": 3.0387321725463e-07, + "logits/chosen": -2.1609604358673096, + "logits/rejected": -2.1351842880249023, + "logps/chosen": -0.29672715067863464, + "logps/rejected": -0.2880227565765381, + "loss": 0.2775880694389343, + "loss/core": 0.2775880694389343, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.147207736968994, + "rewards/margins": 0.9227482080459595, + "rewards/rejected": 1.2244592905044556, + "step": 440 + }, + { + "epoch": 0.42517616147139614, + "grad_norm": 97.0, + "learning_rate": 2.991291523832075e-07, + "logits/chosen": -2.0940496921539307, + "logits/rejected": -2.1732332706451416, + "logps/chosen": -0.285409152507782, + "logps/rejected": -0.27999910712242126, + "loss": 0.576115071773529, + "loss/core": 0.576115071773529, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.9870572090148926, + "rewards/margins": 1.8674147129058838, + "rewards/rejected": 1.1196422576904297, + "step": 445 + }, + { + "epoch": 0.4299534217126478, + "grad_norm": 1600.0, + "learning_rate": 2.943666120468088e-07, + "logits/chosen": -2.1286191940307617, + "logits/rejected": -2.199714422225952, + "logps/chosen": -0.2662820518016815, + "logps/rejected": -0.30065736174583435, + "loss": 0.9491519927978516, + "loss/core": 0.9491519927978516, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.870041608810425, + "rewards/margins": 1.6734178066253662, + "rewards/rejected": 1.1966238021850586, + "step": 450 + }, + { + "epoch": 0.4347306819538994, + "grad_norm": 1.9921875, + "learning_rate": 2.8958738724208073e-07, + "logits/chosen": -1.8988832235336304, + "logits/rejected": -1.9345617294311523, + "logps/chosen": -0.30704838037490845, + "logps/rejected": -0.3010723292827606, + "loss": 0.18666288256645203, + "loss/core": 0.18666288256645203, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.7061916589736938, + "rewards/margins": 0.875575065612793, + "rewards/rejected": 0.8306165933609009, + "step": 455 + }, + { + "epoch": 0.4395079421951511, + "grad_norm": 10.3125, + "learning_rate": 2.8479327524001633e-07, + "logits/chosen": -2.0395889282226562, + "logits/rejected": -2.1937108039855957, + "logps/chosen": -0.2862888276576996, + "logps/rejected": -0.28865212202072144, + "loss": 0.35019630193710327, + "loss/core": 0.35019630193710327, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.244964838027954, + "rewards/margins": 0.8104959726333618, + "rewards/rejected": 1.4344688653945923, + "step": 460 + }, + { + "epoch": 0.44428520243640274, + "grad_norm": 32.5, + "learning_rate": 2.7998607891007493e-07, + "logits/chosen": -2.112020254135132, + "logits/rejected": -2.0445303916931152, + "logps/chosen": -0.2725178003311157, + "logps/rejected": -0.30414050817489624, + "loss": 1.0059139728546143, + "loss/core": 1.0059139728546143, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.83925199508667, + "rewards/margins": 0.9617875814437866, + "rewards/rejected": 1.8774642944335938, + "step": 465 + }, + { + "epoch": 0.4490624626776544, + "grad_norm": 102.5, + "learning_rate": 2.7516760604219616e-07, + "logits/chosen": -2.1016361713409424, + "logits/rejected": -2.077967643737793, + "logps/chosen": -0.2712469696998596, + "logps/rejected": -0.29456794261932373, + "loss": 0.46677565574645996, + "loss/core": 0.46677565574645996, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.2126553058624268, + "rewards/margins": 0.6549152135848999, + "rewards/rejected": 1.557740330696106, + "step": 470 + }, + { + "epoch": 0.453839722918906, + "grad_norm": 11.6875, + "learning_rate": 2.703396686669646e-07, + "logits/chosen": -2.209343433380127, + "logits/rejected": -2.228372097015381, + "logps/chosen": -0.27892497181892395, + "logps/rejected": -0.2862241864204407, + "loss": 0.31267350912094116, + "loss/core": 0.31267350912094116, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.3414828777313232, + "rewards/margins": 0.8732233047485352, + "rewards/rejected": 1.468259572982788, + "step": 475 + }, + { + "epoch": 0.45861698316015764, + "grad_norm": 1168.0, + "learning_rate": 2.6550408237417884e-07, + "logits/chosen": -1.982518196105957, + "logits/rejected": -1.9973068237304688, + "logps/chosen": -0.28591465950012207, + "logps/rejected": -0.2720577120780945, + "loss": 2.5910723209381104, + "loss/core": 2.5910723209381104, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 5.177981376647949, + "rewards/margins": 3.9536633491516113, + "rewards/rejected": 1.2243181467056274, + "step": 480 + }, + { + "epoch": 0.4633942434014093, + "grad_norm": 11.4375, + "learning_rate": 2.6066266563008265e-07, + "logits/chosen": -2.1890599727630615, + "logits/rejected": -2.172191858291626, + "logps/chosen": -0.2838982939720154, + "logps/rejected": -0.30362966656684875, + "loss": 0.41411668062210083, + "loss/core": 0.41411668062210083, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.087114095687866, + "rewards/margins": 1.0661743879318237, + "rewards/rejected": 1.020939588546753, + "step": 485 + }, + { + "epoch": 0.4681715036426609, + "grad_norm": 44.0, + "learning_rate": 2.5581723909351404e-07, + "logits/chosen": -2.1797804832458496, + "logits/rejected": -2.127960681915283, + "logps/chosen": -0.2894228994846344, + "logps/rejected": -0.3229179084300995, + "loss": 0.21129579842090607, + "loss/core": 0.21129579842090607, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.6542030572891235, + "rewards/margins": 0.47612959146499634, + "rewards/rejected": 1.1780736446380615, + "step": 490 + }, + { + "epoch": 0.47294876388391255, + "grad_norm": 1896.0, + "learning_rate": 2.509696249312301e-07, + "logits/chosen": -2.0821642875671387, + "logits/rejected": -2.087721347808838, + "logps/chosen": -0.3256458342075348, + "logps/rejected": -0.3126862943172455, + "loss": 1.8491243124008179, + "loss/core": 1.8491243124008179, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.5472946166992188, + "rewards/margins": 1.9033483266830444, + "rewards/rejected": 1.6439464092254639, + "step": 495 + }, + { + "epoch": 0.47772602412516424, + "grad_norm": 400.0, + "learning_rate": 2.461216461326642e-07, + "logits/chosen": -2.1551461219787598, + "logits/rejected": -2.116971015930176, + "logps/chosen": -0.279205858707428, + "logps/rejected": -0.27398478984832764, + "loss": 1.2034151554107666, + "loss/core": 1.2034151554107666, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.4934756755828857, + "rewards/margins": 2.316671371459961, + "rewards/rejected": 1.176804542541504, + "step": 500 + }, + { + "epoch": 0.4825032843664159, + "grad_norm": 7.875, + "learning_rate": 2.412751258243748e-07, + "logits/chosen": -1.734514832496643, + "logits/rejected": -1.755017638206482, + "logps/chosen": -0.3091053366661072, + "logps/rejected": -0.31018969416618347, + "loss": 0.7064386606216431, + "loss/core": 0.7064386606216431, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.180860996246338, + "rewards/margins": 1.8545881509780884, + "rewards/rejected": 1.32627272605896, + "step": 505 + }, + { + "epoch": 0.4872805446076675, + "grad_norm": 502.0, + "learning_rate": 2.3643188658444158e-07, + "logits/chosen": -2.0605289936065674, + "logits/rejected": -2.1547398567199707, + "logps/chosen": -0.2701338529586792, + "logps/rejected": -0.2844063639640808, + "loss": 0.5944812297821045, + "loss/core": 0.5944812297821045, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.014646053314209, + "rewards/margins": 1.663463830947876, + "rewards/rejected": 1.351182222366333, + "step": 510 + }, + { + "epoch": 0.49205780484891914, + "grad_norm": 14.5, + "learning_rate": 2.315937497570688e-07, + "logits/chosen": -1.9274842739105225, + "logits/rejected": -2.0313713550567627, + "logps/chosen": -0.2993752360343933, + "logps/rejected": -0.3588908612728119, + "loss": 0.5572088956832886, + "loss/core": 0.5572088956832886, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.2762691974639893, + "rewards/margins": 1.200260043144226, + "rewards/rejected": 1.0760090351104736, + "step": 515 + }, + { + "epoch": 0.4968350650901708, + "grad_norm": 2.265625, + "learning_rate": 2.2676253476765194e-07, + "logits/chosen": -1.9769691228866577, + "logits/rejected": -2.1776771545410156, + "logps/chosen": -0.32004016637802124, + "logps/rejected": -0.2949730157852173, + "loss": 0.6923333406448364, + "loss/core": 0.6923333406448364, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.3702404499053955, + "rewards/margins": 1.4442033767700195, + "rewards/rejected": 0.9260371923446655, + "step": 520 + }, + { + "epoch": 0.5016123253314224, + "grad_norm": 0.89453125, + "learning_rate": 2.2194005843856633e-07, + "logits/chosen": -2.003140687942505, + "logits/rejected": -2.0340893268585205, + "logps/chosen": -0.28318530321121216, + "logps/rejected": -0.281009316444397, + "loss": 0.7909449338912964, + "loss/core": 0.7909449338912964, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.3549766540527344, + "rewards/margins": 2.027585744857788, + "rewards/rejected": 1.3273913860321045, + "step": 525 + }, + { + "epoch": 0.5063895855726741, + "grad_norm": 7.53125, + "learning_rate": 2.1712813430593434e-07, + "logits/chosen": -2.0626168251037598, + "logits/rejected": -2.1819634437561035, + "logps/chosen": -0.2981792986392975, + "logps/rejected": -0.297743558883667, + "loss": 0.17196998000144958, + "loss/core": 0.17196998000144958, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.795514464378357, + "rewards/margins": 0.9798916578292847, + "rewards/rejected": 0.8156226873397827, + "step": 530 + }, + { + "epoch": 0.5111668458139257, + "grad_norm": 39.5, + "learning_rate": 2.123285719376292e-07, + "logits/chosen": -2.0366547107696533, + "logits/rejected": -2.0415263175964355, + "logps/chosen": -0.28957507014274597, + "logps/rejected": -0.28460192680358887, + "loss": 1.1312463283538818, + "loss/core": 1.1312463283538818, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.3058319091796875, + "rewards/margins": 1.5662612915039062, + "rewards/rejected": 1.7395708560943604, + "step": 535 + }, + { + "epoch": 0.5159441060551774, + "grad_norm": 43.5, + "learning_rate": 2.0754317625276982e-07, + "logits/chosen": -2.0326008796691895, + "logits/rejected": -2.0860085487365723, + "logps/chosen": -0.2913965582847595, + "logps/rejected": -0.2849191427230835, + "loss": 0.28950196504592896, + "loss/core": 0.28950196504592896, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.971967101097107, + "rewards/margins": 1.0436313152313232, + "rewards/rejected": 0.9283358454704285, + "step": 540 + }, + { + "epoch": 0.520721366296429, + "grad_norm": 208.0, + "learning_rate": 2.0277374684296498e-07, + "logits/chosen": -2.021231174468994, + "logits/rejected": -2.0813746452331543, + "logps/chosen": -0.2889094650745392, + "logps/rejected": -0.27322858572006226, + "loss": 0.8897081613540649, + "loss/core": 0.8897081613540649, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.6772232055664062, + "rewards/margins": 2.866091012954712, + "rewards/rejected": 0.8111323118209839, + "step": 545 + }, + { + "epoch": 0.5254986265376806, + "grad_norm": 6.90625, + "learning_rate": 1.980220772955602e-07, + "logits/chosen": -1.9826974868774414, + "logits/rejected": -2.0089569091796875, + "logps/chosen": -0.2836936116218567, + "logps/rejected": -0.28125476837158203, + "loss": 0.17911246418952942, + "loss/core": 0.17911246418952942, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.2390589714050293, + "rewards/margins": 1.1507548093795776, + "rewards/rejected": 1.0883041620254517, + "step": 550 + }, + { + "epoch": 0.5302758867789323, + "grad_norm": 15.25, + "learning_rate": 1.932899545191433e-07, + "logits/chosen": -2.034287452697754, + "logits/rejected": -2.1081697940826416, + "logps/chosen": -0.3206217288970947, + "logps/rejected": -0.30311864614486694, + "loss": 1.040518045425415, + "loss/core": 1.040518045425415, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.1932551860809326, + "rewards/margins": 1.7633758783340454, + "rewards/rejected": 1.429879069328308, + "step": 555 + }, + { + "epoch": 0.5350531470201839, + "grad_norm": 14.6875, + "learning_rate": 1.885791580715609e-07, + "logits/chosen": -1.850868582725525, + "logits/rejected": -1.8559157848358154, + "logps/chosen": -0.3106183409690857, + "logps/rejected": -0.29663991928100586, + "loss": 0.6726303100585938, + "loss/core": 0.6726303100585938, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.4848763942718506, + "rewards/margins": 1.1928892135620117, + "rewards/rejected": 1.291987419128418, + "step": 560 + }, + { + "epoch": 0.5398304072614356, + "grad_norm": 45.25, + "learning_rate": 1.8389145949069951e-07, + "logits/chosen": -1.9359796047210693, + "logits/rejected": -2.0416994094848633, + "logps/chosen": -0.28728121519088745, + "logps/rejected": -0.2942139506340027, + "loss": 2.735677719116211, + "loss/core": 2.735677719116211, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 5.07828950881958, + "rewards/margins": 3.2359542846679688, + "rewards/rejected": 1.8423347473144531, + "step": 565 + }, + { + "epoch": 0.5446076675026872, + "grad_norm": 91.5, + "learning_rate": 1.792286216282824e-07, + "logits/chosen": -2.077359437942505, + "logits/rejected": -2.0332412719726562, + "logps/chosen": -0.3251068890094757, + "logps/rejected": -0.31182152032852173, + "loss": 0.7743596434593201, + "loss/core": 0.7743596434593201, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.263739585876465, + "rewards/margins": 0.2986152172088623, + "rewards/rejected": 1.965124487876892, + "step": 570 + }, + { + "epoch": 0.5493849277439389, + "grad_norm": 20.625, + "learning_rate": 1.745923979869336e-07, + "logits/chosen": -2.3297030925750732, + "logits/rejected": -2.3048276901245117, + "logps/chosen": -0.28046730160713196, + "logps/rejected": -0.2963675260543823, + "loss": 0.5231956839561462, + "loss/core": 0.5231956839561462, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.400116443634033, + "rewards/margins": 1.4617714881896973, + "rewards/rejected": 0.9383450746536255, + "step": 575 + }, + { + "epoch": 0.5541621879851905, + "grad_norm": 8.8125, + "learning_rate": 1.6998453206075708e-07, + "logits/chosen": -2.00852632522583, + "logits/rejected": -2.128060817718506, + "logps/chosen": -0.30730748176574707, + "logps/rejected": -0.3067978024482727, + "loss": 0.4244763255119324, + "loss/core": 0.4244763255119324, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.4532878398895264, + "rewards/margins": 1.449927568435669, + "rewards/rejected": 1.0033600330352783, + "step": 580 + }, + { + "epoch": 0.5589394482264421, + "grad_norm": 234.0, + "learning_rate": 1.6540675667967973e-07, + "logits/chosen": -2.2257282733917236, + "logits/rejected": -2.239046096801758, + "logps/chosen": -0.2853415906429291, + "logps/rejected": -0.28488466143608093, + "loss": 0.8494731187820435, + "loss/core": 0.8494731187820435, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.5698318481445312, + "rewards/margins": 0.717768669128418, + "rewards/rejected": 1.8520629405975342, + "step": 585 + }, + { + "epoch": 0.5637167084676937, + "grad_norm": 185.0, + "learning_rate": 1.60860793357805e-07, + "logits/chosen": -1.9403209686279297, + "logits/rejected": -1.9107745885849, + "logps/chosen": -0.30443838238716125, + "logps/rejected": -0.3014160990715027, + "loss": 0.7370032668113708, + "loss/core": 0.7370032668113708, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.7498955726623535, + "rewards/margins": 1.003064513206482, + "rewards/rejected": 1.746830940246582, + "step": 590 + }, + { + "epoch": 0.5684939687089454, + "grad_norm": 2.40625, + "learning_rate": 1.5634835164602196e-07, + "logits/chosen": -2.0419182777404785, + "logits/rejected": -1.969354271888733, + "logps/chosen": -0.2744390368461609, + "logps/rejected": -0.2946432828903198, + "loss": 0.6912421584129333, + "loss/core": 0.6912421584129333, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.8153032064437866, + "rewards/margins": -0.3735305368900299, + "rewards/rejected": 2.188833713531494, + "step": 595 + }, + { + "epoch": 0.5732712289501971, + "grad_norm": 40.25, + "learning_rate": 1.518711284891132e-07, + "logits/chosen": -2.1388328075408936, + "logits/rejected": -2.0539865493774414, + "logps/chosen": -0.2972756028175354, + "logps/rejected": -0.31171417236328125, + "loss": 0.36328691244125366, + "loss/core": 0.36328691244125366, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.7741825580596924, + "rewards/margins": 0.6642401814460754, + "rewards/rejected": 1.1099423170089722, + "step": 600 + }, + { + "epoch": 0.5780484891914487, + "grad_norm": 28.5, + "learning_rate": 1.47430807587603e-07, + "logits/chosen": -2.2820992469787598, + "logits/rejected": -2.370114803314209, + "logps/chosen": -0.29316216707229614, + "logps/rejected": -0.3045360743999481, + "loss": 0.418804794549942, + "loss/core": 0.418804794549942, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.924261450767517, + "rewards/margins": 1.342273473739624, + "rewards/rejected": 0.5819882154464722, + "step": 605 + }, + { + "epoch": 0.5828257494327004, + "grad_norm": 5.03125, + "learning_rate": 1.430290587645865e-07, + "logits/chosen": -2.0365357398986816, + "logits/rejected": -2.1053943634033203, + "logps/chosen": -0.30090898275375366, + "logps/rejected": -0.30962470173835754, + "loss": 0.7318702936172485, + "loss/core": 0.7318702936172485, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.31716251373291, + "rewards/margins": 0.7470144033432007, + "rewards/rejected": 1.5701478719711304, + "step": 610 + }, + { + "epoch": 0.587603009673952, + "grad_norm": 1.0390625, + "learning_rate": 1.3866753733777765e-07, + "logits/chosen": -2.0896778106689453, + "logits/rejected": -2.0692594051361084, + "logps/chosen": -0.3204464912414551, + "logps/rejected": -0.31083038449287415, + "loss": 0.15200874209403992, + "loss/core": 0.15200874209403992, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.405200481414795, + "rewards/margins": 0.6567575335502625, + "rewards/rejected": 0.7484430074691772, + "step": 615 + }, + { + "epoch": 0.5923802699152036, + "grad_norm": 10.25, + "learning_rate": 1.343478834970121e-07, + "logits/chosen": -2.2016444206237793, + "logits/rejected": -2.162689447402954, + "logps/chosen": -0.2618758976459503, + "logps/rejected": -0.28418415784835815, + "loss": 0.2642122209072113, + "loss/core": 0.2642122209072113, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.8917810916900635, + "rewards/margins": 0.6098601818084717, + "rewards/rejected": 1.2819210290908813, + "step": 620 + }, + { + "epoch": 0.5971575301564552, + "grad_norm": 13.5, + "learning_rate": 1.3007172168743852e-07, + "logits/chosen": -2.0167224407196045, + "logits/rejected": -1.9906463623046875, + "logps/chosen": -0.2946910262107849, + "logps/rejected": -0.29906517267227173, + "loss": 0.1281525194644928, + "loss/core": 0.1281525194644928, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.5117716789245605, + "rewards/margins": 0.6090092658996582, + "rewards/rejected": 0.9027624130249023, + "step": 625 + }, + { + "epoch": 0.6019347903977069, + "grad_norm": 48.0, + "learning_rate": 1.25840659998631e-07, + "logits/chosen": -2.24929141998291, + "logits/rejected": -2.1729183197021484, + "logps/chosen": -0.2794455885887146, + "logps/rejected": -0.2963079810142517, + "loss": 0.09536317735910416, + "loss/core": 0.09536317735910416, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.2493692636489868, + "rewards/margins": 0.36890143156051636, + "rewards/rejected": 0.8804677724838257, + "step": 630 + }, + { + "epoch": 0.6067120506389586, + "grad_norm": 0.60546875, + "learning_rate": 1.2165628955985313e-07, + "logits/chosen": -2.070937395095825, + "logits/rejected": -2.0991628170013428, + "logps/chosen": -0.30978572368621826, + "logps/rejected": -0.2937181890010834, + "loss": 1.2708914279937744, + "loss/core": 1.2708914279937744, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.1841537952423096, + "rewards/margins": 2.332937240600586, + "rewards/rejected": 0.851216197013855, + "step": 635 + }, + { + "epoch": 0.6114893108802102, + "grad_norm": 255.0, + "learning_rate": 1.175201839416988e-07, + "logits/chosen": -1.9998000860214233, + "logits/rejected": -2.1376776695251465, + "logps/chosen": -0.2979816794395447, + "logps/rejected": -0.2850627899169922, + "loss": 0.48012876510620117, + "loss/core": 0.48012876510620117, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.3821167945861816, + "rewards/margins": 1.731583833694458, + "rewards/rejected": 0.6505327820777893, + "step": 640 + }, + { + "epoch": 0.6162665711214619, + "grad_norm": 13.0, + "learning_rate": 1.1343389856433658e-07, + "logits/chosen": -2.054605484008789, + "logits/rejected": -2.1557681560516357, + "logps/chosen": -0.2970808148384094, + "logps/rejected": -0.26274365186691284, + "loss": 2.3463776111602783, + "loss/core": 2.3463776111602783, + "rewards/accuracies": 0.9375, + "rewards/chosen": 4.414872169494629, + "rewards/margins": 2.85589337348938, + "rewards/rejected": 1.5589792728424072, + "step": 645 + }, + { + "epoch": 0.6210438313627135, + "grad_norm": 7.71875, + "learning_rate": 1.0939897011258e-07, + "logits/chosen": -2.2726993560791016, + "logits/rejected": -2.288119077682495, + "logps/chosen": -0.3080156445503235, + "logps/rejected": -0.2879447340965271, + "loss": 1.3906691074371338, + "loss/core": 1.3906691074371338, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.284862995147705, + "rewards/margins": 2.400153875350952, + "rewards/rejected": 0.8847091794013977, + "step": 650 + }, + { + "epoch": 0.6258210916039652, + "grad_norm": 17.75, + "learning_rate": 1.0541691595800336e-07, + "logits/chosen": -1.922623872756958, + "logits/rejected": -1.9154341220855713, + "logps/chosen": -0.28070202469825745, + "logps/rejected": -0.3093103766441345, + "loss": 0.5413604974746704, + "loss/core": 0.5413604974746704, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 2.844754457473755, + "rewards/margins": 1.1954736709594727, + "rewards/rejected": 1.6492809057235718, + "step": 655 + }, + { + "epoch": 0.6305983518452167, + "grad_norm": 16.75, + "learning_rate": 1.0148923358832021e-07, + "logits/chosen": -2.2555432319641113, + "logits/rejected": -2.311061382293701, + "logps/chosen": -0.27462202310562134, + "logps/rejected": -0.2811489999294281, + "loss": 0.3550433814525604, + "loss/core": 0.3550433814525604, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.0413260459899902, + "rewards/margins": 0.9709696769714355, + "rewards/rejected": 1.0703566074371338, + "step": 660 + }, + { + "epoch": 0.6353756120864684, + "grad_norm": 26.875, + "learning_rate": 9.761740004423926e-08, + "logits/chosen": -2.12736439704895, + "logits/rejected": -2.2038140296936035, + "logps/chosen": -0.3001564145088196, + "logps/rejected": -0.32446950674057007, + "loss": 0.2872669994831085, + "loss/core": 0.2872669994831085, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.7709327936172485, + "rewards/margins": 0.7759586572647095, + "rewards/rejected": 0.9949741363525391, + "step": 665 + }, + { + "epoch": 0.64015287232772, + "grad_norm": 14.125, + "learning_rate": 9.380287136400999e-08, + "logits/chosen": -2.269469738006592, + "logits/rejected": -2.301842212677002, + "logps/chosen": -0.285754531621933, + "logps/rejected": -0.2945065200328827, + "loss": 0.39818090200424194, + "loss/core": 0.39818090200424194, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9868361949920654, + "rewards/margins": 0.8125089406967163, + "rewards/rejected": 1.1743274927139282, + "step": 670 + }, + { + "epoch": 0.6449301325689717, + "grad_norm": 23.375, + "learning_rate": 9.004708203586628e-08, + "logits/chosen": -2.080825090408325, + "logits/rejected": -2.2083404064178467, + "logps/chosen": -0.2972838282585144, + "logps/rejected": -0.28944870829582214, + "loss": 0.3145357370376587, + "loss/core": 0.3145357370376587, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.903146505355835, + "rewards/margins": 0.8571847677230835, + "rewards/rejected": 1.0459617376327515, + "step": 675 + }, + { + "epoch": 0.6497073928102234, + "grad_norm": 704.0, + "learning_rate": 8.635144445857407e-08, + "logits/chosen": -2.1277308464050293, + "logits/rejected": -2.093435525894165, + "logps/chosen": -0.28518134355545044, + "logps/rejected": -0.2920211851596832, + "loss": 0.24321074783802032, + "loss/core": 0.24321074783802032, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.2132742404937744, + "rewards/margins": -0.060915298759937286, + "rewards/rejected": 1.2741894721984863, + "step": 680 + }, + { + "epoch": 0.654484653051475, + "grad_norm": 392.0, + "learning_rate": 8.271734841028552e-08, + "logits/chosen": -2.1869044303894043, + "logits/rejected": -2.1886041164398193, + "logps/chosen": -0.2989751398563385, + "logps/rejected": -0.30234605073928833, + "loss": 0.5350499153137207, + "loss/core": 0.5350499153137207, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.4281145334243774, + "rewards/margins": -0.04558032751083374, + "rewards/rejected": 1.4736945629119873, + "step": 685 + }, + { + "epoch": 0.6592619132927267, + "grad_norm": 21.625, + "learning_rate": 7.91461605259007e-08, + "logits/chosen": -1.9535391330718994, + "logits/rejected": -1.9515883922576904, + "logps/chosen": -0.26703256368637085, + "logps/rejected": -0.28917747735977173, + "loss": 0.5689412355422974, + "loss/core": 0.5689412355422974, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.011037826538086, + "rewards/margins": 1.4389708042144775, + "rewards/rejected": 1.5720670223236084, + "step": 690 + }, + { + "epoch": 0.6640391735339782, + "grad_norm": 6.96875, + "learning_rate": 7.563922378313217e-08, + "logits/chosen": -2.2548301219940186, + "logits/rejected": -2.1238934993743896, + "logps/chosen": -0.2951969504356384, + "logps/rejected": -0.3159661591053009, + "loss": 0.14107027649879456, + "loss/core": 0.14107027649879456, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.4432185888290405, + "rewards/margins": 0.7674707174301147, + "rewards/rejected": 0.6757478713989258, + "step": 695 + }, + { + "epoch": 0.6688164337752299, + "grad_norm": 183.0, + "learning_rate": 7.219785699746572e-08, + "logits/chosen": -2.0966005325317383, + "logits/rejected": -2.1340537071228027, + "logps/chosen": -0.2893856465816498, + "logps/rejected": -0.2702532708644867, + "loss": 0.855477511882782, + "loss/core": 0.855477511882782, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.0240743160247803, + "rewards/margins": 1.6322101354599, + "rewards/rejected": 1.3918644189834595, + "step": 700 + }, + { + "epoch": 0.6735936940164815, + "grad_norm": 148.0, + "learning_rate": 6.882335432620779e-08, + "logits/chosen": -2.008580446243286, + "logits/rejected": -2.0907862186431885, + "logps/chosen": -0.2771678864955902, + "logps/rejected": -0.2853749692440033, + "loss": 0.19750399887561798, + "loss/core": 0.19750399887561798, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.4055383205413818, + "rewards/margins": 0.3061255216598511, + "rewards/rejected": 1.0994129180908203, + "step": 705 + }, + { + "epoch": 0.6783709542577332, + "grad_norm": 5.15625, + "learning_rate": 6.551698478180589e-08, + "logits/chosen": -2.002742052078247, + "logits/rejected": -2.040894031524658, + "logps/chosen": -0.28560441732406616, + "logps/rejected": -0.2906520962715149, + "loss": 0.36187195777893066, + "loss/core": 0.36187195777893066, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.5490121841430664, + "rewards/margins": 1.2243046760559082, + "rewards/rejected": 1.3247077465057373, + "step": 710 + }, + { + "epoch": 0.6831482144989849, + "grad_norm": 2.125, + "learning_rate": 6.22799917546252e-08, + "logits/chosen": -2.0940136909484863, + "logits/rejected": -2.1399710178375244, + "logps/chosen": -0.28695395588874817, + "logps/rejected": -0.2956109941005707, + "loss": 0.47707152366638184, + "loss/core": 0.47707152366638184, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.8282418251037598, + "rewards/margins": 1.4551374912261963, + "rewards/rejected": 1.3731043338775635, + "step": 715 + }, + { + "epoch": 0.6879254747402365, + "grad_norm": 1.9140625, + "learning_rate": 5.9113592545359944e-08, + "logits/chosen": -2.2517735958099365, + "logits/rejected": -2.2257275581359863, + "logps/chosen": -0.3003559112548828, + "logps/rejected": -0.2943252921104431, + "loss": 0.47043830156326294, + "loss/core": 0.47043830156326294, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.214423418045044, + "rewards/margins": 1.1652237176895142, + "rewards/rejected": 1.0491998195648193, + "step": 720 + }, + { + "epoch": 0.6927027349814882, + "grad_norm": 3.765625, + "learning_rate": 5.601897790725643e-08, + "logits/chosen": -2.103532314300537, + "logits/rejected": -2.1393818855285645, + "logps/chosen": -0.31979304552078247, + "logps/rejected": -0.31625136733055115, + "loss": 0.08710397779941559, + "loss/core": 0.08710397779941559, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.1311936378479004, + "rewards/margins": 0.3907800316810608, + "rewards/rejected": 0.7404135465621948, + "step": 725 + }, + { + "epoch": 0.6974799952227397, + "grad_norm": 19.875, + "learning_rate": 5.299731159831952e-08, + "logits/chosen": -2.185694456100464, + "logits/rejected": -2.1851601600646973, + "logps/chosen": -0.29845187067985535, + "logps/rejected": -0.30084067583084106, + "loss": 0.5585907697677612, + "loss/core": 0.5585907697677612, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.5317673683166504, + "rewards/margins": 1.6783374547958374, + "rewards/rejected": 0.8534297943115234, + "step": 730 + }, + { + "epoch": 0.7022572554639914, + "grad_norm": 7.65625, + "learning_rate": 5.0049729943671013e-08, + "logits/chosen": -2.2042665481567383, + "logits/rejected": -2.212019681930542, + "logps/chosen": -0.270427942276001, + "logps/rejected": -0.29099661111831665, + "loss": 0.2325257807970047, + "loss/core": 0.2325257807970047, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.025036573410034, + "rewards/margins": 0.7466187477111816, + "rewards/rejected": 1.2784178256988525, + "step": 735 + }, + { + "epoch": 0.707034515705243, + "grad_norm": 5.625, + "learning_rate": 4.7177341408223994e-08, + "logits/chosen": -2.023667812347412, + "logits/rejected": -2.055938243865967, + "logps/chosen": -0.267364501953125, + "logps/rejected": -0.284996896982193, + "loss": 0.3424833416938782, + "loss/core": 0.3424833416938782, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.194387912750244, + "rewards/margins": 1.0326035022735596, + "rewards/rejected": 1.1617845296859741, + "step": 740 + }, + { + "epoch": 0.7118117759464947, + "grad_norm": 62.25, + "learning_rate": 4.438122617983442e-08, + "logits/chosen": -1.9847673177719116, + "logits/rejected": -2.012366771697998, + "logps/chosen": -0.2840479910373688, + "logps/rejected": -0.27548593282699585, + "loss": 1.2490966320037842, + "loss/core": 1.2490966320037842, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.142892360687256, + "rewards/margins": 1.1425834894180298, + "rewards/rejected": 2.0003087520599365, + "step": 745 + }, + { + "epoch": 0.7165890361877463, + "grad_norm": 10.9375, + "learning_rate": 4.1662435763087114e-08, + "logits/chosen": -2.047241687774658, + "logits/rejected": -2.079470634460449, + "logps/chosen": -0.2526155114173889, + "logps/rejected": -0.2926296293735504, + "loss": 0.3922721743583679, + "loss/core": 0.3922721743583679, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.4976963996887207, + "rewards/margins": 1.1639492511749268, + "rewards/rejected": 1.333747148513794, + "step": 750 + }, + { + "epoch": 0.721366296428998, + "grad_norm": 1.0078125, + "learning_rate": 3.902199258386732e-08, + "logits/chosen": -2.1882448196411133, + "logits/rejected": -2.1855483055114746, + "logps/chosen": -0.36757299304008484, + "logps/rejected": -0.29461368918418884, + "loss": 1.8846288919448853, + "loss/core": 1.8846288919448853, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.939770221710205, + "rewards/margins": 1.8737109899520874, + "rewards/rejected": 2.06605863571167, + "step": 755 + }, + { + "epoch": 0.7261435566702497, + "grad_norm": 6.875, + "learning_rate": 3.646088960486862e-08, + "logits/chosen": -2.087844133377075, + "logits/rejected": -2.1318325996398926, + "logps/chosen": -0.2983017563819885, + "logps/rejected": -0.30041682720184326, + "loss": 0.2357165366411209, + "loss/core": 0.2357165366411209, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.57404363155365, + "rewards/margins": 0.8634489178657532, + "rewards/rejected": 0.7105947136878967, + "step": 760 + }, + { + "epoch": 0.7309208169115012, + "grad_norm": 4.3125, + "learning_rate": 3.398008995217988e-08, + "logits/chosen": -2.253822088241577, + "logits/rejected": -2.278644323348999, + "logps/chosen": -0.2737087607383728, + "logps/rejected": -0.2841106057167053, + "loss": 0.0988527461886406, + "loss/core": 0.0988527461886406, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.4775121212005615, + "rewards/margins": 0.5869768261909485, + "rewards/rejected": 0.8905353546142578, + "step": 765 + }, + { + "epoch": 0.7356980771527529, + "grad_norm": 69.0, + "learning_rate": 3.1580526553093315e-08, + "logits/chosen": -2.1181020736694336, + "logits/rejected": -2.1448464393615723, + "logps/chosen": -0.30012568831443787, + "logps/rejected": -0.2899145185947418, + "loss": 0.3107560873031616, + "loss/core": 0.3107560873031616, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.013152599334717, + "rewards/margins": 0.9449566006660461, + "rewards/rejected": 1.0681959390640259, + "step": 770 + }, + { + "epoch": 0.7404753373940045, + "grad_norm": 251.0, + "learning_rate": 2.9263101785268252e-08, + "logits/chosen": -2.248095750808716, + "logits/rejected": -2.2297964096069336, + "logps/chosen": -0.27153995633125305, + "logps/rejected": -0.27207639813423157, + "loss": 0.322858989238739, + "loss/core": 0.322858989238739, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 2.0420784950256348, + "rewards/margins": 1.176545262336731, + "rewards/rejected": 0.8655333518981934, + "step": 775 + }, + { + "epoch": 0.7452525976352562, + "grad_norm": 20.625, + "learning_rate": 2.7028687137384264e-08, + "logits/chosen": -2.028242826461792, + "logits/rejected": -2.135806083679199, + "logps/chosen": -0.2820461392402649, + "logps/rejected": -0.2808576226234436, + "loss": 0.5030468702316284, + "loss/core": 0.5030468702316284, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.855433940887451, + "rewards/margins": 1.1535121202468872, + "rewards/rejected": 1.701921820640564, + "step": 780 + }, + { + "epoch": 0.7500298578765078, + "grad_norm": 14.875, + "learning_rate": 2.4878122881409447e-08, + "logits/chosen": -1.9230458736419678, + "logits/rejected": -1.8952453136444092, + "logps/chosen": -0.28769412636756897, + "logps/rejected": -0.2964576482772827, + "loss": 0.5485445261001587, + "loss/core": 0.5485445261001587, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.40129017829895, + "rewards/margins": 0.9055936932563782, + "rewards/rejected": 1.4956964254379272, + "step": 785 + }, + { + "epoch": 0.7548071181177595, + "grad_norm": 3.84375, + "learning_rate": 2.2812217756608937e-08, + "logits/chosen": -2.113828420639038, + "logits/rejected": -2.184523344039917, + "logps/chosen": -0.30777841806411743, + "logps/rejected": -0.34668484330177307, + "loss": 0.178707093000412, + "loss/core": 0.178707093000412, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.7738380432128906, + "rewards/margins": 1.0590564012527466, + "rewards/rejected": 0.7147814631462097, + "step": 790 + }, + { + "epoch": 0.7595843783590112, + "grad_norm": 57.25, + "learning_rate": 2.0831748665410763e-08, + "logits/chosen": -2.0712850093841553, + "logits/rejected": -2.047006368637085, + "logps/chosen": -0.2615426182746887, + "logps/rejected": -0.2844923138618469, + "loss": 0.5626412630081177, + "loss/core": 0.5626412630081177, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.4369277954101562, + "rewards/margins": 0.5682427883148193, + "rewards/rejected": 1.8686845302581787, + "step": 795 + }, + { + "epoch": 0.7643616386002627, + "grad_norm": 9.0, + "learning_rate": 1.8937460381244967e-08, + "logits/chosen": -1.9848846197128296, + "logits/rejected": -2.050729751586914, + "logps/chosen": -0.3180027902126312, + "logps/rejected": -0.2833881378173828, + "loss": 1.3084042072296143, + "loss/core": 1.3084042072296143, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.4368815422058105, + "rewards/margins": 2.4918322563171387, + "rewards/rejected": 0.9450491666793823, + "step": 800 + }, + { + "epoch": 0.7691388988415144, + "grad_norm": 49.0, + "learning_rate": 1.713006526846439e-08, + "logits/chosen": -2.084895610809326, + "logits/rejected": -2.2298989295959473, + "logps/chosen": -0.2837081849575043, + "logps/rejected": -0.28118696808815, + "loss": 0.3433898687362671, + "loss/core": 0.3433898687362671, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.079929828643799, + "rewards/margins": 1.1563465595245361, + "rewards/rejected": 0.9235833287239075, + "step": 805 + }, + { + "epoch": 0.773916159082766, + "grad_norm": 10.9375, + "learning_rate": 1.541024301445404e-08, + "logits/chosen": -2.3721537590026855, + "logits/rejected": -2.3594298362731934, + "logps/chosen": -0.27915048599243164, + "logps/rejected": -0.2653045058250427, + "loss": 0.35306280851364136, + "loss/core": 0.35306280851364136, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.984630823135376, + "rewards/margins": 1.1519166231155396, + "rewards/rejected": 0.8327140808105469, + "step": 810 + }, + { + "epoch": 0.7786934193240177, + "grad_norm": 45.5, + "learning_rate": 1.3778640374027983e-08, + "logits/chosen": -2.0560569763183594, + "logits/rejected": -2.0075690746307373, + "logps/chosen": -0.29053401947021484, + "logps/rejected": -0.3152034282684326, + "loss": 0.1969846785068512, + "loss/core": 0.1969846785068512, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.955904245376587, + "rewards/margins": 0.8895227313041687, + "rewards/rejected": 1.0663814544677734, + "step": 815 + }, + { + "epoch": 0.7834706795652693, + "grad_norm": 30.75, + "learning_rate": 1.2235870926211616e-08, + "logits/chosen": -2.0074410438537598, + "logits/rejected": -1.998299241065979, + "logps/chosen": -0.2895830273628235, + "logps/rejected": -0.2900421619415283, + "loss": 0.6202723979949951, + "loss/core": 0.6202723979949951, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.5479187965393066, + "rewards/margins": 1.322956919670105, + "rewards/rejected": 1.2249616384506226, + "step": 820 + }, + { + "epoch": 0.788247939806521, + "grad_norm": 1.15625, + "learning_rate": 1.0782514843499652e-08, + "logits/chosen": -2.06480073928833, + "logits/rejected": -2.2259457111358643, + "logps/chosen": -0.301548570394516, + "logps/rejected": -0.3503800928592682, + "loss": 0.5075355768203735, + "loss/core": 0.5075355768203735, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.2786221504211426, + "rewards/margins": 1.6066519021987915, + "rewards/rejected": 0.6719702482223511, + "step": 825 + }, + { + "epoch": 0.7930252000477725, + "grad_norm": 55.0, + "learning_rate": 9.419118673676924e-09, + "logits/chosen": -2.1703529357910156, + "logits/rejected": -2.184058666229248, + "logps/chosen": -0.300544798374176, + "logps/rejected": -0.2891280949115753, + "loss": 0.2319040298461914, + "loss/core": 0.2319040298461914, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8630399703979492, + "rewards/margins": 1.0912376642227173, + "rewards/rejected": 0.7718024849891663, + "step": 830 + }, + { + "epoch": 0.7978024602890242, + "grad_norm": 7.75, + "learning_rate": 8.14619513428405e-09, + "logits/chosen": -2.2036571502685547, + "logits/rejected": -2.212895631790161, + "logps/chosen": -0.2536885142326355, + "logps/rejected": -0.2815439701080322, + "loss": 0.3949839472770691, + "loss/core": 0.3949839472770691, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.112321138381958, + "rewards/margins": 1.3582528829574585, + "rewards/rejected": 0.7540683150291443, + "step": 835 + }, + { + "epoch": 0.8025797205302759, + "grad_norm": 1256.0, + "learning_rate": 6.96422291980539e-09, + "logits/chosen": -2.035339832305908, + "logits/rejected": -2.0912997722625732, + "logps/chosen": -0.29334017634391785, + "logps/rejected": -0.29095450043678284, + "loss": 1.3885018825531006, + "loss/core": 1.3885018825531006, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.5305423736572266, + "rewards/margins": 2.387678861618042, + "rewards/rejected": 1.1428637504577637, + "step": 840 + }, + { + "epoch": 0.8073569807715275, + "grad_norm": 58.25, + "learning_rate": 5.8736465216517594e-09, + "logits/chosen": -1.8922698497772217, + "logits/rejected": -1.9340620040893555, + "logps/chosen": -0.30387482047080994, + "logps/rejected": -0.31475311517715454, + "loss": 1.267748236656189, + "loss/core": 1.267748236656189, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.673053503036499, + "rewards/margins": 0.4096958041191101, + "rewards/rejected": 2.263357639312744, + "step": 845 + }, + { + "epoch": 0.8121342410127792, + "grad_norm": 19.0, + "learning_rate": 4.874876061005173e-09, + "logits/chosen": -1.9448999166488647, + "logits/rejected": -2.031862735748291, + "logps/chosen": -0.2876799702644348, + "logps/rejected": -0.29397642612457275, + "loss": 0.3203374445438385, + "loss/core": 0.3203374445438385, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.3861019611358643, + "rewards/margins": 1.2526428699493408, + "rewards/rejected": 1.1334589719772339, + "step": 850 + }, + { + "epoch": 0.8169115012540308, + "grad_norm": 16.5, + "learning_rate": 3.968287134589188e-09, + "logits/chosen": -1.8941272497177124, + "logits/rejected": -1.8450911045074463, + "logps/chosen": -0.285461962223053, + "logps/rejected": -0.28394120931625366, + "loss": 0.4327033460140228, + "loss/core": 0.4327033460140228, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.151184558868408, + "rewards/margins": 0.9308632016181946, + "rewards/rejected": 1.220321536064148, + "step": 855 + }, + { + "epoch": 0.8216887614952825, + "grad_norm": 5.65625, + "learning_rate": 3.154220673422192e-09, + "logits/chosen": -1.9777624607086182, + "logits/rejected": -2.0467350482940674, + "logps/chosen": -0.28861913084983826, + "logps/rejected": -0.3350909352302551, + "loss": 0.501311182975769, + "loss/core": 0.501311182975769, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.9023014307022095, + "rewards/margins": 0.7318955659866333, + "rewards/rejected": 1.1704059839248657, + "step": 860 + }, + { + "epoch": 0.826466021736534, + "grad_norm": 105.5, + "learning_rate": 2.4329828146074096e-09, + "logits/chosen": -1.8944776058197021, + "logits/rejected": -1.953665018081665, + "logps/chosen": -0.2609250247478485, + "logps/rejected": -0.29491904377937317, + "loss": 0.4003438353538513, + "loss/core": 0.4003438353538513, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.2690205574035645, + "rewards/margins": 1.027840256690979, + "rewards/rejected": 1.241180181503296, + "step": 865 + }, + { + "epoch": 0.8312432819777857, + "grad_norm": 924.0, + "learning_rate": 1.8048447862070714e-09, + "logits/chosen": -2.322716236114502, + "logits/rejected": -2.29233980178833, + "logps/chosen": -0.3064175248146057, + "logps/rejected": -0.29991593956947327, + "loss": 1.024073839187622, + "loss/core": 1.024073839187622, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.370023012161255, + "rewards/margins": 1.0268133878707886, + "rewards/rejected": 1.343209981918335, + "step": 870 + }, + { + "epoch": 0.8360205422190374, + "grad_norm": 35.5, + "learning_rate": 1.2700428052447033e-09, + "logits/chosen": -2.17930006980896, + "logits/rejected": -2.034334897994995, + "logps/chosen": -0.27654317021369934, + "logps/rejected": -0.310249388217926, + "loss": 0.34868985414505005, + "loss/core": 0.34868985414505005, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.6901212930679321, + "rewards/margins": -0.11936825513839722, + "rewards/rejected": 1.8094894886016846, + "step": 875 + }, + { + "epoch": 0.840797802460289, + "grad_norm": 12.25, + "learning_rate": 8.287779888734858e-10, + "logits/chosen": -2.0425121784210205, + "logits/rejected": -2.0631158351898193, + "logps/chosen": -0.31236395239830017, + "logps/rejected": -0.32643812894821167, + "loss": 0.7430340647697449, + "loss/core": 0.7430340647697449, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.8382582664489746, + "rewards/margins": 1.6181519031524658, + "rewards/rejected": 1.2201061248779297, + "step": 880 + }, + { + "epoch": 0.8455750627015407, + "grad_norm": 9.375, + "learning_rate": 4.812162787445062e-10, + "logits/chosen": -2.1633739471435547, + "logits/rejected": -2.2033333778381348, + "logps/chosen": -0.309976726770401, + "logps/rejected": -0.3008884787559509, + "loss": 0.05537397414445877, + "loss/core": 0.05537397414445877, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.1624144315719604, + "rewards/margins": 0.6319789290428162, + "rewards/rejected": 0.5304354429244995, + "step": 885 + }, + { + "epoch": 0.8503523229427923, + "grad_norm": 201.0, + "learning_rate": 2.2748837860270265e-10, + "logits/chosen": -2.046548366546631, + "logits/rejected": -2.1142578125, + "logps/chosen": -0.33651334047317505, + "logps/rejected": -0.32960960268974304, + "loss": 0.38854843378067017, + "loss/core": 0.38854843378067017, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.0308008193969727, + "rewards/margins": 1.3045727014541626, + "rewards/rejected": 0.7262283563613892, + "step": 890 + }, + { + "epoch": 0.855129583184044, + "grad_norm": 6.53125, + "learning_rate": 6.768970513457151e-11, + "logits/chosen": -2.0591623783111572, + "logits/rejected": -2.0446290969848633, + "logps/chosen": -0.2869151532649994, + "logps/rejected": -0.2770235538482666, + "loss": 0.31208789348602295, + "loss/core": 0.31208789348602295, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.1741526126861572, + "rewards/margins": 1.1323846578598022, + "rewards/rejected": 1.0417680740356445, + "step": 895 + }, + { + "epoch": 0.8599068434252956, + "grad_norm": 117.5, + "learning_rate": 1.8803520859811406e-12, + "logits/chosen": -2.1362462043762207, + "logits/rejected": -2.0990567207336426, + "logps/chosen": -0.3250855803489685, + "logps/rejected": -0.332465261220932, + "loss": 1.7968904972076416, + "loss/core": 1.7968904972076416, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.322796821594238, + "rewards/margins": 2.9066953659057617, + "rewards/rejected": 1.4161012172698975, + "step": 900 + }, + { + "epoch": 0.8599068434252956, + "step": 900, + "total_flos": 0.0, + "train_loss": 0.6663506550259061, + "train_runtime": 7058.8291, + "train_samples_per_second": 2.04, + "train_steps_per_second": 0.127 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..b873c8d --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f9ede3f864bb24bc565c81dc3f941d7495bc51f0c479d1a1a485d32bc9d994a +size 7057