commit 5ef7c18ef122a503bf3ea17c418dedff6b0c0f7a Author: ModelHub XC Date: Thu Aug 6 12:51:16 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: promotion/qwen3-8b-aaai27-flagship-ipo-s42 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..0adc040 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +--- +library_name: transformers +base_model: Qwen/Qwen3-8B +tags: +- ronpo +- mnpo +- preference-optimization +--- + +# qwen3-8b-aaai27-flagship-ipo-s42 + +Research checkpoint for the RONPO AAAI revision experiments. + +- Method: ipo +- Base model: `Qwen/Qwen3-8B` +- Seed: 42 +- Local source at upload time: `/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/ipo/seed42/attempt1` +- Uploaded at UTC: 2026-07-12T19:49:10Z +- Run status metadata: `{"attempt": 1, "effective_batch_size": 16, "method": "ipo", "objective_protocol": "objective_protocol.json", "optimizer_steps": 900, "resource_profile": "resource_profile.json", "seed": 42, "split_manifest": "split_manifest.json", "stability_gate": "stability_gate.json", "status": "completed", "wandb_run_id": "7929f1a79c80", "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/7929f1a79c80"}` + +AAAI-27 P1 matched budget: 900 optimizer steps, effective batch 16; passed non-thinking and collapse stability gates; sealed-test metrics are not used for checkpoint selection. + +Intended use: reproducibility and evaluation for the RONPO paper. This +checkpoint is not intended as a production assistant. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..3eba55b --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 97.93875423855252, + "train_runtime": 8489.6134, + "train_samples": 16746, + "train_samples_per_second": 1.696, + "train_steps_per_second": 0.106 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..1f13a00 --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +model_name_or_path: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218 +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/avg: 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: ipo +eta: 0.0075 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.0 +preference_sft_weight: 0.0 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 5.0e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 42 +gradient_accumulation_steps: 8 +gradient_checkpointing: false +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 2 +per_device_eval_batch_size: 2 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/ipo/seed42/attempt1 +run_name: aaai27-flagship-full-ipo-s42-a1 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/job_status.json b/job_status.json new file mode 100644 index 0000000..3944ce8 --- /dev/null +++ b/job_status.json @@ -0,0 +1,23 @@ +{ + "status": "completed", + "stage": "full", + "method": "ipo", + "seed": 42, + "attempt": 1, + "gpu": 3, + "gpus": [ + 3 + ], + "returncode": 0, + "finite_metrics": true, + "oom": false, + "traceback": false, + "reason": "passed", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "7929f1a79c80", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/7929f1a79c80", + "output_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/ipo/seed42/attempt1", + "log": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/logs/full_ipo_s42_a1.log", + "completed_at": "2026-07-12T19:46:53Z" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..ea0351f --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c68076adddf0ce69244623661c455d3e1aa46fca6222473b2369e720b61a064 +size 16381517208 diff --git a/objective_protocol.json b/objective_protocol.json new file mode 100644 index 0000000..35b3b44 --- /dev/null +++ b/objective_protocol.json @@ -0,0 +1,69 @@ +{ + "schema_version": 1, + "frozen_at_kst": "2026-07-12T21:40:00+09:00", + "base_model": "Qwen/Qwen3-8B", + "primary_triple": { + "helpfulness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "ultrafeedback-helpfulness", + "transform": "identity" + }, + "safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "conciseness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "helpsteer-verbosity", + "transform": "negate" + } + }, + "gate_thresholds": { + "raw_score_std_strictly_greater_than": 0.01, + "aggregate_prompt_pair_spearman_median_strictly_less_than": 0.0, + "aggregate_pairwise_top1_mismatch_at_least": 0.5 + }, + "gate_pool": "source-train Qwen3-8B response pool split into deterministic train/validation; sealed source-test excluded", + "training_seeds": [42, 43, 44], + "optimizer_steps": 900, + "effective_batch_size": 16, + "decode": { + "enable_thinking": false, + "temperature": 0.7, + "top_p": 0.9, + "max_new_tokens": 2048, + "decode_seed": 42 + }, + "methods": [ + "ronpo_full_expect", + "ronpo_k_only", + "dpo", + "ipo", + "simpo", + "kto", + "sppo_avg", + "inpo_avg", + "ht_mnpo_helpfulness", + "ht_mnpo_safety", + "ht_mnpo_conciseness" + ], + "selection_policy": "No sealed-test selection. One preregistered configuration per method; failures use only the declared stability retry recipe, at most three attempts.", + "stability_gate": { + "validation_prompts": 128, + "min_candidate_base_mean_word_ratio": 0.33, + "max_candidate_base_mean_word_ratio": 2.0, + "max_consecutive_identical_word_run": 20, + "think_tag_count": 0, + "empty_response_count": 0, + "retry_recipe": "attempt 2 halves learning rate, enables gradient checkpointing, and adds reference=0.02/SFT=0.002 anchoring to unanchored unified losses; attempt 3 quarters learning rate and uses reference=0.05/SFT=0.005; RONPO uses reference=0.05/SFT=0.005 from attempt 1" + }, + "primary_metric": "mean prompt-level minimum of per-prompt min-max-normalized helpfulness, safety, conciseness", + "bootstrap": { + "unit": "prompt", + "paired": true, + "resamples": 2000, + "interval": "percentile_95", + "seed": 42 + } +} diff --git a/pair_manifest.json b/pair_manifest.json new file mode 100644 index 0000000..be034b0 --- /dev/null +++ b/pair_manifest.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "objectives": [ + "helpfulness", + "safety", + "conciseness" + ], + "normalization": "per_prompt_minmax", + "adversary": { + "steps": 25, + "alpha": 1.0, + "kappa": 0.05, + "preference_scale": 8.0 + }, + "ronpo_estimator": "Rao-Blackwellized full sigma expectation with three common response pairs per prompt", + "counts": { + "train": { + "avg": 16746, + "ronpo_full_expect": 50340, + "ronpo_k_only": 50340, + "ht_mnpo_helpfulness": 16743, + "ht_mnpo_safety": 16764, + "ht_mnpo_conciseness": 16755 + }, + "validation": { + "avg": 1859, + "ronpo_full_expect": 5592, + "ronpo_k_only": 5592, + "ht_mnpo_helpfulness": 1864, + "ht_mnpo_safety": 1862, + "ht_mnpo_conciseness": 1858 + } + } +} diff --git a/provenance.json b/provenance.json new file mode 100644 index 0000000..07ee76f --- /dev/null +++ b/provenance.json @@ -0,0 +1,13 @@ +{ + "method": "ipo", + "seed": 42, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "7929f1a79c80", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/7929f1a79c80", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json" +} diff --git a/resource_profile.json b/resource_profile.json new file mode 100644 index 0000000..18eec47 --- /dev/null +++ b/resource_profile.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "measured_at_kst": "2026-07-12T22:38:00+09:00", + "hardware": "NVIDIA B200 183359 MiB reported total; 178.35 GiB PyTorch-visible capacity", + "smoke_evidence": { + "attempted_profile": {"per_device_batch_size": 2, "gradient_accumulation_steps": 8, "gradient_checkpointing": false, "effective_batch_size": 16}, + "outcome": "OOM before completing the 20-step smoke across every launched unified baseline; representative process used 176.87 GiB and failed a 1.62 GiB allocation. KTO also OOMed at the optimizer step.", + "paper_metric": false + }, + "selected_full_training_profile": { + "per_device_batch_size": 2, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": false, + "attention_implementation": "PyTorch SDPA with cuDNN SDPA disabled", + "effective_batch_size": 16, + "applies_symmetrically_to_all_unified_pairwise_methods": true, + "kto_implementation_constraint": "TRL KTO requires actual batch size > 1. KTO uses batch 2 / accumulation 8 and identical AdamW with foreach=False. PyTorch SDPA is used with cuDNN SDPA disabled. The fixed base reference model is placed on a second B200 and its exact logits are returned to the policy device; the policy/optimizer remain on the first B200. This avoids a measured ~6-hour reference-precompute path while leaving the KTO loss, actual/effective batch, optimizer, and 900-step budget unchanged." + }, + "retry_semantics": "Resource selection is separated from method-stability attempts. Full attempt 1 uses the smoke-validated SDPA batch-2 profile and retains the preregistered method LR/anchor; attempts 2 and 3 fall back to batch 1 / accumulation 16 / gradient checkpointing and apply the declared LR/anchor stabilization recipe.", + "provenance_note": "No reward or sealed-evaluation result was available when this hardware profile was selected." +} diff --git a/run_status.json b/run_status.json new file mode 100644 index 0000000..8fe6ef0 --- /dev/null +++ b/run_status.json @@ -0,0 +1,14 @@ +{ + "method": "ipo", + "seed": 42, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "7929f1a79c80", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/7929f1a79c80", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json", + "status": "completed" +} diff --git a/split_manifest.json b/split_manifest.json new file mode 100644 index 0000000..4fb96bd --- /dev/null +++ b/split_manifest.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "split_rule": "sort source-train prompts by sha256('flagship-v1||'+prompt_id); first round(10%) validation", + "validation_fraction": 0.1, + "source_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/avg_oracle", + "base_model": "Qwen/Qwen3-8B", + "counts": { + "train": 16780, + "validation": 1864, + "sealed_test": 604 + }, + "prompt_id_sha256": { + "train": "fcf6f2ea5a0b7bcd41179e6f5288f4d850b2d04bdb729da91907bd18bf733a69", + "validation": "a87ba6f519192164daa487b109a6547dd8cbde46a27a373d6da201ca2d061b8f", + "sealed_test": "b398ced6af6cf73d731e1a5f32a0631d8060323d51eda1e375695566bf5f1867" + }, + "overlaps": { + "train_validation": 0, + "train_sealed_test": 0, + "validation_sealed_test": 0 + }, + "sealed_test_policy": "prompts only until S4a; no objective scores or model generations used in S0-S3" +} diff --git a/stability_gate.json b/stability_gate.json new file mode 100644 index 0000000..17a9313 --- /dev/null +++ b/stability_gate.json @@ -0,0 +1,38 @@ +{ + "status": "passed", + "passed": true, + "base": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 388.1484375, + "max_words": 1379, + "max_repeat_run": 4 + }, + "candidate": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 389.0625, + "max_words": 1430, + "max_repeat_run": 5 + }, + "candidate_base_mean_word_ratio": 1.0023549302578347, + "thresholds": { + "min_length_ratio": 0.33, + "max_length_ratio": 2.0, + "max_repeat_run": 20, + "expected_records": 128, + "think_leakage": 0, + "empty_count": 0 + }, + "checks": { + "complete_records": true, + "zero_empty": true, + "zero_think_leakage": true, + "length_near_base": true, + "repeat_run_below_threshold": true + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ffd19d2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..3eba55b --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 97.93875423855252, + "train_runtime": 8489.6134, + "train_samples": 16746, + "train_samples_per_second": 1.696, + "train_steps_per_second": 0.106 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..55b7027 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,2923 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8599068434252956, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.004777260241251642, + "grad_norm": 318.0, + "learning_rate": 2.222222222222222e-08, + "logits/chosen": -1.7139909267425537, + "logits/rejected": -1.6882292032241821, + "logps/chosen": -0.27932801842689514, + "logps/rejected": -0.2822762131690979, + "loss": 97.24929809570312, + "loss/core": 97.24929809570312, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.420787811279297, + "rewards/margins": 1.4634535312652588, + "rewards/rejected": 0.9573341608047485, + "step": 5 + }, + { + "epoch": 0.009554520482503284, + "grad_norm": 2240.0, + "learning_rate": 5e-08, + "logits/chosen": -1.9642683267593384, + "logits/rejected": -2.0518295764923096, + "logps/chosen": -0.28963160514831543, + "logps/rejected": -0.28823795914649963, + "loss": 99.95600128173828, + "loss/core": 99.95600128173828, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.6873565912246704, + "rewards/margins": 0.07187187671661377, + "rewards/rejected": 1.615484595298767, + "step": 10 + }, + { + "epoch": 0.014331780723754926, + "grad_norm": 296.0, + "learning_rate": 7.777777777777778e-08, + "logits/chosen": -1.79877507686615, + "logits/rejected": -1.7281420230865479, + "logps/chosen": -0.27656567096710205, + "logps/rejected": -0.2845936417579651, + "loss": 98.14198303222656, + "loss/core": 98.14198303222656, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.1426477432250977, + "rewards/margins": 1.0079020261764526, + "rewards/rejected": 1.134745478630066, + "step": 15 + }, + { + "epoch": 0.01910904096500657, + "grad_norm": 232.0, + "learning_rate": 1.0555555555555555e-07, + "logits/chosen": -1.9977076053619385, + "logits/rejected": -1.9979604482650757, + "logps/chosen": -0.30147844552993774, + "logps/rejected": -0.28688931465148926, + "loss": 98.10304260253906, + "loss/core": 98.10304260253906, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9145663976669312, + "rewards/margins": 1.0382013320922852, + "rewards/rejected": 0.876365065574646, + "step": 20 + }, + { + "epoch": 0.02388630120625821, + "grad_norm": 184.0, + "learning_rate": 1.3333333333333334e-07, + "logits/chosen": -1.9077049493789673, + "logits/rejected": -1.9242079257965088, + "logps/chosen": -0.2705998420715332, + "logps/rejected": -0.25868508219718933, + "loss": 96.9577407836914, + "loss/core": 96.9577407836914, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.849167585372925, + "rewards/margins": 2.2114977836608887, + "rewards/rejected": 1.637669324874878, + "step": 25 + }, + { + "epoch": 0.028663561447509853, + "grad_norm": 288.0, + "learning_rate": 1.611111111111111e-07, + "logits/chosen": -2.088418483734131, + "logits/rejected": -2.0663094520568848, + "logps/chosen": -0.30147606134414673, + "logps/rejected": -0.3242497146129608, + "loss": 97.91767883300781, + "loss/core": 97.91767883300781, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.264533758163452, + "rewards/margins": 1.2376611232757568, + "rewards/rejected": 1.0268726348876953, + "step": 30 + }, + { + "epoch": 0.033440821688761495, + "grad_norm": 103.0, + "learning_rate": 1.8888888888888888e-07, + "logits/chosen": -1.867283821105957, + "logits/rejected": -1.8813632726669312, + "logps/chosen": -0.26848095655441284, + "logps/rejected": -0.26555073261260986, + "loss": 97.41613006591797, + "loss/core": 97.41613006591797, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.6509482860565186, + "rewards/margins": 1.494160771369934, + "rewards/rejected": 1.1567875146865845, + "step": 35 + }, + { + "epoch": 0.03821808193001314, + "grad_norm": 326.0, + "learning_rate": 2.1666666666666667e-07, + "logits/chosen": -1.6814746856689453, + "logits/rejected": -1.7323602437973022, + "logps/chosen": -0.2946423888206482, + "logps/rejected": -0.2834400236606598, + "loss": 95.7684097290039, + "loss/core": 95.7684097290039, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.45792293548584, + "rewards/margins": 2.8708791732788086, + "rewards/rejected": 1.5870436429977417, + "step": 40 + }, + { + "epoch": 0.04299534217126478, + "grad_norm": 144.0, + "learning_rate": 2.4444444444444445e-07, + "logits/chosen": -1.8644527196884155, + "logits/rejected": -1.9930543899536133, + "logps/chosen": -0.2883604168891907, + "logps/rejected": -0.28879010677337646, + "loss": 97.59373474121094, + "loss/core": 97.59373474121094, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.0999531745910645, + "rewards/margins": 1.261714220046997, + "rewards/rejected": 0.8382388949394226, + "step": 45 + }, + { + "epoch": 0.04777260241251642, + "grad_norm": 812.0, + "learning_rate": 2.7222222222222216e-07, + "logits/chosen": -1.9386039972305298, + "logits/rejected": -1.9440422058105469, + "logps/chosen": -0.2817879319190979, + "logps/rejected": -0.2857362627983093, + "loss": 97.58670043945312, + "loss/core": 97.58670043945312, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.8786211013793945, + "rewards/margins": 1.4362843036651611, + "rewards/rejected": 1.4423367977142334, + "step": 50 + }, + { + "epoch": 0.05254986265376806, + "grad_norm": 62.75, + "learning_rate": 3e-07, + "logits/chosen": -1.906114935874939, + "logits/rejected": -1.8699897527694702, + "logps/chosen": -0.2895704507827759, + "logps/rejected": -0.2879449427127838, + "loss": 100.38077545166016, + "loss/core": 100.38077545166016, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.2526936531066895, + "rewards/margins": 0.2057073414325714, + "rewards/rejected": 2.0469868183135986, + "step": 55 + }, + { + "epoch": 0.057327122895019705, + "grad_norm": 3264.0, + "learning_rate": 3.2777777777777776e-07, + "logits/chosen": -1.828805923461914, + "logits/rejected": -1.8161197900772095, + "logps/chosen": -0.2744297683238983, + "logps/rejected": -0.28596776723861694, + "loss": 98.89493560791016, + "loss/core": 98.89493560791016, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.156543731689453, + "rewards/margins": 0.6224328279495239, + "rewards/rejected": 1.5341110229492188, + "step": 60 + }, + { + "epoch": 0.06210438313627135, + "grad_norm": 132.0, + "learning_rate": 3.5555555555555553e-07, + "logits/chosen": -1.6605703830718994, + "logits/rejected": -1.72246515750885, + "logps/chosen": -0.3099282383918762, + "logps/rejected": -0.3059017062187195, + "loss": 96.74524688720703, + "loss/core": 96.74524688720703, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.9625723361968994, + "rewards/margins": 1.7866805791854858, + "rewards/rejected": 1.175891637802124, + "step": 65 + }, + { + "epoch": 0.06688164337752299, + "grad_norm": 442.0, + "learning_rate": 3.8333333333333335e-07, + "logits/chosen": -1.65080988407135, + "logits/rejected": -1.679664969444275, + "logps/chosen": -0.3835355043411255, + "logps/rejected": -0.2955405116081238, + "loss": 98.49574279785156, + "loss/core": 98.49574279785156, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.6017367839813232, + "rewards/margins": 1.4039216041564941, + "rewards/rejected": 2.197815418243408, + "step": 70 + }, + { + "epoch": 0.07165890361877464, + "grad_norm": 102.5, + "learning_rate": 4.1111111111111107e-07, + "logits/chosen": -1.711356520652771, + "logits/rejected": -1.7348101139068604, + "logps/chosen": -0.3151581585407257, + "logps/rejected": -0.2897110879421234, + "loss": 98.30010223388672, + "loss/core": 98.30010223388672, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.6052944660186768, + "rewards/margins": 1.0178263187408447, + "rewards/rejected": 1.5874683856964111, + "step": 75 + }, + { + "epoch": 0.07643616386002627, + "grad_norm": 112.0, + "learning_rate": 4.3888888888888884e-07, + "logits/chosen": -1.752758264541626, + "logits/rejected": -1.8684453964233398, + "logps/chosen": -0.26047635078430176, + "logps/rejected": -0.27302050590515137, + "loss": 96.66983795166016, + "loss/core": 96.66983795166016, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.242232084274292, + "rewards/margins": 1.980281114578247, + "rewards/rejected": 1.2619515657424927, + "step": 80 + }, + { + "epoch": 0.08121342410127792, + "grad_norm": 159.0, + "learning_rate": 4.6666666666666666e-07, + "logits/chosen": -1.8509635925292969, + "logits/rejected": -1.8500969409942627, + "logps/chosen": -0.3067242503166199, + "logps/rejected": -0.27861541509628296, + "loss": 96.42810821533203, + "loss/core": 96.42810821533203, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.5060653686523438, + "rewards/margins": 2.184356212615967, + "rewards/rejected": 1.321709394454956, + "step": 85 + }, + { + "epoch": 0.08599068434252956, + "grad_norm": 64.5, + "learning_rate": 4.944444444444445e-07, + "logits/chosen": -2.0346593856811523, + "logits/rejected": -2.0443930625915527, + "logps/chosen": -0.28125816583633423, + "logps/rejected": -0.29576218128204346, + "loss": 98.27728271484375, + "loss/core": 98.27728271484375, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.7548061609268188, + "rewards/margins": 0.9071924090385437, + "rewards/rejected": 0.8476136922836304, + "step": 90 + }, + { + "epoch": 0.09076794458378121, + "grad_norm": 276.0, + "learning_rate": 4.999699149323369e-07, + "logits/chosen": -1.6161243915557861, + "logits/rejected": -1.6432390213012695, + "logps/chosen": -0.26751023530960083, + "logps/rejected": -0.2801710367202759, + "loss": 98.1581039428711, + "loss/core": 98.1581039428711, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.2086246013641357, + "rewards/margins": 1.1166317462921143, + "rewards/rejected": 2.0919928550720215, + "step": 95 + }, + { + "epoch": 0.09554520482503284, + "grad_norm": 156.0, + "learning_rate": 4.998477067547739e-07, + "logits/chosen": -1.895495057106018, + "logits/rejected": -1.9128100872039795, + "logps/chosen": -0.25704631209373474, + "logps/rejected": -0.24669785797595978, + "loss": 97.79266357421875, + "loss/core": 97.79266357421875, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.4641220569610596, + "rewards/margins": 1.1781212091445923, + "rewards/rejected": 1.2860008478164673, + "step": 100 + }, + { + "epoch": 0.10032246506628449, + "grad_norm": 388.0, + "learning_rate": 4.996315410727229e-07, + "logits/chosen": -1.9339812994003296, + "logits/rejected": -1.9512369632720947, + "logps/chosen": -0.28484174609184265, + "logps/rejected": -0.28055232763290405, + "loss": 97.86317443847656, + "loss/core": 97.86317443847656, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.9908250570297241, + "rewards/margins": 1.2003737688064575, + "rewards/rejected": 0.7904513478279114, + "step": 105 + }, + { + "epoch": 0.10509972530753613, + "grad_norm": 149.0, + "learning_rate": 4.993214991772562e-07, + "logits/chosen": -1.73562753200531, + "logits/rejected": -1.6282856464385986, + "logps/chosen": -0.34042173624038696, + "logps/rejected": -0.2966521084308624, + "loss": 95.82826232910156, + "loss/core": 95.82826232910156, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.451060771942139, + "rewards/margins": 2.5516746044158936, + "rewards/rejected": 1.8993858098983765, + "step": 110 + }, + { + "epoch": 0.10987698554878778, + "grad_norm": 696.0, + "learning_rate": 4.989176976624511e-07, + "logits/chosen": -1.6403529644012451, + "logits/rejected": -1.5681908130645752, + "logps/chosen": -0.3004806637763977, + "logps/rejected": -0.2985830008983612, + "loss": 99.58712005615234, + "loss/core": 99.58712005615234, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7745640277862549, + "rewards/margins": 0.32534095644950867, + "rewards/rejected": 1.4492229223251343, + "step": 115 + }, + { + "epoch": 0.11465424579003941, + "grad_norm": 1528.0, + "learning_rate": 4.984202883815428e-07, + "logits/chosen": -1.6866496801376343, + "logits/rejected": -1.690090537071228, + "logps/chosen": -0.3020043969154358, + "logps/rejected": -0.29816195368766785, + "loss": 98.63155364990234, + "loss/core": 98.63155364990234, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.811497449874878, + "rewards/margins": 0.7096825838088989, + "rewards/rejected": 1.101815104484558, + "step": 120 + }, + { + "epoch": 0.11943150603129106, + "grad_norm": 103.0, + "learning_rate": 4.978294583898195e-07, + "logits/chosen": -1.8087574243545532, + "logits/rejected": -1.8217788934707642, + "logps/chosen": -0.2990450859069824, + "logps/rejected": -0.306710809469223, + "loss": 98.02079772949219, + "loss/core": 98.02079772949219, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.0650312900543213, + "rewards/margins": 1.0487768650054932, + "rewards/rejected": 1.0162544250488281, + "step": 125 + }, + { + "epoch": 0.1242087662725427, + "grad_norm": 215.0, + "learning_rate": 4.971454298742779e-07, + "logits/chosen": -1.6154258251190186, + "logits/rejected": -1.73139226436615, + "logps/chosen": -0.27999377250671387, + "logps/rejected": -0.278720498085022, + "loss": 98.19866943359375, + "loss/core": 98.19866943359375, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8072621822357178, + "rewards/margins": 0.9196640849113464, + "rewards/rejected": 0.8875982165336609, + "step": 130 + }, + { + "epoch": 0.12898602651379434, + "grad_norm": 298.0, + "learning_rate": 4.963684600700678e-07, + "logits/chosen": -1.7737205028533936, + "logits/rejected": -1.8461641073226929, + "logps/chosen": -0.3029175102710724, + "logps/rejected": -0.2972210645675659, + "loss": 96.26443481445312, + "loss/core": 96.26443481445312, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.3659565448760986, + "rewards/margins": 2.0419297218322754, + "rewards/rejected": 1.3240267038345337, + "step": 135 + }, + { + "epoch": 0.13376328675504598, + "grad_norm": 133.0, + "learning_rate": 4.954988411637571e-07, + "logits/chosen": -2.091590404510498, + "logits/rejected": -2.117504596710205, + "logps/chosen": -0.2827909588813782, + "logps/rejected": -0.2891230285167694, + "loss": 98.2292251586914, + "loss/core": 98.2292251586914, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.625351905822754, + "rewards/margins": 0.9071060419082642, + "rewards/rejected": 0.7182460427284241, + "step": 140 + }, + { + "epoch": 0.13854054699629761, + "grad_norm": 148.0, + "learning_rate": 4.945369001834514e-07, + "logits/chosen": -1.560279130935669, + "logits/rejected": -1.5604007244110107, + "logps/chosen": -0.30558672547340393, + "logps/rejected": -0.2993968725204468, + "loss": 96.93885040283203, + "loss/core": 96.93885040283203, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.561701536178589, + "rewards/margins": 1.752966284751892, + "rewards/rejected": 1.8087351322174072, + "step": 145 + }, + { + "epoch": 0.14331780723754928, + "grad_norm": 256.0, + "learning_rate": 4.93482998875813e-07, + "logits/chosen": -1.9131495952606201, + "logits/rejected": -1.9743051528930664, + "logps/chosen": -0.3162950873374939, + "logps/rejected": -0.30938461422920227, + "loss": 96.15104675292969, + "loss/core": 96.15104675292969, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.649721622467041, + "rewards/margins": 2.2039794921875, + "rewards/rejected": 1.4457422494888306, + "step": 150 + }, + { + "epoch": 0.1480950674788009, + "grad_norm": 157.0, + "learning_rate": 4.923375335700223e-07, + "logits/chosen": -1.809692144393921, + "logits/rejected": -1.869560956954956, + "logps/chosen": -0.2845507562160492, + "logps/rejected": -0.28922533988952637, + "loss": 96.42918395996094, + "loss/core": 96.42918395996094, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.84999418258667, + "rewards/margins": 1.9925158023834229, + "rewards/rejected": 0.8574784994125366, + "step": 155 + }, + { + "epoch": 0.15287232772005255, + "grad_norm": 556.0, + "learning_rate": 4.911009350287347e-07, + "logits/chosen": -1.7576831579208374, + "logits/rejected": -1.7200205326080322, + "logps/chosen": -0.31108418107032776, + "logps/rejected": -0.31466466188430786, + "loss": 97.29283905029297, + "loss/core": 97.29283905029297, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.1445982456207275, + "rewards/margins": 1.7368183135986328, + "rewards/rejected": 1.4077800512313843, + "step": 160 + }, + { + "epoch": 0.15764958796130418, + "grad_norm": 173.0, + "learning_rate": 4.897736682860885e-07, + "logits/chosen": -1.7794075012207031, + "logits/rejected": -1.7277469635009766, + "logps/chosen": -0.31061312556266785, + "logps/rejected": -0.3058010935783386, + "loss": 98.41602325439453, + "loss/core": 98.41602325439453, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6071736812591553, + "rewards/margins": 0.8191731572151184, + "rewards/rejected": 0.7880005240440369, + "step": 165 + }, + { + "epoch": 0.16242684820255585, + "grad_norm": 448.0, + "learning_rate": 4.883562324728241e-07, + "logits/chosen": -1.8817628622055054, + "logits/rejected": -1.8192886114120483, + "logps/chosen": -0.2736721634864807, + "logps/rejected": -0.2607767879962921, + "loss": 101.3716049194336, + "loss/core": 101.3716049194336, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.6075308322906494, + "rewards/margins": -0.15226136147975922, + "rewards/rejected": 2.7597920894622803, + "step": 170 + }, + { + "epoch": 0.16720410844380748, + "grad_norm": 940.0, + "learning_rate": 4.868491606285823e-07, + "logits/chosen": -1.764796495437622, + "logits/rejected": -1.7796127796173096, + "logps/chosen": -0.28747016191482544, + "logps/rejected": -0.29904478788375854, + "loss": 97.57106018066406, + "loss/core": 97.57106018066406, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.577658176422119, + "rewards/margins": 1.2913696765899658, + "rewards/rejected": 1.2862884998321533, + "step": 175 + }, + { + "epoch": 0.17198136868505912, + "grad_norm": 484.0, + "learning_rate": 4.852530195014489e-07, + "logits/chosen": -1.860839605331421, + "logits/rejected": -1.8101718425750732, + "logps/chosen": -0.2720187306404114, + "logps/rejected": -0.26114434003829956, + "loss": 97.21582794189453, + "loss/core": 97.21582794189453, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.2341485023498535, + "rewards/margins": 1.7361303567886353, + "rewards/rejected": 1.4980182647705078, + "step": 180 + }, + { + "epoch": 0.17675862892631075, + "grad_norm": 128.0, + "learning_rate": 4.835684093348244e-07, + "logits/chosen": -1.9565016031265259, + "logits/rejected": -1.9064795970916748, + "logps/chosen": -0.28875815868377686, + "logps/rejected": -0.2922629415988922, + "loss": 99.18455505371094, + "loss/core": 99.18455505371094, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.042085886001587, + "rewards/margins": 0.4169158339500427, + "rewards/rejected": 0.625170111656189, + "step": 185 + }, + { + "epoch": 0.18153588916756241, + "grad_norm": 260.0, + "learning_rate": 4.817959636416969e-07, + "logits/chosen": -1.9324171543121338, + "logits/rejected": -1.926792860031128, + "logps/chosen": -0.28232958912849426, + "logps/rejected": -0.285550057888031, + "loss": 97.28160095214844, + "loss/core": 97.28160095214844, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.8508822917938232, + "rewards/margins": 1.6247138977050781, + "rewards/rejected": 1.2261682748794556, + "step": 190 + }, + { + "epoch": 0.18631314940881405, + "grad_norm": 173.0, + "learning_rate": 4.799363489664039e-07, + "logits/chosen": -1.651599645614624, + "logits/rejected": -1.645272970199585, + "logps/chosen": -0.2920708656311035, + "logps/rejected": -0.2902786135673523, + "loss": 97.38813781738281, + "loss/core": 97.38813781738281, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.6577506065368652, + "rewards/margins": 1.697007417678833, + "rewards/rejected": 0.9607435464859009, + "step": 195 + }, + { + "epoch": 0.19109040965006568, + "grad_norm": 158.0, + "learning_rate": 4.779902646339721e-07, + "logits/chosen": -1.7238547801971436, + "logits/rejected": -1.7459415197372437, + "logps/chosen": -0.3239200711250305, + "logps/rejected": -0.31623491644859314, + "loss": 97.3945541381836, + "loss/core": 97.3945541381836, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.610980987548828, + "rewards/margins": 1.3865725994110107, + "rewards/rejected": 1.2244083881378174, + "step": 200 + }, + { + "epoch": 0.19586766989131732, + "grad_norm": 107.5, + "learning_rate": 4.759584424871301e-07, + "logits/chosen": -1.8025785684585571, + "logits/rejected": -1.8798892498016357, + "logps/chosen": -0.25078368186950684, + "logps/rejected": -0.270027220249176, + "loss": 98.32032775878906, + "loss/core": 98.32032775878906, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.282686948776245, + "rewards/margins": 1.18528151512146, + "rewards/rejected": 2.097405195236206, + "step": 205 + }, + { + "epoch": 0.20064493013256898, + "grad_norm": 274.0, + "learning_rate": 4.738416466110917e-07, + "logits/chosen": -1.8228957653045654, + "logits/rejected": -1.8074302673339844, + "logps/chosen": -0.31188255548477173, + "logps/rejected": -0.32124748826026917, + "loss": 97.54026794433594, + "loss/core": 97.54026794433594, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.8372724056243896, + "rewards/margins": 1.347383737564087, + "rewards/rejected": 1.4898887872695923, + "step": 210 + }, + { + "epoch": 0.20542219037382062, + "grad_norm": 250.0, + "learning_rate": 4.716406730462153e-07, + "logits/chosen": -1.8536853790283203, + "logits/rejected": -1.8146007061004639, + "logps/chosen": -0.27515512704849243, + "logps/rejected": -0.2840956151485443, + "loss": 98.03938293457031, + "loss/core": 98.03938293457031, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.1946370601654053, + "rewards/margins": 1.4905591011047363, + "rewards/rejected": 1.7040780782699585, + "step": 215 + }, + { + "epoch": 0.21019945061507225, + "grad_norm": 276.0, + "learning_rate": 4.693563494886454e-07, + "logits/chosen": -1.770841360092163, + "logits/rejected": -1.7254314422607422, + "logps/chosen": -0.3026730716228485, + "logps/rejected": -0.30042824149131775, + "loss": 98.44063568115234, + "loss/core": 98.44063568115234, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.8839222192764282, + "rewards/margins": 0.8008639216423035, + "rewards/rejected": 1.0830583572387695, + "step": 220 + }, + { + "epoch": 0.2149767108563239, + "grad_norm": 146.0, + "learning_rate": 4.6698953497905016e-07, + "logits/chosen": -1.822609305381775, + "logits/rejected": -1.8853318691253662, + "logps/chosen": -0.29000169038772583, + "logps/rejected": -0.2747119069099426, + "loss": 96.0133285522461, + "loss/core": 96.0133285522461, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.628028154373169, + "rewards/margins": 2.211141586303711, + "rewards/rejected": 1.416886568069458, + "step": 225 + }, + { + "epoch": 0.21975397109757555, + "grad_norm": 1568.0, + "learning_rate": 4.645411195795709e-07, + "logits/chosen": -1.956534743309021, + "logits/rejected": -1.9918060302734375, + "logps/chosen": -0.2819087505340576, + "logps/rejected": -0.3088639974594116, + "loss": 97.88602447509766, + "loss/core": 97.88602447509766, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8525536060333252, + "rewards/margins": 1.1093358993530273, + "rewards/rejected": 0.7432177066802979, + "step": 230 + }, + { + "epoch": 0.2245312313388272, + "grad_norm": 177.0, + "learning_rate": 4.6201202403910643e-07, + "logits/chosen": -1.6948636770248413, + "logits/rejected": -1.711146354675293, + "logps/chosen": -0.29289984703063965, + "logps/rejected": -0.28607800602912903, + "loss": 98.97444152832031, + "loss/core": 98.97444152832031, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.9021542072296143, + "rewards/margins": 0.5686413049697876, + "rewards/rejected": 1.3335130214691162, + "step": 235 + }, + { + "epoch": 0.22930849158007882, + "grad_norm": 169.0, + "learning_rate": 4.594031994470573e-07, + "logits/chosen": -1.7635622024536133, + "logits/rejected": -1.7835123538970947, + "logps/chosen": -0.29360511898994446, + "logps/rejected": -0.29403769969940186, + "loss": 98.54844665527344, + "loss/core": 98.54844665527344, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7672475576400757, + "rewards/margins": 0.739784836769104, + "rewards/rejected": 1.0274627208709717, + "step": 240 + }, + { + "epoch": 0.23408575182133046, + "grad_norm": 1576.0, + "learning_rate": 4.567156268756593e-07, + "logits/chosen": -1.9231055974960327, + "logits/rejected": -1.9007583856582642, + "logps/chosen": -0.27654072642326355, + "logps/rejected": -0.28899580240249634, + "loss": 96.15101623535156, + "loss/core": 96.15101623535156, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.6142005920410156, + "rewards/margins": 2.4411978721618652, + "rewards/rejected": 1.1730026006698608, + "step": 245 + }, + { + "epoch": 0.23886301206258212, + "grad_norm": 212.0, + "learning_rate": 4.5395031701104303e-07, + "logits/chosen": -1.6372286081314087, + "logits/rejected": -1.6989704370498657, + "logps/chosen": -0.3004458248615265, + "logps/rejected": -0.29450005292892456, + "loss": 97.6203842163086, + "loss/core": 97.6203842163086, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.332899570465088, + "rewards/margins": 1.2636841535568237, + "rewards/rejected": 1.069215178489685, + "step": 250 + }, + { + "epoch": 0.24364027230383375, + "grad_norm": 133.0, + "learning_rate": 4.511083097731555e-07, + "logits/chosen": -1.8383357524871826, + "logits/rejected": -1.847869634628296, + "logps/chosen": -0.28603678941726685, + "logps/rejected": -0.2877523601055145, + "loss": 98.42637634277344, + "loss/core": 98.42637634277344, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.4403929710388184, + "rewards/margins": 0.8402155041694641, + "rewards/rejected": 1.600177526473999, + "step": 255 + }, + { + "epoch": 0.2484175325450854, + "grad_norm": 68.5, + "learning_rate": 4.481906739246894e-07, + "logits/chosen": -1.909732460975647, + "logits/rejected": -1.9944489002227783, + "logps/chosen": -0.28810587525367737, + "logps/rejected": -0.2701200842857361, + "loss": 98.69931030273438, + "loss/core": 98.69931030273438, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.3960041999816895, + "rewards/margins": 0.8098465800285339, + "rewards/rejected": 1.5861575603485107, + "step": 260 + }, + { + "epoch": 0.25319479278633705, + "grad_norm": 68.0, + "learning_rate": 4.451985066691648e-07, + "logits/chosen": -1.796696662902832, + "logits/rejected": -1.8159329891204834, + "logps/chosen": -0.2996280789375305, + "logps/rejected": -0.30264267325401306, + "loss": 100.24183654785156, + "loss/core": 100.24183654785156, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.2289425134658813, + "rewards/margins": 0.06570056825876236, + "rewards/rejected": 1.163241982460022, + "step": 265 + }, + { + "epoch": 0.2579720530275887, + "grad_norm": 122.5, + "learning_rate": 4.421329332383158e-07, + "logits/chosen": -1.6407196521759033, + "logits/rejected": -1.7502638101577759, + "logps/chosen": -0.29027777910232544, + "logps/rejected": -0.2889750599861145, + "loss": 98.38270568847656, + "loss/core": 98.38270568847656, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.2634711265563965, + "rewards/margins": 1.1196836233139038, + "rewards/rejected": 2.143787145614624, + "step": 270 + }, + { + "epoch": 0.2627493132688403, + "grad_norm": 1012.0, + "learning_rate": 4.3899510646893696e-07, + "logits/chosen": -1.8677374124526978, + "logits/rejected": -1.8112701177597046, + "logps/chosen": -0.2966800630092621, + "logps/rejected": -0.2986254394054413, + "loss": 97.82476043701172, + "loss/core": 97.82476043701172, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.3230066299438477, + "rewards/margins": 1.1454260349273682, + "rewards/rejected": 1.177580714225769, + "step": 275 + }, + { + "epoch": 0.26752657351009196, + "grad_norm": 115.0, + "learning_rate": 4.357862063693485e-07, + "logits/chosen": -1.7707321643829346, + "logits/rejected": -1.7221782207489014, + "logps/chosen": -0.28835004568099976, + "logps/rejected": -0.3050932288169861, + "loss": 97.6014633178711, + "loss/core": 97.6014633178711, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.0437865257263184, + "rewards/margins": 1.259574294090271, + "rewards/rejected": 0.7842122316360474, + "step": 280 + }, + { + "epoch": 0.2723038337513436, + "grad_norm": 536.0, + "learning_rate": 4.3250743967564364e-07, + "logits/chosen": -1.8891651630401611, + "logits/rejected": -1.8469898700714111, + "logps/chosen": -0.29696494340896606, + "logps/rejected": -0.29243922233581543, + "loss": 98.27204132080078, + "loss/core": 98.27204132080078, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.607239246368408, + "rewards/margins": 0.9751089215278625, + "rewards/rejected": 1.6321302652359009, + "step": 285 + }, + { + "epoch": 0.27708109399259523, + "grad_norm": 2144.0, + "learning_rate": 4.29160039397884e-07, + "logits/chosen": -1.8728892803192139, + "logits/rejected": -1.9022899866104126, + "logps/chosen": -0.2866348624229431, + "logps/rejected": -0.28832441568374634, + "loss": 100.2232437133789, + "loss/core": 100.2232437133789, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.4807735681533813, + "rewards/margins": 0.033438198268413544, + "rewards/rejected": 1.4473353624343872, + "step": 290 + }, + { + "epoch": 0.28185835423384686, + "grad_norm": 134.0, + "learning_rate": 4.2574526435641546e-07, + "logits/chosen": -2.1759440898895264, + "logits/rejected": -2.1741747856140137, + "logps/chosen": -0.24452364444732666, + "logps/rejected": -0.25654464960098267, + "loss": 98.74320983886719, + "loss/core": 98.74320983886719, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.3100438117980957, + "rewards/margins": 0.6354137659072876, + "rewards/rejected": 0.6746301651000977, + "step": 295 + }, + { + "epoch": 0.28663561447509855, + "grad_norm": 1488.0, + "learning_rate": 4.22264398708477e-07, + "logits/chosen": -1.5932027101516724, + "logits/rejected": -1.6199067831039429, + "logps/chosen": -0.3133957087993622, + "logps/rejected": -0.31795522570610046, + "loss": 95.31610107421875, + "loss/core": 95.31610107421875, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.7138900756835938, + "rewards/margins": 2.6641526222229004, + "rewards/rejected": 1.0497372150421143, + "step": 300 + }, + { + "epoch": 0.2914128747163502, + "grad_norm": 160.0, + "learning_rate": 4.187187514652819e-07, + "logits/chosen": -1.7853899002075195, + "logits/rejected": -1.8296200037002563, + "logps/chosen": -0.2687693238258362, + "logps/rejected": -0.2597236633300781, + "loss": 97.58959197998047, + "loss/core": 97.58959197998047, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.1659064292907715, + "rewards/margins": 1.2582628726959229, + "rewards/rejected": 0.9076434969902039, + "step": 305 + }, + { + "epoch": 0.2961901349576018, + "grad_norm": 1936.0, + "learning_rate": 4.151096559997519e-07, + "logits/chosen": -1.829178810119629, + "logits/rejected": -1.731296181678772, + "logps/chosen": -0.27323228120803833, + "logps/rejected": -0.2782127559185028, + "loss": 99.45023345947266, + "loss/core": 99.45023345947266, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.610325574874878, + "rewards/margins": 0.5705730319023132, + "rewards/rejected": 2.03975248336792, + "step": 310 + }, + { + "epoch": 0.30096739519885346, + "grad_norm": 125.0, + "learning_rate": 4.114384695450905e-07, + "logits/chosen": -1.7197242975234985, + "logits/rejected": -1.8159154653549194, + "logps/chosen": -0.26879817247390747, + "logps/rejected": -0.2760879397392273, + "loss": 96.94550323486328, + "loss/core": 96.94550323486328, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.9166643619537354, + "rewards/margins": 1.574958086013794, + "rewards/rejected": 1.3417065143585205, + "step": 315 + }, + { + "epoch": 0.3057446554401051, + "grad_norm": 104.5, + "learning_rate": 4.077065726843828e-07, + "logits/chosen": -1.7181727886199951, + "logits/rejected": -1.6899751424789429, + "logps/chosen": -0.2890922427177429, + "logps/rejected": -0.2932893633842468, + "loss": 99.34120178222656, + "loss/core": 99.34120178222656, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.5913293361663818, + "rewards/margins": 0.36940068006515503, + "rewards/rejected": 1.221928596496582, + "step": 320 + }, + { + "epoch": 0.31052191568135673, + "grad_norm": 484.0, + "learning_rate": 4.039153688314145e-07, + "logits/chosen": -1.967686653137207, + "logits/rejected": -1.9203250408172607, + "logps/chosen": -0.2864150404930115, + "logps/rejected": -0.28275853395462036, + "loss": 97.01065826416016, + "loss/core": 97.01065826416016, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.6977322101593018, + "rewards/margins": 2.1541683673858643, + "rewards/rejected": 1.5435636043548584, + "step": 325 + }, + { + "epoch": 0.31529917592260837, + "grad_norm": 82.5, + "learning_rate": 4.0006628370290613e-07, + "logits/chosen": -1.899263620376587, + "logits/rejected": -1.861311674118042, + "logps/chosen": -0.27477145195007324, + "logps/rejected": -0.27060219645500183, + "loss": 98.25178527832031, + "loss/core": 98.25178527832031, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.069042682647705, + "rewards/margins": 0.9386270642280579, + "rewards/rejected": 1.130415678024292, + "step": 330 + }, + { + "epoch": 0.32007643616386, + "grad_norm": 1248.0, + "learning_rate": 3.9616076478235826e-07, + "logits/chosen": -1.807976484298706, + "logits/rejected": -1.8044418096542358, + "logps/chosen": -0.30185666680336, + "logps/rejected": -0.3223247230052948, + "loss": 97.50824737548828, + "loss/core": 97.50824737548828, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.1243371963500977, + "rewards/margins": 1.4735743999481201, + "rewards/rejected": 0.6507625579833984, + "step": 335 + }, + { + "epoch": 0.3248536964051117, + "grad_norm": 644.0, + "learning_rate": 3.922002807757129e-07, + "logits/chosen": -1.8412240743637085, + "logits/rejected": -1.9032318592071533, + "logps/chosen": -0.27988195419311523, + "logps/rejected": -0.28126993775367737, + "loss": 97.8116455078125, + "loss/core": 97.8116455078125, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.042039394378662, + "rewards/margins": 1.1677625179290771, + "rewards/rejected": 0.8742771148681641, + "step": 340 + }, + { + "epoch": 0.3296309566463633, + "grad_norm": 202.0, + "learning_rate": 3.8818632105903315e-07, + "logits/chosen": -1.8139426708221436, + "logits/rejected": -1.8884384632110596, + "logps/chosen": -0.28163427114486694, + "logps/rejected": -0.2937542796134949, + "loss": 98.46784210205078, + "loss/core": 98.46784210205078, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.6491235494613647, + "rewards/margins": 0.817004382610321, + "rewards/rejected": 0.8321191668510437, + "step": 345 + }, + { + "epoch": 0.33440821688761496, + "grad_norm": 165.0, + "learning_rate": 3.841203951184094e-07, + "logits/chosen": -1.8352000713348389, + "logits/rejected": -1.8296496868133545, + "logps/chosen": -0.26140180230140686, + "logps/rejected": -0.2555782198905945, + "loss": 98.01313018798828, + "loss/core": 98.01313018798828, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9879388809204102, + "rewards/margins": 1.031970500946045, + "rewards/rejected": 0.9559683799743652, + "step": 350 + }, + { + "epoch": 0.3391854771288666, + "grad_norm": 165.0, + "learning_rate": 3.800040319823038e-07, + "logits/chosen": -1.7930519580841064, + "logits/rejected": -1.874668836593628, + "logps/chosen": -0.2808493673801422, + "logps/rejected": -0.29185348749160767, + "loss": 99.32098388671875, + "loss/core": 99.32098388671875, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8493973016738892, + "rewards/margins": 0.49081817269325256, + "rewards/rejected": 1.358579397201538, + "step": 355 + }, + { + "epoch": 0.34396273737011823, + "grad_norm": 1024.0, + "learning_rate": 3.75838779646545e-07, + "logits/chosen": -1.8572410345077515, + "logits/rejected": -1.9341695308685303, + "logps/chosen": -0.27888017892837524, + "logps/rejected": -0.280597448348999, + "loss": 97.60267639160156, + "loss/core": 97.60267639160156, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.646106481552124, + "rewards/margins": 1.358185052871704, + "rewards/rejected": 1.28792142868042, + "step": 360 + }, + { + "epoch": 0.34873999761136987, + "grad_norm": 362.0, + "learning_rate": 3.7162620449218993e-07, + "logits/chosen": -1.8237988948822021, + "logits/rejected": -1.8687849044799805, + "logps/chosen": -0.28639885783195496, + "logps/rejected": -0.2829182744026184, + "loss": 96.79151916503906, + "loss/core": 96.79151916503906, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.860142946243286, + "rewards/margins": 2.0410332679748535, + "rewards/rejected": 0.8191097378730774, + "step": 365 + }, + { + "epoch": 0.3535172578526215, + "grad_norm": 324.0, + "learning_rate": 3.673678906964727e-07, + "logits/chosen": -1.913863182067871, + "logits/rejected": -2.0241215229034424, + "logps/chosen": -0.28453686833381653, + "logps/rejected": -0.2855282425880432, + "loss": 98.85942840576172, + "loss/core": 98.85942840576172, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.647334098815918, + "rewards/margins": 0.5878774523735046, + "rewards/rejected": 1.059456706047058, + "step": 370 + }, + { + "epoch": 0.35829451809387314, + "grad_norm": 153.0, + "learning_rate": 3.6306543963705936e-07, + "logits/chosen": -1.7168223857879639, + "logits/rejected": -1.7345936298370361, + "logps/chosen": -0.2586119472980499, + "logps/rejected": -0.2558966279029846, + "loss": 98.92908477783203, + "loss/core": 98.92908477783203, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.0151703357696533, + "rewards/margins": 0.7727279663085938, + "rewards/rejected": 2.2424423694610596, + "step": 375 + }, + { + "epoch": 0.36307177833512483, + "grad_norm": 213.0, + "learning_rate": 3.5872046928983623e-07, + "logits/chosen": -1.8439680337905884, + "logits/rejected": -1.8838622570037842, + "logps/chosen": -0.27940452098846436, + "logps/rejected": -0.28964605927467346, + "loss": 99.63495635986328, + "loss/core": 99.63495635986328, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.0571694374084473, + "rewards/margins": 0.45843571424484253, + "rewards/rejected": 1.5987335443496704, + "step": 380 + }, + { + "epoch": 0.36784903857637646, + "grad_norm": 276.0, + "learning_rate": 3.5433461362045447e-07, + "logits/chosen": -1.8193256855010986, + "logits/rejected": -1.8221651315689087, + "logps/chosen": -0.2699766159057617, + "logps/rejected": -0.2746611535549164, + "loss": 98.85408782958984, + "loss/core": 98.85408782958984, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.1804919242858887, + "rewards/margins": 0.6561237573623657, + "rewards/rejected": 1.524368405342102, + "step": 385 + }, + { + "epoch": 0.3726262988176281, + "grad_norm": 87.0, + "learning_rate": 3.4990952196986305e-07, + "logits/chosen": -1.6936248540878296, + "logits/rejected": -1.7653439044952393, + "logps/chosen": -0.30715465545654297, + "logps/rejected": -0.30202072858810425, + "loss": 98.37455749511719, + "loss/core": 98.37455749511719, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.762188196182251, + "rewards/margins": 0.8616089820861816, + "rewards/rejected": 0.9005790948867798, + "step": 390 + }, + { + "epoch": 0.37740355905887973, + "grad_norm": 241.0, + "learning_rate": 3.4544685843405875e-07, + "logits/chosen": -1.9225870370864868, + "logits/rejected": -1.9785583019256592, + "logps/chosen": -0.2938076853752136, + "logps/rejected": -0.28969907760620117, + "loss": 98.27570343017578, + "loss/core": 98.27570343017578, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 1.694026231765747, + "rewards/margins": 0.8879431486129761, + "rewards/rejected": 0.8060829043388367, + "step": 395 + }, + { + "epoch": 0.38218081930013137, + "grad_norm": 175.0, + "learning_rate": 3.4094830123828786e-07, + "logits/chosen": -1.6314131021499634, + "logits/rejected": -1.6901371479034424, + "logps/chosen": -0.29122158885002136, + "logps/rejected": -0.2936597466468811, + "loss": 96.5455093383789, + "loss/core": 96.5455093383789, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.9411308765411377, + "rewards/margins": 2.1050190925598145, + "rewards/rejected": 0.8361119031906128, + "step": 400 + }, + { + "epoch": 0.386958079541383, + "grad_norm": 133.0, + "learning_rate": 3.3641554210593414e-07, + "logits/chosen": -1.834407091140747, + "logits/rejected": -1.8702154159545898, + "logps/chosen": -0.3132235109806061, + "logps/rejected": -0.30386728048324585, + "loss": 98.06851196289062, + "loss/core": 98.06851196289062, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.3328118324279785, + "rewards/margins": 1.0983474254608154, + "rewards/rejected": 1.234464406967163, + "step": 405 + }, + { + "epoch": 0.39173533978263464, + "grad_norm": 174.0, + "learning_rate": 3.3185028562233107e-07, + "logits/chosen": -2.085736036300659, + "logits/rejected": -2.084230899810791, + "logps/chosen": -0.27615100145339966, + "logps/rejected": -0.2881914973258972, + "loss": 98.22357940673828, + "loss/core": 98.22357940673828, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.096590518951416, + "rewards/margins": 0.9094375371932983, + "rewards/rejected": 1.1871533393859863, + "step": 410 + }, + { + "epoch": 0.3965126000238863, + "grad_norm": 167.0, + "learning_rate": 3.272542485937368e-07, + "logits/chosen": -1.6657861471176147, + "logits/rejected": -1.7485307455062866, + "logps/chosen": -0.33018797636032104, + "logps/rejected": -0.3208489716053009, + "loss": 98.79418182373047, + "loss/core": 98.79418182373047, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 0.9962074160575867, + "rewards/margins": 0.6192155480384827, + "rewards/rejected": 0.3769919276237488, + "step": 415 + }, + { + "epoch": 0.40128986026513797, + "grad_norm": 358.0, + "learning_rate": 3.226291594017137e-07, + "logits/chosen": -2.0174975395202637, + "logits/rejected": -2.013953447341919, + "logps/chosen": -0.2835124135017395, + "logps/rejected": -0.2982017695903778, + "loss": 97.60393524169922, + "loss/core": 97.60393524169922, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.0437941551208496, + "rewards/margins": 1.3167150020599365, + "rewards/rejected": 0.7270792722702026, + "step": 420 + }, + { + "epoch": 0.4060671205063896, + "grad_norm": 728.0, + "learning_rate": 3.1797675735315454e-07, + "logits/chosen": -1.7338950634002686, + "logits/rejected": -1.8055951595306396, + "logps/chosen": -0.2909316420555115, + "logps/rejected": -0.2653912901878357, + "loss": 99.46458435058594, + "loss/core": 99.46458435058594, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.2419204711914062, + "rewards/margins": 0.5987423658370972, + "rewards/rejected": 1.6431777477264404, + "step": 425 + }, + { + "epoch": 0.41084438074764124, + "grad_norm": 280.0, + "learning_rate": 3.1329879202620047e-07, + "logits/chosen": -1.812638521194458, + "logits/rejected": -1.8498079776763916, + "logps/chosen": -0.2791460454463959, + "logps/rejected": -0.2673080265522003, + "loss": 95.21307373046875, + "loss/core": 95.21307373046875, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.406863212585449, + "rewards/margins": 2.995854139328003, + "rewards/rejected": 1.4110084772109985, + "step": 430 + }, + { + "epoch": 0.41562164098889287, + "grad_norm": 194.0, + "learning_rate": 3.0859702261229613e-07, + "logits/chosen": -2.0415005683898926, + "logits/rejected": -1.9983999729156494, + "logps/chosen": -0.2776305079460144, + "logps/rejected": -0.29190298914909363, + "loss": 98.83068084716797, + "loss/core": 98.83068084716797, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.6318155527114868, + "rewards/margins": 0.6132717728614807, + "rewards/rejected": 1.0185438394546509, + "step": 435 + }, + { + "epoch": 0.4203989012301445, + "grad_norm": 1056.0, + "learning_rate": 3.0387321725463e-07, + "logits/chosen": -1.7515466213226318, + "logits/rejected": -1.754542589187622, + "logps/chosen": -0.29860061407089233, + "logps/rejected": -0.2998600900173187, + "loss": 97.38858032226562, + "loss/core": 97.38858032226562, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.053305149078369, + "rewards/margins": 1.6653683185577393, + "rewards/rejected": 1.3879367113113403, + "step": 440 + }, + { + "epoch": 0.42517616147139614, + "grad_norm": 286.0, + "learning_rate": 2.991291523832075e-07, + "logits/chosen": -1.676851511001587, + "logits/rejected": -1.6223485469818115, + "logps/chosen": -0.29457855224609375, + "logps/rejected": -0.29545649886131287, + "loss": 97.41128540039062, + "loss/core": 97.41128540039062, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.970858097076416, + "rewards/margins": 1.3911173343658447, + "rewards/rejected": 1.57974112033844, + "step": 445 + }, + { + "epoch": 0.4299534217126478, + "grad_norm": 1464.0, + "learning_rate": 2.943666120468088e-07, + "logits/chosen": -1.6469995975494385, + "logits/rejected": -1.6655452251434326, + "logps/chosen": -0.29804956912994385, + "logps/rejected": -0.2897404432296753, + "loss": 96.7783432006836, + "loss/core": 96.7783432006836, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.4604811668395996, + "rewards/margins": 1.8570493459701538, + "rewards/rejected": 1.6034319400787354, + "step": 450 + }, + { + "epoch": 0.4347306819538994, + "grad_norm": 474.0, + "learning_rate": 2.8958738724208073e-07, + "logits/chosen": -1.6406848430633545, + "logits/rejected": -1.831834077835083, + "logps/chosen": -0.3217054009437561, + "logps/rejected": -0.31396520137786865, + "loss": 96.65595245361328, + "loss/core": 96.65595245361328, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.475750684738159, + "rewards/margins": 1.9972509145736694, + "rewards/rejected": 1.4784997701644897, + "step": 455 + }, + { + "epoch": 0.4395079421951511, + "grad_norm": 306.0, + "learning_rate": 2.8479327524001633e-07, + "logits/chosen": -1.6828476190567017, + "logits/rejected": -1.7117191553115845, + "logps/chosen": -0.2919991612434387, + "logps/rejected": -0.3055281937122345, + "loss": 96.70384216308594, + "loss/core": 96.70384216308594, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.7047760486602783, + "rewards/margins": 1.8250391483306885, + "rewards/rejected": 1.879737138748169, + "step": 460 + }, + { + "epoch": 0.44428520243640274, + "grad_norm": 127.0, + "learning_rate": 2.7998607891007493e-07, + "logits/chosen": -1.5566418170928955, + "logits/rejected": -1.6399599313735962, + "logps/chosen": -0.2965025007724762, + "logps/rejected": -0.2912459373474121, + "loss": 97.3532943725586, + "loss/core": 97.3532943725586, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 2.3075928688049316, + "rewards/margins": 1.3737890720367432, + "rewards/rejected": 0.9338043332099915, + "step": 465 + }, + { + "epoch": 0.4490624626776544, + "grad_norm": 1104.0, + "learning_rate": 2.7516760604219616e-07, + "logits/chosen": -1.879783272743225, + "logits/rejected": -1.9588508605957031, + "logps/chosen": -0.2798837423324585, + "logps/rejected": -0.2600264847278595, + "loss": 96.83983612060547, + "loss/core": 96.83983612060547, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.0017495155334473, + "rewards/margins": 1.8324419260025024, + "rewards/rejected": 1.1693079471588135, + "step": 470 + }, + { + "epoch": 0.453839722918906, + "grad_norm": 592.0, + "learning_rate": 2.703396686669646e-07, + "logits/chosen": -1.8083826303482056, + "logits/rejected": -1.7853968143463135, + "logps/chosen": -0.2943006753921509, + "logps/rejected": -0.3146829903125763, + "loss": 99.76537322998047, + "loss/core": 99.76537322998047, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.8425248861312866, + "rewards/margins": 0.3144264817237854, + "rewards/rejected": 1.5280983448028564, + "step": 475 + }, + { + "epoch": 0.45861698316015764, + "grad_norm": 150.0, + "learning_rate": 2.6550408237417884e-07, + "logits/chosen": -1.9771804809570312, + "logits/rejected": -2.06240177154541, + "logps/chosen": -0.2760615050792694, + "logps/rejected": -0.271167129278183, + "loss": 97.69608306884766, + "loss/core": 97.69608306884766, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.3518784046173096, + "rewards/margins": 1.3141744136810303, + "rewards/rejected": 1.0377036333084106, + "step": 480 + }, + { + "epoch": 0.4633942434014093, + "grad_norm": 133.0, + "learning_rate": 2.6066266563008265e-07, + "logits/chosen": -2.0386388301849365, + "logits/rejected": -2.0184037685394287, + "logps/chosen": -0.2914060056209564, + "logps/rejected": -0.2953184247016907, + "loss": 98.33658599853516, + "loss/core": 98.33658599853516, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7446434497833252, + "rewards/margins": 0.8531507253646851, + "rewards/rejected": 0.8914925456047058, + "step": 485 + }, + { + "epoch": 0.4681715036426609, + "grad_norm": 243.0, + "learning_rate": 2.5581723909351404e-07, + "logits/chosen": -1.876051902770996, + "logits/rejected": -1.899723768234253, + "logps/chosen": -0.30638667941093445, + "logps/rejected": -0.2942095100879669, + "loss": 98.0177993774414, + "loss/core": 98.0177993774414, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.0851686000823975, + "rewards/margins": 1.030152678489685, + "rewards/rejected": 1.0550158023834229, + "step": 490 + }, + { + "epoch": 0.47294876388391255, + "grad_norm": 274.0, + "learning_rate": 2.509696249312301e-07, + "logits/chosen": -1.882607102394104, + "logits/rejected": -1.9915786981582642, + "logps/chosen": -0.2932814061641693, + "logps/rejected": -0.2904340624809265, + "loss": 96.98285675048828, + "loss/core": 96.98285675048828, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.6567623615264893, + "rewards/margins": 1.7608906030654907, + "rewards/rejected": 0.8958717584609985, + "step": 495 + }, + { + "epoch": 0.47772602412516424, + "grad_norm": 414.0, + "learning_rate": 2.461216461326642e-07, + "logits/chosen": -1.7806377410888672, + "logits/rejected": -1.7007198333740234, + "logps/chosen": -0.2761852443218231, + "logps/rejected": -0.30116838216781616, + "loss": 97.001220703125, + "loss/core": 97.001220703125, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.212673664093018, + "rewards/margins": 2.1719820499420166, + "rewards/rejected": 2.04069185256958, + "step": 500 + }, + { + "epoch": 0.4825032843664159, + "grad_norm": 592.0, + "learning_rate": 2.412751258243748e-07, + "logits/chosen": -1.5999929904937744, + "logits/rejected": -1.6960639953613281, + "logps/chosen": -0.2828037738800049, + "logps/rejected": -0.2773839831352234, + "loss": 97.8866195678711, + "loss/core": 97.8866195678711, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.6385066509246826, + "rewards/margins": 1.7813966274261475, + "rewards/rejected": 1.857109785079956, + "step": 505 + }, + { + "epoch": 0.4872805446076675, + "grad_norm": 48.0, + "learning_rate": 2.3643188658444158e-07, + "logits/chosen": -1.815176010131836, + "logits/rejected": -1.9088054895401, + "logps/chosen": -0.31443700194358826, + "logps/rejected": -0.29087454080581665, + "loss": 97.86416625976562, + "loss/core": 97.86416625976562, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.0255415439605713, + "rewards/margins": 1.2207753658294678, + "rewards/rejected": 0.8047658205032349, + "step": 510 + }, + { + "epoch": 0.49205780484891914, + "grad_norm": 280.0, + "learning_rate": 2.315937497570688e-07, + "logits/chosen": -1.6068187952041626, + "logits/rejected": -1.6211055517196655, + "logps/chosen": -0.2836398482322693, + "logps/rejected": -0.3053371012210846, + "loss": 98.37687683105469, + "loss/core": 98.37687683105469, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.53507399559021, + "rewards/margins": 0.8702079057693481, + "rewards/rejected": 1.6648662090301514, + "step": 515 + }, + { + "epoch": 0.4968350650901708, + "grad_norm": 310.0, + "learning_rate": 2.2676253476765194e-07, + "logits/chosen": -1.8465595245361328, + "logits/rejected": -1.9376262426376343, + "logps/chosen": -0.3172188401222229, + "logps/rejected": -0.3324028551578522, + "loss": 96.29479217529297, + "loss/core": 96.29479217529297, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.1222901344299316, + "rewards/margins": 2.14201021194458, + "rewards/rejected": 0.9802799224853516, + "step": 520 + }, + { + "epoch": 0.5016123253314224, + "grad_norm": 980.0, + "learning_rate": 2.2194005843856633e-07, + "logits/chosen": -1.9088995456695557, + "logits/rejected": -1.8524242639541626, + "logps/chosen": -0.28945475816726685, + "logps/rejected": -0.29769039154052734, + "loss": 95.80438995361328, + "loss/core": 95.80438995361328, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.41683292388916, + "rewards/margins": 2.928234338760376, + "rewards/rejected": 1.4885984659194946, + "step": 525 + }, + { + "epoch": 0.5063895855726741, + "grad_norm": 54.75, + "learning_rate": 2.1712813430593434e-07, + "logits/chosen": -1.7017314434051514, + "logits/rejected": -1.697610855102539, + "logps/chosen": -0.28460589051246643, + "logps/rejected": -0.28730088472366333, + "loss": 97.26568603515625, + "loss/core": 97.26568603515625, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.064678907394409, + "rewards/margins": 1.5623985528945923, + "rewards/rejected": 1.5022804737091064, + "step": 530 + }, + { + "epoch": 0.5111668458139257, + "grad_norm": 262.0, + "learning_rate": 2.123285719376292e-07, + "logits/chosen": -1.7895727157592773, + "logits/rejected": -1.7456676959991455, + "logps/chosen": -0.28233015537261963, + "logps/rejected": -0.27622172236442566, + "loss": 96.65437316894531, + "loss/core": 96.65437316894531, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.126199722290039, + "rewards/margins": 2.0186972618103027, + "rewards/rejected": 1.1075026988983154, + "step": 535 + }, + { + "epoch": 0.5159441060551774, + "grad_norm": 83.5, + "learning_rate": 2.0754317625276982e-07, + "logits/chosen": -1.7324750423431396, + "logits/rejected": -1.6515331268310547, + "logps/chosen": -0.2788090407848358, + "logps/rejected": -0.28251534700393677, + "loss": 100.15641784667969, + "loss/core": 100.15641784667969, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.587709665298462, + "rewards/margins": 0.028971601277589798, + "rewards/rejected": 1.5587379932403564, + "step": 540 + }, + { + "epoch": 0.520721366296429, + "grad_norm": 612.0, + "learning_rate": 2.0277374684296498e-07, + "logits/chosen": -2.0007410049438477, + "logits/rejected": -1.9063104391098022, + "logps/chosen": -0.2854381501674652, + "logps/rejected": -0.28918930888175964, + "loss": 97.22596740722656, + "loss/core": 97.22596740722656, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.619149923324585, + "rewards/margins": 1.7024046182632446, + "rewards/rejected": 0.9167453646659851, + "step": 545 + }, + { + "epoch": 0.5254986265376806, + "grad_norm": 203.0, + "learning_rate": 1.980220772955602e-07, + "logits/chosen": -1.5841925144195557, + "logits/rejected": -1.5888211727142334, + "logps/chosen": -0.276597261428833, + "logps/rejected": -0.29771602153778076, + "loss": 97.82579803466797, + "loss/core": 97.82579803466797, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.763697862625122, + "rewards/margins": 1.1396576166152954, + "rewards/rejected": 1.6240402460098267, + "step": 550 + }, + { + "epoch": 0.5302758867789323, + "grad_norm": 112.0, + "learning_rate": 1.932899545191433e-07, + "logits/chosen": -1.8474057912826538, + "logits/rejected": -1.8652210235595703, + "logps/chosen": -0.3112329840660095, + "logps/rejected": -0.2748587727546692, + "loss": 95.9908447265625, + "loss/core": 95.9908447265625, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.2994256019592285, + "rewards/margins": 2.3030340671539307, + "rewards/rejected": 0.9963914155960083, + "step": 555 + }, + { + "epoch": 0.5350531470201839, + "grad_norm": 300.0, + "learning_rate": 1.885791580715609e-07, + "logits/chosen": -1.7741531133651733, + "logits/rejected": -1.8085333108901978, + "logps/chosen": -0.27475112676620483, + "logps/rejected": -0.27942371368408203, + "loss": 97.57888793945312, + "loss/core": 97.57888793945312, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.8250153064727783, + "rewards/margins": 1.2808738946914673, + "rewards/rejected": 1.5441416501998901, + "step": 560 + }, + { + "epoch": 0.5398304072614356, + "grad_norm": 302.0, + "learning_rate": 1.8389145949069951e-07, + "logits/chosen": -1.8415495157241821, + "logits/rejected": -1.8162124156951904, + "logps/chosen": -0.287056028842926, + "logps/rejected": -0.30101385712623596, + "loss": 98.02638244628906, + "loss/core": 98.02638244628906, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.785414695739746, + "rewards/margins": 1.0829885005950928, + "rewards/rejected": 1.702426552772522, + "step": 565 + }, + { + "epoch": 0.5446076675026872, + "grad_norm": 752.0, + "learning_rate": 1.792286216282824e-07, + "logits/chosen": -1.7960054874420166, + "logits/rejected": -1.8172178268432617, + "logps/chosen": -0.28870075941085815, + "logps/rejected": -0.2848988175392151, + "loss": 97.02275085449219, + "loss/core": 97.02275085449219, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.343677043914795, + "rewards/margins": 1.6194202899932861, + "rewards/rejected": 1.7242568731307983, + "step": 570 + }, + { + "epoch": 0.5493849277439389, + "grad_norm": 87.5, + "learning_rate": 1.745923979869336e-07, + "logits/chosen": -1.8886982202529907, + "logits/rejected": -1.9144996404647827, + "logps/chosen": -0.30019474029541016, + "logps/rejected": -0.3224802017211914, + "loss": 97.74614715576172, + "loss/core": 97.74614715576172, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.847407579421997, + "rewards/margins": 1.279578685760498, + "rewards/rejected": 0.5678285360336304, + "step": 575 + }, + { + "epoch": 0.5541621879851905, + "grad_norm": 59.75, + "learning_rate": 1.6998453206075708e-07, + "logits/chosen": -1.974991798400879, + "logits/rejected": -2.0810978412628174, + "logps/chosen": -0.26656287908554077, + "logps/rejected": -0.27996402978897095, + "loss": 99.16670989990234, + "loss/core": 99.16670989990234, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.4196202754974365, + "rewards/margins": 0.45080775022506714, + "rewards/rejected": 0.9688124656677246, + "step": 580 + }, + { + "epoch": 0.5589394482264421, + "grad_norm": 157.0, + "learning_rate": 1.6540675667967973e-07, + "logits/chosen": -1.9742958545684814, + "logits/rejected": -1.868665337562561, + "logps/chosen": -0.2701478898525238, + "logps/rejected": -0.28200751543045044, + "loss": 98.4051284790039, + "loss/core": 98.4051284790039, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.223400115966797, + "rewards/margins": 0.8472920656204224, + "rewards/rejected": 1.3761080503463745, + "step": 585 + }, + { + "epoch": 0.5637167084676937, + "grad_norm": 102.0, + "learning_rate": 1.60860793357805e-07, + "logits/chosen": -1.8315715789794922, + "logits/rejected": -1.8810384273529053, + "logps/chosen": -0.2877020239830017, + "logps/rejected": -0.28643354773521423, + "loss": 95.52645111083984, + "loss/core": 95.52645111083984, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.6193325519561768, + "rewards/margins": 2.5816495418548584, + "rewards/rejected": 1.0376828908920288, + "step": 590 + }, + { + "epoch": 0.5684939687089454, + "grad_norm": 139.0, + "learning_rate": 1.5634835164602196e-07, + "logits/chosen": -1.7726303339004517, + "logits/rejected": -1.782941222190857, + "logps/chosen": -0.27482694387435913, + "logps/rejected": -0.2720108926296234, + "loss": 98.37557220458984, + "loss/core": 98.37557220458984, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.714624047279358, + "rewards/margins": 0.8266318440437317, + "rewards/rejected": 0.8879922032356262, + "step": 595 + }, + { + "epoch": 0.5732712289501971, + "grad_norm": 230.0, + "learning_rate": 1.518711284891132e-07, + "logits/chosen": -1.8123525381088257, + "logits/rejected": -1.7736984491348267, + "logps/chosen": -0.2656711935997009, + "logps/rejected": -0.281463086605072, + "loss": 98.5457763671875, + "loss/core": 98.5457763671875, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.8796374797821045, + "rewards/margins": 0.7882030606269836, + "rewards/rejected": 1.0914344787597656, + "step": 600 + }, + { + "epoch": 0.5780484891914487, + "grad_norm": 408.0, + "learning_rate": 1.47430807587603e-07, + "logits/chosen": -1.9366077184677124, + "logits/rejected": -2.003767728805542, + "logps/chosen": -0.2733065187931061, + "logps/rejected": -0.2762848734855652, + "loss": 95.27114868164062, + "loss/core": 95.27114868164062, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.252087593078613, + "rewards/margins": 3.052070140838623, + "rewards/rejected": 1.2000172138214111, + "step": 605 + }, + { + "epoch": 0.5828257494327004, + "grad_norm": 366.0, + "learning_rate": 1.430290587645865e-07, + "logits/chosen": -1.6373779773712158, + "logits/rejected": -1.6163759231567383, + "logps/chosen": -0.2743273377418518, + "logps/rejected": -0.2724093794822693, + "loss": 96.03132629394531, + "loss/core": 96.03132629394531, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 4.291282653808594, + "rewards/margins": 2.326350212097168, + "rewards/rejected": 1.9649326801300049, + "step": 610 + }, + { + "epoch": 0.587603009673952, + "grad_norm": 152.0, + "learning_rate": 1.3866753733777765e-07, + "logits/chosen": -1.4851322174072266, + "logits/rejected": -1.566975712776184, + "logps/chosen": -0.31360191106796265, + "logps/rejected": -0.30080071091651917, + "loss": 97.76426696777344, + "loss/core": 97.76426696777344, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.744011163711548, + "rewards/margins": 1.2261463403701782, + "rewards/rejected": 1.5178649425506592, + "step": 615 + }, + { + "epoch": 0.5923802699152036, + "grad_norm": 124.5, + "learning_rate": 1.343478834970121e-07, + "logits/chosen": -1.8698352575302124, + "logits/rejected": -1.901095986366272, + "logps/chosen": -0.2850908637046814, + "logps/rejected": -0.2894435524940491, + "loss": 99.82987976074219, + "loss/core": 99.82987976074219, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.6998624801635742, + "rewards/margins": 0.2176227569580078, + "rewards/rejected": 1.4822396039962769, + "step": 620 + }, + { + "epoch": 0.5971575301564552, + "grad_norm": 1840.0, + "learning_rate": 1.3007172168743852e-07, + "logits/chosen": -1.7007501125335693, + "logits/rejected": -1.6904194355010986, + "logps/chosen": -0.3048066198825836, + "logps/rejected": -0.33391109108924866, + "loss": 96.62384033203125, + "loss/core": 96.62384033203125, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.886601686477661, + "rewards/margins": 2.3527793884277344, + "rewards/rejected": 1.5338222980499268, + "step": 625 + }, + { + "epoch": 0.6019347903977069, + "grad_norm": 146.0, + "learning_rate": 1.25840659998631e-07, + "logits/chosen": -1.7266985177993774, + "logits/rejected": -1.8024879693984985, + "logps/chosen": -0.27022111415863037, + "logps/rejected": -0.2823617458343506, + "loss": 96.9268798828125, + "loss/core": 96.9268798828125, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.9654762744903564, + "rewards/margins": 1.7113138437271118, + "rewards/rejected": 1.2541625499725342, + "step": 630 + }, + { + "epoch": 0.6067120506389586, + "grad_norm": 636.0, + "learning_rate": 1.2165628955985313e-07, + "logits/chosen": -1.757341742515564, + "logits/rejected": -1.8153082132339478, + "logps/chosen": -0.3024021089076996, + "logps/rejected": -0.29852160811424255, + "loss": 97.65138244628906, + "loss/core": 97.65138244628906, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.330906391143799, + "rewards/margins": 1.3007912635803223, + "rewards/rejected": 1.030115008354187, + "step": 635 + }, + { + "epoch": 0.6114893108802102, + "grad_norm": 170.0, + "learning_rate": 1.175201839416988e-07, + "logits/chosen": -1.9056514501571655, + "logits/rejected": -1.9911472797393799, + "logps/chosen": -0.29884806275367737, + "logps/rejected": -0.2987546920776367, + "loss": 97.85620880126953, + "loss/core": 97.85620880126953, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.9839823246002197, + "rewards/margins": 1.1865044832229614, + "rewards/rejected": 0.7974779009819031, + "step": 640 + }, + { + "epoch": 0.6162665711214619, + "grad_norm": 235.0, + "learning_rate": 1.1343389856433658e-07, + "logits/chosen": -1.9996535778045654, + "logits/rejected": -2.021444797515869, + "logps/chosen": -0.28937220573425293, + "logps/rejected": -0.29186397790908813, + "loss": 99.2889404296875, + "loss/core": 99.2889404296875, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.6865224838256836, + "rewards/margins": 0.42166757583618164, + "rewards/rejected": 1.264854907989502, + "step": 645 + }, + { + "epoch": 0.6210438313627135, + "grad_norm": 137.0, + "learning_rate": 1.0939897011258e-07, + "logits/chosen": -1.9977695941925049, + "logits/rejected": -1.9729766845703125, + "logps/chosen": -0.2890729010105133, + "logps/rejected": -0.2976759374141693, + "loss": 99.13239288330078, + "loss/core": 99.13239288330078, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.3376524448394775, + "rewards/margins": 0.48343315720558167, + "rewards/rejected": 0.8542193174362183, + "step": 650 + }, + { + "epoch": 0.6258210916039652, + "grad_norm": 584.0, + "learning_rate": 1.0541691595800336e-07, + "logits/chosen": -1.7728780508041382, + "logits/rejected": -1.8141391277313232, + "logps/chosen": -0.2818259596824646, + "logps/rejected": -0.28773942589759827, + "loss": 98.06580352783203, + "loss/core": 98.06580352783203, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.008873701095581, + "rewards/margins": 1.0006710290908813, + "rewards/rejected": 1.0082027912139893, + "step": 655 + }, + { + "epoch": 0.6305983518452167, + "grad_norm": 93.0, + "learning_rate": 1.0148923358832021e-07, + "logits/chosen": -1.9082142114639282, + "logits/rejected": -1.9239473342895508, + "logps/chosen": -0.2919521927833557, + "logps/rejected": -0.30435115098953247, + "loss": 97.76885223388672, + "loss/core": 97.76885223388672, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.796613931655884, + "rewards/margins": 1.4722524881362915, + "rewards/rejected": 1.3243615627288818, + "step": 660 + }, + { + "epoch": 0.6353756120864684, + "grad_norm": 350.0, + "learning_rate": 9.761740004423926e-08, + "logits/chosen": -1.9143612384796143, + "logits/rejected": -1.8848596811294556, + "logps/chosen": -0.2826116383075714, + "logps/rejected": -0.27971625328063965, + "loss": 98.54499816894531, + "loss/core": 98.54499816894531, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.364816427230835, + "rewards/margins": 0.8677363395690918, + "rewards/rejected": 1.4970804452896118, + "step": 665 + }, + { + "epoch": 0.64015287232772, + "grad_norm": 88.0, + "learning_rate": 9.380287136400999e-08, + "logits/chosen": -1.7525949478149414, + "logits/rejected": -1.8127027750015259, + "logps/chosen": -0.2727015018463135, + "logps/rejected": -0.31174588203430176, + "loss": 95.95095825195312, + "loss/core": 95.95095825195312, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.6611416339874268, + "rewards/margins": 2.319688320159912, + "rewards/rejected": 1.3414534330368042, + "step": 670 + }, + { + "epoch": 0.6449301325689717, + "grad_norm": 170.0, + "learning_rate": 9.004708203586628e-08, + "logits/chosen": -1.6614296436309814, + "logits/rejected": -1.6599369049072266, + "logps/chosen": -0.2885296940803528, + "logps/rejected": -0.28654050827026367, + "loss": 98.60887145996094, + "loss/core": 98.60887145996094, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6211833953857422, + "rewards/margins": 0.7235764265060425, + "rewards/rejected": 0.897607147693634, + "step": 675 + }, + { + "epoch": 0.6497073928102234, + "grad_norm": 87.0, + "learning_rate": 8.635144445857407e-08, + "logits/chosen": -1.8365249633789062, + "logits/rejected": -1.7715469598770142, + "logps/chosen": -0.2822161316871643, + "logps/rejected": -0.29134494066238403, + "loss": 98.63749694824219, + "loss/core": 98.63749694824219, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.1443328857421875, + "rewards/margins": 0.7564715147018433, + "rewards/rejected": 1.3878613710403442, + "step": 680 + }, + { + "epoch": 0.654484653051475, + "grad_norm": 116.0, + "learning_rate": 8.271734841028552e-08, + "logits/chosen": -1.6401258707046509, + "logits/rejected": -1.6516834497451782, + "logps/chosen": -0.2528996169567108, + "logps/rejected": -0.2761356830596924, + "loss": 97.95409393310547, + "loss/core": 97.95409393310547, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.8932156562805176, + "rewards/margins": 1.3494775295257568, + "rewards/rejected": 1.5437384843826294, + "step": 685 + }, + { + "epoch": 0.6592619132927267, + "grad_norm": 104.5, + "learning_rate": 7.91461605259007e-08, + "logits/chosen": -1.5243980884552002, + "logits/rejected": -1.5546753406524658, + "logps/chosen": -0.3075758218765259, + "logps/rejected": -0.31810295581817627, + "loss": 98.9781494140625, + "loss/core": 98.9781494140625, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.7742786407470703, + "rewards/margins": 0.5831277966499329, + "rewards/rejected": 1.1911507844924927, + "step": 690 + }, + { + "epoch": 0.6640391735339782, + "grad_norm": 130.0, + "learning_rate": 7.563922378313217e-08, + "logits/chosen": -1.8852567672729492, + "logits/rejected": -1.9301567077636719, + "logps/chosen": -0.30527764558792114, + "logps/rejected": -0.292079359292984, + "loss": 97.71669006347656, + "loss/core": 97.71669006347656, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.9805892705917358, + "rewards/margins": 1.289915680885315, + "rewards/rejected": 0.6906735301017761, + "step": 695 + }, + { + "epoch": 0.6688164337752299, + "grad_norm": 2576.0, + "learning_rate": 7.219785699746572e-08, + "logits/chosen": -1.7994495630264282, + "logits/rejected": -1.8756701946258545, + "logps/chosen": -0.29662424325942993, + "logps/rejected": -0.3126930594444275, + "loss": 99.03868103027344, + "loss/core": 99.03868103027344, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.2323403358459473, + "rewards/margins": 0.5778122544288635, + "rewards/rejected": 1.6545279026031494, + "step": 700 + }, + { + "epoch": 0.6735936940164815, + "grad_norm": 354.0, + "learning_rate": 6.882335432620779e-08, + "logits/chosen": -1.8467237949371338, + "logits/rejected": -1.8279880285263062, + "logps/chosen": -0.2809194028377533, + "logps/rejected": -0.2668314576148987, + "loss": 96.70755004882812, + "loss/core": 96.70755004882812, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.3989696502685547, + "rewards/margins": 1.8612855672836304, + "rewards/rejected": 1.5376842021942139, + "step": 705 + }, + { + "epoch": 0.6783709542577332, + "grad_norm": 1328.0, + "learning_rate": 6.551698478180589e-08, + "logits/chosen": -1.628369688987732, + "logits/rejected": -1.6482683420181274, + "logps/chosen": -0.27590495347976685, + "logps/rejected": -0.29148903489112854, + "loss": 96.99464416503906, + "loss/core": 96.99464416503906, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.6098170280456543, + "rewards/margins": 1.649452805519104, + "rewards/rejected": 1.9603639841079712, + "step": 710 + }, + { + "epoch": 0.6831482144989849, + "grad_norm": 122.0, + "learning_rate": 6.22799917546252e-08, + "logits/chosen": -1.8616364002227783, + "logits/rejected": -1.8149951696395874, + "logps/chosen": -0.28448861837387085, + "logps/rejected": -0.3247087597846985, + "loss": 98.08930969238281, + "loss/core": 98.08930969238281, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.8618444204330444, + "rewards/margins": 1.042851209640503, + "rewards/rejected": 0.818993091583252, + "step": 715 + }, + { + "epoch": 0.6879254747402365, + "grad_norm": 312.0, + "learning_rate": 5.9113592545359944e-08, + "logits/chosen": -1.7652199268341064, + "logits/rejected": -1.791394829750061, + "logps/chosen": -0.29653897881507874, + "logps/rejected": -0.2996441125869751, + "loss": 99.06591796875, + "loss/core": 99.06591796875, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.1000869274139404, + "rewards/margins": 0.5941171646118164, + "rewards/rejected": 1.505969762802124, + "step": 720 + }, + { + "epoch": 0.6927027349814882, + "grad_norm": 138.0, + "learning_rate": 5.601897790725643e-08, + "logits/chosen": -1.9410686492919922, + "logits/rejected": -1.9667949676513672, + "logps/chosen": -0.2847985327243805, + "logps/rejected": -0.2775443196296692, + "loss": 98.34650421142578, + "loss/core": 98.34650421142578, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.5937460660934448, + "rewards/margins": 0.865003228187561, + "rewards/rejected": 0.7287428379058838, + "step": 725 + }, + { + "epoch": 0.6974799952227397, + "grad_norm": 1392.0, + "learning_rate": 5.299731159831952e-08, + "logits/chosen": -1.902320146560669, + "logits/rejected": -1.948782205581665, + "logps/chosen": -0.28654351830482483, + "logps/rejected": -0.29139798879623413, + "loss": 99.9100570678711, + "loss/core": 99.9100570678711, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.0183780193328857, + "rewards/margins": 0.11728978157043457, + "rewards/rejected": 0.9010881185531616, + "step": 730 + }, + { + "epoch": 0.7022572554639914, + "grad_norm": 278.0, + "learning_rate": 5.0049729943671013e-08, + "logits/chosen": -1.8456531763076782, + "logits/rejected": -1.8964226245880127, + "logps/chosen": -0.3196776807308197, + "logps/rejected": -0.31731048226356506, + "loss": 98.53561401367188, + "loss/core": 98.53561401367188, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.4210125207901, + "rewards/margins": 0.7529805898666382, + "rewards/rejected": 0.6680319309234619, + "step": 735 + }, + { + "epoch": 0.707034515705243, + "grad_norm": 145.0, + "learning_rate": 4.7177341408223994e-08, + "logits/chosen": -1.7140204906463623, + "logits/rejected": -1.7427120208740234, + "logps/chosen": -0.3081023097038269, + "logps/rejected": -0.3138575553894043, + "loss": 99.85032653808594, + "loss/core": 99.85032653808594, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.6136274337768555, + "rewards/margins": 0.7614258527755737, + "rewards/rejected": 1.8522018194198608, + "step": 740 + }, + { + "epoch": 0.7118117759464947, + "grad_norm": 564.0, + "learning_rate": 4.438122617983442e-08, + "logits/chosen": -1.7579317092895508, + "logits/rejected": -1.8332077264785767, + "logps/chosen": -0.2885438799858093, + "logps/rejected": -0.2813180088996887, + "loss": 97.10404968261719, + "loss/core": 97.10404968261719, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.1403250694274902, + "rewards/margins": 1.6182429790496826, + "rewards/rejected": 1.5220820903778076, + "step": 745 + }, + { + "epoch": 0.7165890361877463, + "grad_norm": 668.0, + "learning_rate": 4.1662435763087114e-08, + "logits/chosen": -1.9111521244049072, + "logits/rejected": -1.9151312112808228, + "logps/chosen": -0.3117213845252991, + "logps/rejected": -0.30328887701034546, + "loss": 100.52747344970703, + "loss/core": 100.52747344970703, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.2058069705963135, + "rewards/margins": -0.05395696312189102, + "rewards/rejected": 1.2597639560699463, + "step": 750 + }, + { + "epoch": 0.721366296428998, + "grad_norm": 286.0, + "learning_rate": 3.902199258386732e-08, + "logits/chosen": -1.7503387928009033, + "logits/rejected": -1.6461431980133057, + "logps/chosen": -0.30825966596603394, + "logps/rejected": -0.2929074168205261, + "loss": 98.85442352294922, + "loss/core": 98.85442352294922, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.3893883228302, + "rewards/margins": 0.6656953692436218, + "rewards/rejected": 1.7236926555633545, + "step": 755 + }, + { + "epoch": 0.7261435566702497, + "grad_norm": 147.0, + "learning_rate": 3.646088960486862e-08, + "logits/chosen": -1.7998508214950562, + "logits/rejected": -1.8412847518920898, + "logps/chosen": -0.30292657017707825, + "logps/rejected": -0.29351457953453064, + "loss": 97.87567901611328, + "loss/core": 97.87567901611328, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9885555505752563, + "rewards/margins": 1.1239904165267944, + "rewards/rejected": 0.8645650148391724, + "step": 760 + }, + { + "epoch": 0.7309208169115012, + "grad_norm": 241.0, + "learning_rate": 3.398008995217988e-08, + "logits/chosen": -1.7207539081573486, + "logits/rejected": -1.7541128396987915, + "logps/chosen": -0.2763254940509796, + "logps/rejected": -0.27128952741622925, + "loss": 98.617919921875, + "loss/core": 98.617919921875, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7916427850723267, + "rewards/margins": 0.7221640944480896, + "rewards/rejected": 1.0694787502288818, + "step": 765 + }, + { + "epoch": 0.7356980771527529, + "grad_norm": 430.0, + "learning_rate": 3.1580526553093315e-08, + "logits/chosen": -1.6970971822738647, + "logits/rejected": -1.7250639200210571, + "logps/chosen": -0.3074544072151184, + "logps/rejected": -0.280496746301651, + "loss": 98.15419006347656, + "loss/core": 98.15419006347656, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.7565841674804688, + "rewards/margins": 1.3622013330459595, + "rewards/rejected": 2.394383192062378, + "step": 770 + }, + { + "epoch": 0.7404753373940045, + "grad_norm": 63.0, + "learning_rate": 2.9263101785268252e-08, + "logits/chosen": -1.8713486194610596, + "logits/rejected": -1.898063063621521, + "logps/chosen": -0.3057042062282562, + "logps/rejected": -0.2998577952384949, + "loss": 98.88420104980469, + "loss/core": 98.88420104980469, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.531863808631897, + "rewards/margins": 0.5855029225349426, + "rewards/rejected": 0.9463608860969543, + "step": 775 + }, + { + "epoch": 0.7452525976352562, + "grad_norm": 229.0, + "learning_rate": 2.7028687137384264e-08, + "logits/chosen": -1.7387259006500244, + "logits/rejected": -1.7721675634384155, + "logps/chosen": -0.3017333447933197, + "logps/rejected": -0.2997443377971649, + "loss": 98.89266204833984, + "loss/core": 98.89266204833984, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8950691223144531, + "rewards/margins": 0.5870526432991028, + "rewards/rejected": 1.3080164194107056, + "step": 780 + }, + { + "epoch": 0.7500298578765078, + "grad_norm": 84.5, + "learning_rate": 2.4878122881409447e-08, + "logits/chosen": -1.5948407649993896, + "logits/rejected": -1.566903829574585, + "logps/chosen": -0.30947035551071167, + "logps/rejected": -0.2902279198169708, + "loss": 96.91048431396484, + "loss/core": 96.91048431396484, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.4462432861328125, + "rewards/margins": 2.065670967102051, + "rewards/rejected": 1.380571961402893, + "step": 785 + }, + { + "epoch": 0.7548071181177595, + "grad_norm": 2024.0, + "learning_rate": 2.2812217756608937e-08, + "logits/chosen": -1.8185821771621704, + "logits/rejected": -1.7109493017196655, + "logps/chosen": -0.2856631278991699, + "logps/rejected": -0.31153404712677, + "loss": 99.07926940917969, + "loss/core": 99.07926940917969, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.6235830783843994, + "rewards/margins": 0.6263852715492249, + "rewards/rejected": 1.9971978664398193, + "step": 790 + }, + { + "epoch": 0.7595843783590112, + "grad_norm": 300.0, + "learning_rate": 2.0831748665410763e-08, + "logits/chosen": -1.777958869934082, + "logits/rejected": -1.7994849681854248, + "logps/chosen": -0.30515098571777344, + "logps/rejected": -0.29699045419692993, + "loss": 97.77039337158203, + "loss/core": 97.77039337158203, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.403167486190796, + "rewards/margins": 1.225862979888916, + "rewards/rejected": 1.1773045063018799, + "step": 795 + }, + { + "epoch": 0.7643616386002627, + "grad_norm": 120.5, + "learning_rate": 1.8937460381244967e-08, + "logits/chosen": -1.8459327220916748, + "logits/rejected": -1.8227803707122803, + "logps/chosen": -0.28920984268188477, + "logps/rejected": -0.2841506004333496, + "loss": 97.98655700683594, + "loss/core": 97.98655700683594, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.2972395420074463, + "rewards/margins": 1.0846086740493774, + "rewards/rejected": 1.2126309871673584, + "step": 800 + }, + { + "epoch": 0.7691388988415144, + "grad_norm": 212.0, + "learning_rate": 1.713006526846439e-08, + "logits/chosen": -1.7277157306671143, + "logits/rejected": -1.7201650142669678, + "logps/chosen": -0.27354103326797485, + "logps/rejected": -0.2667585015296936, + "loss": 99.72236633300781, + "loss/core": 99.72236633300781, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.2702925205230713, + "rewards/margins": 0.24597541987895966, + "rewards/rejected": 2.0243170261383057, + "step": 805 + }, + { + "epoch": 0.773916159082766, + "grad_norm": 118.0, + "learning_rate": 1.541024301445404e-08, + "logits/chosen": -1.8017603158950806, + "logits/rejected": -1.7072747945785522, + "logps/chosen": -0.3080991208553314, + "logps/rejected": -0.31515592336654663, + "loss": 100.06008911132812, + "loss/core": 100.06008911132812, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.5322774648666382, + "rewards/margins": 0.039038121700286865, + "rewards/rejected": 1.493239402770996, + "step": 810 + }, + { + "epoch": 0.7786934193240177, + "grad_norm": 302.0, + "learning_rate": 1.3778640374027983e-08, + "logits/chosen": -1.8767938613891602, + "logits/rejected": -1.9253695011138916, + "logps/chosen": -0.2829209566116333, + "logps/rejected": -0.2839273512363434, + "loss": 97.97332000732422, + "loss/core": 97.97332000732422, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.1320853233337402, + "rewards/margins": 1.0345160961151123, + "rewards/rejected": 1.097569227218628, + "step": 815 + }, + { + "epoch": 0.7834706795652693, + "grad_norm": 91.0, + "learning_rate": 1.2235870926211616e-08, + "logits/chosen": -1.7318302392959595, + "logits/rejected": -1.735965371131897, + "logps/chosen": -0.28805407881736755, + "logps/rejected": -0.31744417548179626, + "loss": 97.60137176513672, + "loss/core": 97.60137176513672, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.9624719619750977, + "rewards/margins": 1.2788426876068115, + "rewards/rejected": 0.6836291551589966, + "step": 820 + }, + { + "epoch": 0.788247939806521, + "grad_norm": 394.0, + "learning_rate": 1.0782514843499652e-08, + "logits/chosen": -1.7264869213104248, + "logits/rejected": -1.8847726583480835, + "logps/chosen": -0.29893332719802856, + "logps/rejected": -0.284157931804657, + "loss": 97.54736328125, + "loss/core": 97.54736328125, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.4949123859405518, + "rewards/margins": 1.2742880582809448, + "rewards/rejected": 1.220624327659607, + "step": 825 + }, + { + "epoch": 0.7930252000477725, + "grad_norm": 184.0, + "learning_rate": 9.419118673676924e-09, + "logits/chosen": -1.914811134338379, + "logits/rejected": -1.8754631280899048, + "logps/chosen": -0.27880120277404785, + "logps/rejected": -0.30228403210639954, + "loss": 100.15513610839844, + "loss/core": 100.15513610839844, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.7236683368682861, + "rewards/margins": 0.013875794596970081, + "rewards/rejected": 1.7097924947738647, + "step": 830 + }, + { + "epoch": 0.7978024602890242, + "grad_norm": 596.0, + "learning_rate": 8.14619513428405e-09, + "logits/chosen": -1.7310373783111572, + "logits/rejected": -1.749680757522583, + "logps/chosen": -0.25445908308029175, + "logps/rejected": -0.3001643717288971, + "loss": 97.2359390258789, + "loss/core": 97.2359390258789, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.406795024871826, + "rewards/margins": 1.566819190979004, + "rewards/rejected": 0.8399758338928223, + "step": 835 + }, + { + "epoch": 0.8025797205302759, + "grad_norm": 1280.0, + "learning_rate": 6.96422291980539e-09, + "logits/chosen": -1.8255088329315186, + "logits/rejected": -1.8728702068328857, + "logps/chosen": -0.322468101978302, + "logps/rejected": -0.3011077642440796, + "loss": 97.09306335449219, + "loss/core": 97.09306335449219, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.803194284439087, + "rewards/margins": 1.7630542516708374, + "rewards/rejected": 1.0401400327682495, + "step": 840 + }, + { + "epoch": 0.8073569807715275, + "grad_norm": 117.0, + "learning_rate": 5.8736465216517594e-09, + "logits/chosen": -1.7962318658828735, + "logits/rejected": -1.8419160842895508, + "logps/chosen": -0.2930757701396942, + "logps/rejected": -0.31055259704589844, + "loss": 98.22071838378906, + "loss/core": 98.22071838378906, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.4233886003494263, + "rewards/margins": 0.9614946246147156, + "rewards/rejected": 0.4618939459323883, + "step": 845 + }, + { + "epoch": 0.8121342410127792, + "grad_norm": 628.0, + "learning_rate": 4.874876061005173e-09, + "logits/chosen": -1.9622621536254883, + "logits/rejected": -2.055959701538086, + "logps/chosen": -0.2516617178916931, + "logps/rejected": -0.24817457795143127, + "loss": 98.69639587402344, + "loss/core": 98.69639587402344, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.561449408531189, + "rewards/margins": 0.6674880385398865, + "rewards/rejected": 0.8939614295959473, + "step": 850 + }, + { + "epoch": 0.8169115012540308, + "grad_norm": 330.0, + "learning_rate": 3.968287134589188e-09, + "logits/chosen": -1.766444444656372, + "logits/rejected": -1.7449138164520264, + "logps/chosen": -0.3007269501686096, + "logps/rejected": -0.2932063043117523, + "loss": 97.42681884765625, + "loss/core": 97.42681884765625, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.6898107528686523, + "rewards/margins": 1.440416932106018, + "rewards/rejected": 1.2493940591812134, + "step": 855 + }, + { + "epoch": 0.8216887614952825, + "grad_norm": 75.5, + "learning_rate": 3.154220673422192e-09, + "logits/chosen": -1.6410402059555054, + "logits/rejected": -1.6836130619049072, + "logps/chosen": -0.29940205812454224, + "logps/rejected": -0.29790908098220825, + "loss": 96.29589080810547, + "loss/core": 96.29589080810547, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.08776330947876, + "rewards/margins": 2.1483874320983887, + "rewards/rejected": 1.9393761157989502, + "step": 860 + }, + { + "epoch": 0.826466021736534, + "grad_norm": 196.0, + "learning_rate": 2.4329828146074096e-09, + "logits/chosen": -1.6578218936920166, + "logits/rejected": -1.688619613647461, + "logps/chosen": -0.2899990677833557, + "logps/rejected": -0.29334208369255066, + "loss": 98.53080749511719, + "loss/core": 98.53080749511719, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.461031436920166, + "rewards/margins": 0.8699079751968384, + "rewards/rejected": 1.591123342514038, + "step": 865 + }, + { + "epoch": 0.8312432819777857, + "grad_norm": 191.0, + "learning_rate": 1.8048447862070714e-09, + "logits/chosen": -1.834404706954956, + "logits/rejected": -1.859675645828247, + "logps/chosen": -0.3091299831867218, + "logps/rejected": -0.29531824588775635, + "loss": 96.41517639160156, + "loss/core": 96.41517639160156, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.204866886138916, + "rewards/margins": 2.250494956970215, + "rewards/rejected": 0.9543716311454773, + "step": 870 + }, + { + "epoch": 0.8360205422190374, + "grad_norm": 138.0, + "learning_rate": 1.2700428052447033e-09, + "logits/chosen": -1.7764365673065186, + "logits/rejected": -1.8405851125717163, + "logps/chosen": -0.30803143978118896, + "logps/rejected": -0.32817330956459045, + "loss": 98.6803207397461, + "loss/core": 98.6803207397461, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.861143708229065, + "rewards/margins": 0.8110074996948242, + "rewards/rejected": 1.0501360893249512, + "step": 875 + }, + { + "epoch": 0.840797802460289, + "grad_norm": 540.0, + "learning_rate": 8.287779888734858e-10, + "logits/chosen": -1.8258100748062134, + "logits/rejected": -1.7874729633331299, + "logps/chosen": -0.2907932698726654, + "logps/rejected": -0.3241172432899475, + "loss": 99.53788757324219, + "loss/core": 99.53788757324219, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.4565396308898926, + "rewards/margins": 0.6172792315483093, + "rewards/rejected": 1.839260458946228, + "step": 880 + }, + { + "epoch": 0.8455750627015407, + "grad_norm": 217.0, + "learning_rate": 4.812162787445062e-10, + "logits/chosen": -1.7738778591156006, + "logits/rejected": -1.847890853881836, + "logps/chosen": -0.28218382596969604, + "logps/rejected": -0.2760196328163147, + "loss": 96.96441650390625, + "loss/core": 96.96441650390625, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.7287354469299316, + "rewards/margins": 1.59546959400177, + "rewards/rejected": 1.1332653760910034, + "step": 885 + }, + { + "epoch": 0.8503523229427923, + "grad_norm": 592.0, + "learning_rate": 2.2748837860270265e-10, + "logits/chosen": -1.7101322412490845, + "logits/rejected": -1.8211758136749268, + "logps/chosen": -0.297687828540802, + "logps/rejected": -0.27419671416282654, + "loss": 98.19453430175781, + "loss/core": 98.19453430175781, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.479161024093628, + "rewards/margins": 1.039329171180725, + "rewards/rejected": 1.4398319721221924, + "step": 890 + }, + { + "epoch": 0.855129583184044, + "grad_norm": 362.0, + "learning_rate": 6.768970513457151e-11, + "logits/chosen": -1.7563453912734985, + "logits/rejected": -1.736853837966919, + "logps/chosen": -0.2723637521266937, + "logps/rejected": -0.2854057848453522, + "loss": 98.63446807861328, + "loss/core": 98.63446807861328, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.3016631603240967, + "rewards/margins": 0.736989438533783, + "rewards/rejected": 1.564673662185669, + "step": 895 + }, + { + "epoch": 0.8599068434252956, + "grad_norm": 304.0, + "learning_rate": 1.8803520859811406e-12, + "logits/chosen": -1.7407211065292358, + "logits/rejected": -1.863157868385315, + "logps/chosen": -0.2687687277793884, + "logps/rejected": -0.2672789990901947, + "loss": 96.38660430908203, + "loss/core": 96.38660430908203, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 3.118961811065674, + "rewards/margins": 2.259530782699585, + "rewards/rejected": 0.8594307899475098, + "step": 900 + }, + { + "epoch": 0.8599068434252956, + "step": 900, + "total_flos": 0.0, + "train_loss": 97.93875423855252, + "train_runtime": 8489.6134, + "train_samples_per_second": 1.696, + "train_steps_per_second": 0.106 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..59ea0dc --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2d4e4c1416b4b5a828ef51c03814031b1c6fb13db1b432e1762686bf1b1e4f0 +size 6993