commit e5f17b1d585578b19c55f8b727f5a5720c0d929e Author: ModelHub XC Date: Sun Jul 26 03:32:09 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: promotion/qwen3-8b-aaai27-flagship-dpo-s43 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..a4242ee --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +--- +library_name: transformers +base_model: Qwen/Qwen3-8B +tags: +- ronpo +- mnpo +- preference-optimization +--- + +# qwen3-8b-aaai27-flagship-dpo-s43 + +Research checkpoint for the RONPO AAAI revision experiments. + +- Method: dpo +- Base model: `Qwen/Qwen3-8B` +- Seed: 43 +- Local source at upload time: `/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/dpo/seed43/attempt1` +- Uploaded at UTC: 2026-07-12T17:14:11Z +- Run status metadata: `{"attempt": 1, "effective_batch_size": 16, "method": "dpo", "objective_protocol": "objective_protocol.json", "optimizer_steps": 900, "seed": 43, "split_manifest": "split_manifest.json", "stability_gate": "stability_gate.json", "status": "completed", "wandb_run_id": "5095e5d2222c", "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/5095e5d2222c"}` + +AAAI-27 P1 matched budget: 900 optimizer steps, effective batch 16; passed non-thinking and collapse stability gates; sealed-test metrics are not used for checkpoint selection. + +Intended use: reproducibility and evaluation for the RONPO paper. This +checkpoint is not intended as a production assistant. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..b711000 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 0.6902858657307095, + "train_runtime": 8330.5288, + "train_samples": 16746, + "train_samples_per_second": 1.729, + "train_steps_per_second": 0.108 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..929ae7f --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +model_name_or_path: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/cache/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218 +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/avg: 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: dpo +eta: 0.0075 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.0 +preference_sft_weight: 0.0 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 5.0e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 43 +gradient_accumulation_steps: 8 +gradient_checkpointing: false +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 2 +per_device_eval_batch_size: 2 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/dpo/seed43/attempt1 +run_name: aaai27-flagship-full-dpo-s43-a1 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/job_status.json b/job_status.json new file mode 100644 index 0000000..4631de8 --- /dev/null +++ b/job_status.json @@ -0,0 +1,23 @@ +{ + "status": "completed", + "stage": "full", + "method": "dpo", + "seed": 43, + "attempt": 1, + "gpu": 7, + "gpus": [ + 7 + ], + "returncode": 0, + "finite_metrics": true, + "oom": false, + "traceback": false, + "reason": "passed", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "5095e5d2222c", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/5095e5d2222c", + "output_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/full/dpo/seed43/attempt1", + "log": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/flagship_20260712/logs/full_dpo_s43_a1.log", + "completed_at": "2026-07-12T17:11:50Z" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..b841ed6 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e66edd725df0780323fb54555de528f8395d3cbe812b6c008427027e6138ccd +size 16381517208 diff --git a/objective_protocol.json b/objective_protocol.json new file mode 100644 index 0000000..35b3b44 --- /dev/null +++ b/objective_protocol.json @@ -0,0 +1,69 @@ +{ + "schema_version": 1, + "frozen_at_kst": "2026-07-12T21:40:00+09:00", + "base_model": "Qwen/Qwen3-8B", + "primary_triple": { + "helpfulness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "ultrafeedback-helpfulness", + "transform": "identity" + }, + "safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "conciseness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "helpsteer-verbosity", + "transform": "negate" + } + }, + "gate_thresholds": { + "raw_score_std_strictly_greater_than": 0.01, + "aggregate_prompt_pair_spearman_median_strictly_less_than": 0.0, + "aggregate_pairwise_top1_mismatch_at_least": 0.5 + }, + "gate_pool": "source-train Qwen3-8B response pool split into deterministic train/validation; sealed source-test excluded", + "training_seeds": [42, 43, 44], + "optimizer_steps": 900, + "effective_batch_size": 16, + "decode": { + "enable_thinking": false, + "temperature": 0.7, + "top_p": 0.9, + "max_new_tokens": 2048, + "decode_seed": 42 + }, + "methods": [ + "ronpo_full_expect", + "ronpo_k_only", + "dpo", + "ipo", + "simpo", + "kto", + "sppo_avg", + "inpo_avg", + "ht_mnpo_helpfulness", + "ht_mnpo_safety", + "ht_mnpo_conciseness" + ], + "selection_policy": "No sealed-test selection. One preregistered configuration per method; failures use only the declared stability retry recipe, at most three attempts.", + "stability_gate": { + "validation_prompts": 128, + "min_candidate_base_mean_word_ratio": 0.33, + "max_candidate_base_mean_word_ratio": 2.0, + "max_consecutive_identical_word_run": 20, + "think_tag_count": 0, + "empty_response_count": 0, + "retry_recipe": "attempt 2 halves learning rate, enables gradient checkpointing, and adds reference=0.02/SFT=0.002 anchoring to unanchored unified losses; attempt 3 quarters learning rate and uses reference=0.05/SFT=0.005; RONPO uses reference=0.05/SFT=0.005 from attempt 1" + }, + "primary_metric": "mean prompt-level minimum of per-prompt min-max-normalized helpfulness, safety, conciseness", + "bootstrap": { + "unit": "prompt", + "paired": true, + "resamples": 2000, + "interval": "percentile_95", + "seed": 42 + } +} diff --git a/pair_manifest.json b/pair_manifest.json new file mode 100644 index 0000000..be034b0 --- /dev/null +++ b/pair_manifest.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "objectives": [ + "helpfulness", + "safety", + "conciseness" + ], + "normalization": "per_prompt_minmax", + "adversary": { + "steps": 25, + "alpha": 1.0, + "kappa": 0.05, + "preference_scale": 8.0 + }, + "ronpo_estimator": "Rao-Blackwellized full sigma expectation with three common response pairs per prompt", + "counts": { + "train": { + "avg": 16746, + "ronpo_full_expect": 50340, + "ronpo_k_only": 50340, + "ht_mnpo_helpfulness": 16743, + "ht_mnpo_safety": 16764, + "ht_mnpo_conciseness": 16755 + }, + "validation": { + "avg": 1859, + "ronpo_full_expect": 5592, + "ronpo_k_only": 5592, + "ht_mnpo_helpfulness": 1864, + "ht_mnpo_safety": 1862, + "ht_mnpo_conciseness": 1858 + } + } +} diff --git a/provenance.json b/provenance.json new file mode 100644 index 0000000..01b64d1 --- /dev/null +++ b/provenance.json @@ -0,0 +1,12 @@ +{ + "method": "dpo", + "seed": 43, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "5095e5d2222c", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/5095e5d2222c", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "stability_gate": "stability_gate.json" +} diff --git a/run_status.json b/run_status.json new file mode 100644 index 0000000..f997b10 --- /dev/null +++ b/run_status.json @@ -0,0 +1,13 @@ +{ + "method": "dpo", + "seed": 43, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "5095e5d2222c", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/5095e5d2222c", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "stability_gate": "stability_gate.json", + "status": "completed" +} diff --git a/split_manifest.json b/split_manifest.json new file mode 100644 index 0000000..4fb96bd --- /dev/null +++ b/split_manifest.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "split_rule": "sort source-train prompts by sha256('flagship-v1||'+prompt_id); first round(10%) validation", + "validation_fraction": 0.1, + "source_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/avg_oracle", + "base_model": "Qwen/Qwen3-8B", + "counts": { + "train": 16780, + "validation": 1864, + "sealed_test": 604 + }, + "prompt_id_sha256": { + "train": "fcf6f2ea5a0b7bcd41179e6f5288f4d850b2d04bdb729da91907bd18bf733a69", + "validation": "a87ba6f519192164daa487b109a6547dd8cbde46a27a373d6da201ca2d061b8f", + "sealed_test": "b398ced6af6cf73d731e1a5f32a0631d8060323d51eda1e375695566bf5f1867" + }, + "overlaps": { + "train_validation": 0, + "train_sealed_test": 0, + "validation_sealed_test": 0 + }, + "sealed_test_policy": "prompts only until S4a; no objective scores or model generations used in S0-S3" +} diff --git a/stability_gate.json b/stability_gate.json new file mode 100644 index 0000000..bb1f8fc --- /dev/null +++ b/stability_gate.json @@ -0,0 +1,38 @@ +{ + "status": "passed", + "passed": true, + "base": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 388.1484375, + "max_words": 1379, + "max_repeat_run": 4 + }, + "candidate": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 389.75, + "max_words": 1291, + "max_repeat_run": 4 + }, + "candidate_base_mean_word_ratio": 1.004126159853471, + "thresholds": { + "min_length_ratio": 0.33, + "max_length_ratio": 2.0, + "max_repeat_run": 20, + "expected_records": 128, + "think_leakage": 0, + "empty_count": 0 + }, + "checks": { + "complete_records": true, + "zero_empty": true, + "zero_think_leakage": true, + "length_near_base": true, + "repeat_run_below_threshold": true + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ffd19d2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..b711000 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8599068434252956, + "total_flos": 0.0, + "train_loss": 0.6902858657307095, + "train_runtime": 8330.5288, + "train_samples": 16746, + "train_samples_per_second": 1.729, + "train_steps_per_second": 0.108 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..2930302 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,2923 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8599068434252956, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.004777260241251642, + "grad_norm": 0.142578125, + "learning_rate": 2.222222222222222e-08, + "logits/chosen": -1.9159685373306274, + "logits/rejected": -1.9266211986541748, + "logps/chosen": -0.3117416799068451, + "logps/rejected": -0.2903628647327423, + "loss": 0.689449667930603, + "loss/core": 0.689449667930603, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.2060601711273193, + "rewards/margins": 1.513831377029419, + "rewards/rejected": 0.6922284960746765, + "step": 5 + }, + { + "epoch": 0.009554520482503284, + "grad_norm": 0.42578125, + "learning_rate": 5e-08, + "logits/chosen": -1.8667621612548828, + "logits/rejected": -1.8464374542236328, + "logps/chosen": -0.2869036793708801, + "logps/rejected": -0.3021157681941986, + "loss": 0.6924150586128235, + "loss/core": 0.6924150586128235, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.997037649154663, + "rewards/margins": 0.35158514976501465, + "rewards/rejected": 2.6454522609710693, + "step": 10 + }, + { + "epoch": 0.014331780723754926, + "grad_norm": 0.310546875, + "learning_rate": 7.777777777777778e-08, + "logits/chosen": -1.91054368019104, + "logits/rejected": -1.87662672996521, + "logps/chosen": -0.2590453028678894, + "logps/rejected": -0.27234840393066406, + "loss": 0.6913942098617554, + "loss/core": 0.6913942098617554, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.7586113214492798, + "rewards/margins": 0.7126884460449219, + "rewards/rejected": 1.045923113822937, + "step": 15 + }, + { + "epoch": 0.01910904096500657, + "grad_norm": 0.1298828125, + "learning_rate": 1.0555555555555555e-07, + "logits/chosen": -1.680210828781128, + "logits/rejected": -1.7510398626327515, + "logps/chosen": -0.27236178517341614, + "logps/rejected": -0.27088457345962524, + "loss": 0.687203049659729, + "loss/core": 0.687203049659729, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.647186279296875, + "rewards/margins": 2.474025011062622, + "rewards/rejected": 1.173161268234253, + "step": 20 + }, + { + "epoch": 0.02388630120625821, + "grad_norm": 0.1787109375, + "learning_rate": 1.3333333333333334e-07, + "logits/chosen": -1.7065041065216064, + "logits/rejected": -1.7718740701675415, + "logps/chosen": -0.27959316968917847, + "logps/rejected": -0.27071651816368103, + "loss": 0.6902543902397156, + "loss/core": 0.6902543902397156, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 2.143491744995117, + "rewards/margins": 1.1622021198272705, + "rewards/rejected": 0.9812896847724915, + "step": 25 + }, + { + "epoch": 0.028663561447509853, + "grad_norm": 0.12890625, + "learning_rate": 1.611111111111111e-07, + "logits/chosen": -1.9748786687850952, + "logits/rejected": -1.9123010635375977, + "logps/chosen": -0.2816716730594635, + "logps/rejected": -0.2625161409378052, + "loss": 0.688904345035553, + "loss/core": 0.688904345035553, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.795006275177002, + "rewards/margins": 1.7435691356658936, + "rewards/rejected": 1.0514370203018188, + "step": 30 + }, + { + "epoch": 0.033440821688761495, + "grad_norm": 0.0830078125, + "learning_rate": 1.8888888888888888e-07, + "logits/chosen": -1.93292236328125, + "logits/rejected": -1.9241135120391846, + "logps/chosen": -0.29125192761421204, + "logps/rejected": -0.2757991552352905, + "loss": 0.6889916658401489, + "loss/core": 0.6889916658401489, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.532140016555786, + "rewards/margins": 1.7434587478637695, + "rewards/rejected": 0.7886813879013062, + "step": 35 + }, + { + "epoch": 0.03821808193001314, + "grad_norm": 0.265625, + "learning_rate": 2.1666666666666667e-07, + "logits/chosen": -1.8440186977386475, + "logits/rejected": -1.928789734840393, + "logps/chosen": -0.266449898481369, + "logps/rejected": -0.2915545105934143, + "loss": 0.6877603530883789, + "loss/core": 0.6877603530883789, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.920603036880493, + "rewards/margins": 2.235374927520752, + "rewards/rejected": 0.6852277517318726, + "step": 40 + }, + { + "epoch": 0.04299534217126478, + "grad_norm": 0.162109375, + "learning_rate": 2.4444444444444445e-07, + "logits/chosen": -1.8754253387451172, + "logits/rejected": -1.8980233669281006, + "logps/chosen": -0.26845940947532654, + "logps/rejected": -0.2764447033405304, + "loss": 0.6887972950935364, + "loss/core": 0.6887972950935364, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 3.0673203468322754, + "rewards/margins": 1.7863553762435913, + "rewards/rejected": 1.280964970588684, + "step": 45 + }, + { + "epoch": 0.04777260241251642, + "grad_norm": 0.5546875, + "learning_rate": 2.7222222222222216e-07, + "logits/chosen": -1.7663570642471313, + "logits/rejected": -1.810603380203247, + "logps/chosen": -0.2882184386253357, + "logps/rejected": -0.26946529746055603, + "loss": 0.6920673847198486, + "loss/core": 0.6920673847198486, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.6537147760391235, + "rewards/margins": 0.43855220079421997, + "rewards/rejected": 1.2151625156402588, + "step": 50 + }, + { + "epoch": 0.05254986265376806, + "grad_norm": 0.59375, + "learning_rate": 3e-07, + "logits/chosen": -1.8167644739151, + "logits/rejected": -1.847123146057129, + "logps/chosen": -0.28940513730049133, + "logps/rejected": -0.28783121705055237, + "loss": 0.6892070174217224, + "loss/core": 0.6892070174217224, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.3337934017181396, + "rewards/margins": 1.6257107257843018, + "rewards/rejected": 0.7080828547477722, + "step": 55 + }, + { + "epoch": 0.057327122895019705, + "grad_norm": 0.5703125, + "learning_rate": 3.2777777777777776e-07, + "logits/chosen": -1.7945712804794312, + "logits/rejected": -1.765467882156372, + "logps/chosen": -0.2750927805900574, + "logps/rejected": -0.277031809091568, + "loss": 0.6882904171943665, + "loss/core": 0.6882904171943665, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.7785580158233643, + "rewards/margins": 2.080836772918701, + "rewards/rejected": 1.6977211236953735, + "step": 60 + }, + { + "epoch": 0.06210438313627135, + "grad_norm": 0.7890625, + "learning_rate": 3.5555555555555553e-07, + "logits/chosen": -1.7736009359359741, + "logits/rejected": -1.8232080936431885, + "logps/chosen": -0.2970460057258606, + "logps/rejected": -0.2842680811882019, + "loss": 0.6880653500556946, + "loss/core": 0.6880653500556946, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.2521681785583496, + "rewards/margins": 2.1032259464263916, + "rewards/rejected": 1.148942232131958, + "step": 65 + }, + { + "epoch": 0.06688164337752299, + "grad_norm": 0.224609375, + "learning_rate": 3.8333333333333335e-07, + "logits/chosen": -1.8815971612930298, + "logits/rejected": -1.8222999572753906, + "logps/chosen": -0.27899742126464844, + "logps/rejected": -0.27459582686424255, + "loss": 0.6906721591949463, + "loss/core": 0.6906721591949463, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.500136613845825, + "rewards/margins": 1.0100083351135254, + "rewards/rejected": 1.4901282787322998, + "step": 70 + }, + { + "epoch": 0.07165890361877464, + "grad_norm": 0.109375, + "learning_rate": 4.1111111111111107e-07, + "logits/chosen": -1.6210107803344727, + "logits/rejected": -1.6766055822372437, + "logps/chosen": -0.30045056343078613, + "logps/rejected": -0.3084052801132202, + "loss": 0.6901770830154419, + "loss/core": 0.6901770830154419, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.262094020843506, + "rewards/margins": 1.2137267589569092, + "rewards/rejected": 1.0483672618865967, + "step": 75 + }, + { + "epoch": 0.07643616386002627, + "grad_norm": 0.07373046875, + "learning_rate": 4.3888888888888884e-07, + "logits/chosen": -1.8754079341888428, + "logits/rejected": -1.9132652282714844, + "logps/chosen": -0.29582279920578003, + "logps/rejected": -0.28720319271087646, + "loss": 0.6921669244766235, + "loss/core": 0.6921669244766235, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.3508808612823486, + "rewards/margins": 0.3969760835170746, + "rewards/rejected": 0.9539046287536621, + "step": 80 + }, + { + "epoch": 0.08121342410127792, + "grad_norm": 1.8515625, + "learning_rate": 4.6666666666666666e-07, + "logits/chosen": -2.0209007263183594, + "logits/rejected": -2.061509132385254, + "logps/chosen": -0.27408546209335327, + "logps/rejected": -0.2787121832370758, + "loss": 0.6931945085525513, + "loss/core": 0.6931945085525513, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.682008147239685, + "rewards/margins": 0.032013483345508575, + "rewards/rejected": 1.6499946117401123, + "step": 85 + }, + { + "epoch": 0.08599068434252956, + "grad_norm": 0.287109375, + "learning_rate": 4.944444444444445e-07, + "logits/chosen": -1.8627064228057861, + "logits/rejected": -1.919704794883728, + "logps/chosen": -0.26979389786720276, + "logps/rejected": -0.3131831884384155, + "loss": 0.6907284259796143, + "loss/core": 0.6907284259796143, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.5674381256103516, + "rewards/margins": 1.0534178018569946, + "rewards/rejected": 1.5140199661254883, + "step": 90 + }, + { + "epoch": 0.09076794458378121, + "grad_norm": 0.4375, + "learning_rate": 4.999699149323369e-07, + "logits/chosen": -1.7471721172332764, + "logits/rejected": -1.6695648431777954, + "logps/chosen": -0.29016250371932983, + "logps/rejected": -0.2957083284854889, + "loss": 0.6901000142097473, + "loss/core": 0.6901000142097473, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.2820587158203125, + "rewards/margins": 1.2528122663497925, + "rewards/rejected": 1.029246211051941, + "step": 95 + }, + { + "epoch": 0.09554520482503284, + "grad_norm": 0.1552734375, + "learning_rate": 4.998477067547739e-07, + "logits/chosen": -1.9470646381378174, + "logits/rejected": -1.9981377124786377, + "logps/chosen": -0.2694973051548004, + "logps/rejected": -0.281091570854187, + "loss": 0.6881608963012695, + "loss/core": 0.6881608963012695, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.707597017288208, + "rewards/margins": 2.0738112926483154, + "rewards/rejected": 1.633785605430603, + "step": 100 + }, + { + "epoch": 0.10032246506628449, + "grad_norm": 0.265625, + "learning_rate": 4.996315410727229e-07, + "logits/chosen": -1.66718327999115, + "logits/rejected": -1.654313087463379, + "logps/chosen": -0.3189978301525116, + "logps/rejected": -0.30316898226737976, + "loss": 0.6870630979537964, + "loss/core": 0.6870630979537964, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.3393523693084717, + "rewards/margins": 2.53385329246521, + "rewards/rejected": 0.805498480796814, + "step": 105 + }, + { + "epoch": 0.10509972530753613, + "grad_norm": 0.109375, + "learning_rate": 4.993214991772562e-07, + "logits/chosen": -1.7715818881988525, + "logits/rejected": -1.7875267267227173, + "logps/chosen": -0.2973600924015045, + "logps/rejected": -0.3144645392894745, + "loss": 0.6913233399391174, + "loss/core": 0.6913233399391174, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.6638648509979248, + "rewards/margins": 0.7346082925796509, + "rewards/rejected": 0.9292564392089844, + "step": 110 + }, + { + "epoch": 0.10987698554878778, + "grad_norm": 0.2216796875, + "learning_rate": 4.989176976624511e-07, + "logits/chosen": -1.8163177967071533, + "logits/rejected": -1.8914730548858643, + "logps/chosen": -0.30195850133895874, + "logps/rejected": -0.2931588590145111, + "loss": 0.6890204548835754, + "loss/core": 0.6890204548835754, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.8905234336853027, + "rewards/margins": 1.713559865951538, + "rewards/rejected": 1.1769633293151855, + "step": 115 + }, + { + "epoch": 0.11465424579003941, + "grad_norm": 0.52734375, + "learning_rate": 4.984202883815428e-07, + "logits/chosen": -1.784044861793518, + "logits/rejected": -1.784518837928772, + "logps/chosen": -0.30433687567710876, + "logps/rejected": -0.30718523263931274, + "loss": 0.6902703642845154, + "loss/core": 0.6902703642845154, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.3598580360412598, + "rewards/margins": 1.1782281398773193, + "rewards/rejected": 1.18163001537323, + "step": 120 + }, + { + "epoch": 0.11943150603129106, + "grad_norm": 0.1533203125, + "learning_rate": 4.978294583898195e-07, + "logits/chosen": -1.6721748113632202, + "logits/rejected": -1.5910634994506836, + "logps/chosen": -0.29361215233802795, + "logps/rejected": -0.3014775812625885, + "loss": 0.6942087411880493, + "loss/core": 0.6942087411880493, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.8780733346939087, + "rewards/margins": -0.34356021881103516, + "rewards/rejected": 2.2216334342956543, + "step": 125 + }, + { + "epoch": 0.1242087662725427, + "grad_norm": 0.435546875, + "learning_rate": 4.971454298742779e-07, + "logits/chosen": -2.0185725688934326, + "logits/rejected": -1.9341987371444702, + "logps/chosen": -0.27574944496154785, + "logps/rejected": -0.2965439558029175, + "loss": 0.6927915811538696, + "loss/core": 0.6927915811538696, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.2239298820495605, + "rewards/margins": 0.20492222905158997, + "rewards/rejected": 2.019007682800293, + "step": 130 + }, + { + "epoch": 0.12898602651379434, + "grad_norm": 0.267578125, + "learning_rate": 4.963684600700678e-07, + "logits/chosen": -1.713491439819336, + "logits/rejected": -1.7976768016815186, + "logps/chosen": -0.345987468957901, + "logps/rejected": -0.3146022856235504, + "loss": 0.6902076005935669, + "loss/core": 0.6902076005935669, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.6916372776031494, + "rewards/margins": 1.2287349700927734, + "rewards/rejected": 1.4629024267196655, + "step": 135 + }, + { + "epoch": 0.13376328675504598, + "grad_norm": 0.37890625, + "learning_rate": 4.954988411637571e-07, + "logits/chosen": -1.644727349281311, + "logits/rejected": -1.6644747257232666, + "logps/chosen": -0.33096668124198914, + "logps/rejected": -0.34381940960884094, + "loss": 0.6915746927261353, + "loss/core": 0.6915746927261353, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.1107561588287354, + "rewards/margins": 0.7399330735206604, + "rewards/rejected": 2.3708226680755615, + "step": 140 + }, + { + "epoch": 0.13854054699629761, + "grad_norm": 0.19140625, + "learning_rate": 4.945369001834514e-07, + "logits/chosen": -1.6278400421142578, + "logits/rejected": -1.64529287815094, + "logps/chosen": -0.3012077808380127, + "logps/rejected": -0.28633007407188416, + "loss": 0.690815269947052, + "loss/core": 0.690815269947052, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.7844654321670532, + "rewards/margins": 0.944085419178009, + "rewards/rejected": 0.840380072593689, + "step": 145 + }, + { + "epoch": 0.14331780723754928, + "grad_norm": 0.5859375, + "learning_rate": 4.93482998875813e-07, + "logits/chosen": -1.5810550451278687, + "logits/rejected": -1.5913165807724, + "logps/chosen": -0.28546205163002014, + "logps/rejected": -0.27281785011291504, + "loss": 0.6898037195205688, + "loss/core": 0.6898037195205688, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.6366374492645264, + "rewards/margins": 1.3896093368530273, + "rewards/rejected": 1.247028112411499, + "step": 150 + }, + { + "epoch": 0.1480950674788009, + "grad_norm": 0.1923828125, + "learning_rate": 4.923375335700223e-07, + "logits/chosen": -1.5124685764312744, + "logits/rejected": -1.6061471700668335, + "logps/chosen": -0.2498733252286911, + "logps/rejected": -0.26588135957717896, + "loss": 0.6902016997337341, + "loss/core": 0.6902016997337341, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.6172432899475098, + "rewards/margins": 1.1884584426879883, + "rewards/rejected": 1.4287846088409424, + "step": 155 + }, + { + "epoch": 0.15287232772005255, + "grad_norm": 0.388671875, + "learning_rate": 4.911009350287347e-07, + "logits/chosen": -1.886030912399292, + "logits/rejected": -1.8260345458984375, + "logps/chosen": -0.2788711190223694, + "logps/rejected": -0.2940537631511688, + "loss": 0.6895097494125366, + "loss/core": 0.6895097494125366, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.654872417449951, + "rewards/margins": 1.5049922466278076, + "rewards/rejected": 1.149880290031433, + "step": 160 + }, + { + "epoch": 0.15764958796130418, + "grad_norm": 0.15234375, + "learning_rate": 4.897736682860885e-07, + "logits/chosen": -1.7723588943481445, + "logits/rejected": -1.740827202796936, + "logps/chosen": -0.307345986366272, + "logps/rejected": -0.30495208501815796, + "loss": 0.6911855340003967, + "loss/core": 0.6911855340003967, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.7815402746200562, + "rewards/margins": 0.7892364859580994, + "rewards/rejected": 0.9923038482666016, + "step": 165 + }, + { + "epoch": 0.16242684820255585, + "grad_norm": 0.08642578125, + "learning_rate": 4.883562324728241e-07, + "logits/chosen": -1.828007459640503, + "logits/rejected": -1.7083508968353271, + "logps/chosen": -0.3094002306461334, + "logps/rejected": -0.32762759923934937, + "loss": 0.6900734901428223, + "loss/core": 0.6900734901428223, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.83378005027771, + "rewards/margins": 1.247715950012207, + "rewards/rejected": 0.586064338684082, + "step": 170 + }, + { + "epoch": 0.16720410844380748, + "grad_norm": 0.466796875, + "learning_rate": 4.868491606285823e-07, + "logits/chosen": -1.920536994934082, + "logits/rejected": -1.9467909336090088, + "logps/chosen": -0.2732824683189392, + "logps/rejected": -0.2819834053516388, + "loss": 0.6904429197311401, + "loss/core": 0.6904429197311401, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.7944914102554321, + "rewards/margins": 1.0950840711593628, + "rewards/rejected": 0.699407160282135, + "step": 175 + }, + { + "epoch": 0.17198136868505912, + "grad_norm": 0.189453125, + "learning_rate": 4.852530195014489e-07, + "logits/chosen": -1.8890149593353271, + "logits/rejected": -1.8630263805389404, + "logps/chosen": -0.2616122364997864, + "logps/rejected": -0.2847253978252411, + "loss": 0.6903570294380188, + "loss/core": 0.6903570294380188, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.4658520221710205, + "rewards/margins": 1.127223014831543, + "rewards/rejected": 1.3386290073394775, + "step": 180 + }, + { + "epoch": 0.17675862892631075, + "grad_norm": 1.53125, + "learning_rate": 4.835684093348244e-07, + "logits/chosen": -1.936354398727417, + "logits/rejected": -1.9784005880355835, + "logps/chosen": -0.30487221479415894, + "logps/rejected": -0.30455270409584045, + "loss": 0.6924332976341248, + "loss/core": 0.6924332976341248, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.8872802257537842, + "rewards/margins": 0.34668928384780884, + "rewards/rejected": 1.5405908823013306, + "step": 185 + }, + { + "epoch": 0.18153588916756241, + "grad_norm": 1.34375, + "learning_rate": 4.817959636416969e-07, + "logits/chosen": -1.8787858486175537, + "logits/rejected": -1.877399206161499, + "logps/chosen": -0.28796452283859253, + "logps/rejected": -0.2709272801876068, + "loss": 0.6896687746047974, + "loss/core": 0.6896687746047974, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.630368709564209, + "rewards/margins": 1.4462611675262451, + "rewards/rejected": 1.1841075420379639, + "step": 190 + }, + { + "epoch": 0.18631314940881405, + "grad_norm": 0.140625, + "learning_rate": 4.799363489664039e-07, + "logits/chosen": -1.689291000366211, + "logits/rejected": -1.709299087524414, + "logps/chosen": -0.2852824032306671, + "logps/rejected": -0.29218631982803345, + "loss": 0.692372739315033, + "loss/core": 0.692372739315033, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.6775140762329102, + "rewards/margins": 0.32127708196640015, + "rewards/rejected": 1.3562370538711548, + "step": 195 + }, + { + "epoch": 0.19109040965006568, + "grad_norm": 0.25390625, + "learning_rate": 4.779902646339721e-07, + "logits/chosen": -1.7467161417007446, + "logits/rejected": -1.8196022510528564, + "logps/chosen": -0.27380579710006714, + "logps/rejected": -0.27410024404525757, + "loss": 0.6902837157249451, + "loss/core": 0.6902837157249451, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.508232593536377, + "rewards/margins": 1.1675052642822266, + "rewards/rejected": 1.3407273292541504, + "step": 200 + }, + { + "epoch": 0.19586766989131732, + "grad_norm": 0.1318359375, + "learning_rate": 4.759584424871301e-07, + "logits/chosen": -1.8901665210723877, + "logits/rejected": -1.9521496295928955, + "logps/chosen": -0.28199759125709534, + "logps/rejected": -0.29019278287887573, + "loss": 0.6913297772407532, + "loss/core": 0.6913297772407532, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.3713281154632568, + "rewards/margins": 0.7334555983543396, + "rewards/rejected": 0.6378724575042725, + "step": 205 + }, + { + "epoch": 0.20064493013256898, + "grad_norm": 0.2255859375, + "learning_rate": 4.738416466110917e-07, + "logits/chosen": -1.899572730064392, + "logits/rejected": -1.8477237224578857, + "logps/chosen": -0.28993353247642517, + "logps/rejected": -0.28719550371170044, + "loss": 0.6906791925430298, + "loss/core": 0.6906791925430298, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.508605480194092, + "rewards/margins": 0.9953238368034363, + "rewards/rejected": 1.5132817029953003, + "step": 210 + }, + { + "epoch": 0.20542219037382062, + "grad_norm": 0.1396484375, + "learning_rate": 4.716406730462153e-07, + "logits/chosen": -1.7165281772613525, + "logits/rejected": -1.7286529541015625, + "logps/chosen": -0.2964112162590027, + "logps/rejected": -0.2740923762321472, + "loss": 0.6906309723854065, + "loss/core": 0.6906309723854065, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.0557491779327393, + "rewards/margins": 1.0147982835769653, + "rewards/rejected": 1.0409510135650635, + "step": 215 + }, + { + "epoch": 0.21019945061507225, + "grad_norm": 0.123046875, + "learning_rate": 4.693563494886454e-07, + "logits/chosen": -1.7657253742218018, + "logits/rejected": -1.7964322566986084, + "logps/chosen": -0.2821979224681854, + "logps/rejected": -0.2735520005226135, + "loss": 0.6908767819404602, + "loss/core": 0.6908767819404602, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.770012617111206, + "rewards/margins": 0.9165030717849731, + "rewards/rejected": 0.8535095453262329, + "step": 220 + }, + { + "epoch": 0.2149767108563239, + "grad_norm": 0.265625, + "learning_rate": 4.6698953497905016e-07, + "logits/chosen": -1.840344786643982, + "logits/rejected": -1.8655697107315063, + "logps/chosen": -0.3119342029094696, + "logps/rejected": -0.30928319692611694, + "loss": 0.6905477643013, + "loss/core": 0.6905477643013, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.063857078552246, + "rewards/margins": 1.085402488708496, + "rewards/rejected": 1.97845458984375, + "step": 225 + }, + { + "epoch": 0.21975397109757555, + "grad_norm": 0.287109375, + "learning_rate": 4.645411195795709e-07, + "logits/chosen": -1.8058847188949585, + "logits/rejected": -1.7770591974258423, + "logps/chosen": -0.2736867368221283, + "logps/rejected": -0.2652318775653839, + "loss": 0.6898475289344788, + "loss/core": 0.6898475289344788, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.0367789268493652, + "rewards/margins": 1.344928503036499, + "rewards/rejected": 1.6918503046035767, + "step": 230 + }, + { + "epoch": 0.2245312313388272, + "grad_norm": 0.1015625, + "learning_rate": 4.6201202403910643e-07, + "logits/chosen": -1.8290420770645142, + "logits/rejected": -1.9100230932235718, + "logps/chosen": -0.2853394150733948, + "logps/rejected": -0.2796414792537689, + "loss": 0.6906234622001648, + "loss/core": 0.6906234622001648, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.7059528827667236, + "rewards/margins": 1.020578145980835, + "rewards/rejected": 0.6853744387626648, + "step": 235 + }, + { + "epoch": 0.22930849158007882, + "grad_norm": 0.1953125, + "learning_rate": 4.594031994470573e-07, + "logits/chosen": -1.8122670650482178, + "logits/rejected": -1.8503730297088623, + "logps/chosen": -0.2719040811061859, + "logps/rejected": -0.28884705901145935, + "loss": 0.6911881566047668, + "loss/core": 0.6911881566047668, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.7802484035491943, + "rewards/margins": 0.8196385502815247, + "rewards/rejected": 1.960609793663025, + "step": 240 + }, + { + "epoch": 0.23408575182133046, + "grad_norm": 1.609375, + "learning_rate": 4.567156268756593e-07, + "logits/chosen": -1.7871983051300049, + "logits/rejected": -1.8605146408081055, + "logps/chosen": -0.3058835566043854, + "logps/rejected": -0.29272013902664185, + "loss": 0.6895998120307922, + "loss/core": 0.6895998120307922, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 2.141623020172119, + "rewards/margins": 1.4836769104003906, + "rewards/rejected": 0.6579458117485046, + "step": 245 + }, + { + "epoch": 0.23886301206258212, + "grad_norm": 0.1728515625, + "learning_rate": 4.5395031701104303e-07, + "logits/chosen": -1.5382251739501953, + "logits/rejected": -1.5597591400146484, + "logps/chosen": -0.3067648708820343, + "logps/rejected": -0.30108946561813354, + "loss": 0.6889725923538208, + "loss/core": 0.6889725923538208, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.3333942890167236, + "rewards/margins": 1.7330058813095093, + "rewards/rejected": 1.6003879308700562, + "step": 250 + }, + { + "epoch": 0.24364027230383375, + "grad_norm": 0.1123046875, + "learning_rate": 4.511083097731555e-07, + "logits/chosen": -1.8064721822738647, + "logits/rejected": -1.7805678844451904, + "logps/chosen": -0.28534120321273804, + "logps/rejected": -0.29203271865844727, + "loss": 0.6893185973167419, + "loss/core": 0.6893185973167419, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.7705674171447754, + "rewards/margins": 1.617296576499939, + "rewards/rejected": 1.1532708406448364, + "step": 255 + }, + { + "epoch": 0.2484175325450854, + "grad_norm": 0.57421875, + "learning_rate": 4.481906739246894e-07, + "logits/chosen": -1.767867088317871, + "logits/rejected": -1.7515289783477783, + "logps/chosen": -0.30535268783569336, + "logps/rejected": -0.3085159361362457, + "loss": 0.6914692521095276, + "loss/core": 0.6914692521095276, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9446821212768555, + "rewards/margins": 0.6940180063247681, + "rewards/rejected": 1.2506643533706665, + "step": 260 + }, + { + "epoch": 0.25319479278633705, + "grad_norm": 0.1357421875, + "learning_rate": 4.451985066691648e-07, + "logits/chosen": -1.7518078088760376, + "logits/rejected": -1.70755136013031, + "logps/chosen": -0.27699360251426697, + "logps/rejected": -0.28743523359298706, + "loss": 0.6904750466346741, + "loss/core": 0.6904750466346741, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.121932029724121, + "rewards/margins": 1.081810474395752, + "rewards/rejected": 1.0401216745376587, + "step": 265 + }, + { + "epoch": 0.2579720530275887, + "grad_norm": 0.08837890625, + "learning_rate": 4.421329332383158e-07, + "logits/chosen": -1.7966949939727783, + "logits/rejected": -1.7754967212677002, + "logps/chosen": -0.3063514232635498, + "logps/rejected": -0.3063802719116211, + "loss": 0.6925251483917236, + "loss/core": 0.6925251483917236, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.6563818454742432, + "rewards/margins": 0.26670390367507935, + "rewards/rejected": 1.389677882194519, + "step": 270 + }, + { + "epoch": 0.2627493132688403, + "grad_norm": 0.1552734375, + "learning_rate": 4.3899510646893696e-07, + "logits/chosen": -1.8795068264007568, + "logits/rejected": -1.963724136352539, + "logps/chosen": -0.29647111892700195, + "logps/rejected": -0.2953276038169861, + "loss": 0.6920241117477417, + "loss/core": 0.6920241117477417, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.034947156906128, + "rewards/margins": 0.4514775276184082, + "rewards/rejected": 0.5834697484970093, + "step": 275 + }, + { + "epoch": 0.26752657351009196, + "grad_norm": 0.3046875, + "learning_rate": 4.357862063693485e-07, + "logits/chosen": -1.7779220342636108, + "logits/rejected": -1.7593568563461304, + "logps/chosen": -0.30200910568237305, + "logps/rejected": -0.30040618777275085, + "loss": 0.6907645463943481, + "loss/core": 0.6907645463943481, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.8066829442977905, + "rewards/margins": 0.9670276641845703, + "rewards/rejected": 0.8396552801132202, + "step": 280 + }, + { + "epoch": 0.2723038337513436, + "grad_norm": 0.185546875, + "learning_rate": 4.3250743967564364e-07, + "logits/chosen": -1.9163156747817993, + "logits/rejected": -1.8812675476074219, + "logps/chosen": -0.2867981791496277, + "logps/rejected": -0.2877008616924286, + "loss": 0.6907505989074707, + "loss/core": 0.6907505989074707, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.9703712463378906, + "rewards/margins": 1.0003026723861694, + "rewards/rejected": 1.9700685739517212, + "step": 285 + }, + { + "epoch": 0.27708109399259523, + "grad_norm": 1.5625, + "learning_rate": 4.29160039397884e-07, + "logits/chosen": -1.8704643249511719, + "logits/rejected": -1.9002145528793335, + "logps/chosen": -0.2817132771015167, + "logps/rejected": -0.27127501368522644, + "loss": 0.6853945851325989, + "loss/core": 0.6853945851325989, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.546133041381836, + "rewards/margins": 3.254213809967041, + "rewards/rejected": 1.2919186353683472, + "step": 290 + }, + { + "epoch": 0.28185835423384686, + "grad_norm": 0.26953125, + "learning_rate": 4.2574526435641546e-07, + "logits/chosen": -1.634866714477539, + "logits/rejected": -1.7080461978912354, + "logps/chosen": -0.31140029430389404, + "logps/rejected": -0.3021450638771057, + "loss": 0.6891725659370422, + "loss/core": 0.6891725659370422, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.7522711753845215, + "rewards/margins": 1.6253259181976318, + "rewards/rejected": 1.1269454956054688, + "step": 295 + }, + { + "epoch": 0.28663561447509855, + "grad_norm": 0.12353515625, + "learning_rate": 4.22264398708477e-07, + "logits/chosen": -1.887028455734253, + "logits/rejected": -1.9402997493743896, + "logps/chosen": -0.2800259292125702, + "logps/rejected": -0.3047648072242737, + "loss": 0.6893907785415649, + "loss/core": 0.6893907785415649, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.504063844680786, + "rewards/margins": 1.5716952085494995, + "rewards/rejected": 1.9323689937591553, + "step": 300 + }, + { + "epoch": 0.2914128747163502, + "grad_norm": 0.8984375, + "learning_rate": 4.187187514652819e-07, + "logits/chosen": -1.5624942779541016, + "logits/rejected": -1.5863012075424194, + "logps/chosen": -0.30193594098091125, + "logps/rejected": -0.3065473437309265, + "loss": 0.690039336681366, + "loss/core": 0.690039336681366, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.7968106269836426, + "rewards/margins": 1.314178228378296, + "rewards/rejected": 2.4826323986053467, + "step": 305 + }, + { + "epoch": 0.2961901349576018, + "grad_norm": 0.486328125, + "learning_rate": 4.151096559997519e-07, + "logits/chosen": -1.7193517684936523, + "logits/rejected": -1.706871747970581, + "logps/chosen": -0.277077317237854, + "logps/rejected": -0.27818840742111206, + "loss": 0.6909591555595398, + "loss/core": 0.6909591555595398, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.3378546237945557, + "rewards/margins": 0.8950866460800171, + "rewards/rejected": 1.4427679777145386, + "step": 310 + }, + { + "epoch": 0.30096739519885346, + "grad_norm": 0.24609375, + "learning_rate": 4.114384695450905e-07, + "logits/chosen": -1.7371066808700562, + "logits/rejected": -1.7233482599258423, + "logps/chosen": -0.3089512884616852, + "logps/rejected": -0.3026643395423889, + "loss": 0.6932816505432129, + "loss/core": 0.6932816505432129, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.1888813972473145, + "rewards/margins": -0.031425733119249344, + "rewards/rejected": 1.2203071117401123, + "step": 315 + }, + { + "epoch": 0.3057446554401051, + "grad_norm": 1.984375, + "learning_rate": 4.077065726843828e-07, + "logits/chosen": -1.4896411895751953, + "logits/rejected": -1.460703730583191, + "logps/chosen": -0.3152364492416382, + "logps/rejected": -0.31800299882888794, + "loss": 0.6897588968276978, + "loss/core": 0.6897588968276978, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.180734634399414, + "rewards/margins": 1.4465312957763672, + "rewards/rejected": 1.7342031002044678, + "step": 320 + }, + { + "epoch": 0.31052191568135673, + "grad_norm": 0.09619140625, + "learning_rate": 4.039153688314145e-07, + "logits/chosen": -1.7158210277557373, + "logits/rejected": -1.7079627513885498, + "logps/chosen": -0.28858762979507446, + "logps/rejected": -0.2933047413825989, + "loss": 0.6915832757949829, + "loss/core": 0.6915832757949829, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.429247260093689, + "rewards/margins": 0.630083441734314, + "rewards/rejected": 0.799163818359375, + "step": 325 + }, + { + "epoch": 0.31529917592260837, + "grad_norm": 2.671875, + "learning_rate": 4.0006628370290613e-07, + "logits/chosen": -1.802638053894043, + "logits/rejected": -1.7583692073822021, + "logps/chosen": -0.31713518500328064, + "logps/rejected": -0.3377764821052551, + "loss": 0.6928405165672302, + "loss/core": 0.6928405165672302, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.3360774517059326, + "rewards/margins": 0.1572122424840927, + "rewards/rejected": 1.1788651943206787, + "step": 330 + }, + { + "epoch": 0.32007643616386, + "grad_norm": 0.1533203125, + "learning_rate": 3.9616076478235826e-07, + "logits/chosen": -1.9015684127807617, + "logits/rejected": -1.9405295848846436, + "logps/chosen": -0.3054332733154297, + "logps/rejected": -0.3069126307964325, + "loss": 0.6918673515319824, + "loss/core": 0.6918673515319824, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.0781052112579346, + "rewards/margins": 0.5133062601089478, + "rewards/rejected": 0.5647989511489868, + "step": 335 + }, + { + "epoch": 0.3248536964051117, + "grad_norm": 0.1416015625, + "learning_rate": 3.922002807757129e-07, + "logits/chosen": -1.830971360206604, + "logits/rejected": -1.8372182846069336, + "logps/chosen": -0.28015151619911194, + "logps/rejected": -0.2828627824783325, + "loss": 0.6899048686027527, + "loss/core": 0.6899048686027527, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.2420430183410645, + "rewards/margins": 1.3439006805419922, + "rewards/rejected": 0.8981423377990723, + "step": 340 + }, + { + "epoch": 0.3296309566463633, + "grad_norm": 0.61328125, + "learning_rate": 3.8818632105903315e-07, + "logits/chosen": -1.8057060241699219, + "logits/rejected": -1.802642822265625, + "logps/chosen": -0.26754963397979736, + "logps/rejected": -0.273385614156723, + "loss": 0.69206303358078, + "loss/core": 0.69206303358078, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.1810123920440674, + "rewards/margins": 0.46981877088546753, + "rewards/rejected": 1.7111934423446655, + "step": 345 + }, + { + "epoch": 0.33440821688761496, + "grad_norm": 0.1796875, + "learning_rate": 3.841203951184094e-07, + "logits/chosen": -1.942135214805603, + "logits/rejected": -1.9843320846557617, + "logps/chosen": -0.2976142466068268, + "logps/rejected": -0.30821579694747925, + "loss": 0.6909466981887817, + "loss/core": 0.6909466981887817, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.338165283203125, + "rewards/margins": 0.9373272657394409, + "rewards/rejected": 1.4008381366729736, + "step": 350 + }, + { + "epoch": 0.3391854771288666, + "grad_norm": 0.302734375, + "learning_rate": 3.800040319823038e-07, + "logits/chosen": -1.8573143482208252, + "logits/rejected": -1.8273032903671265, + "logps/chosen": -0.28388920426368713, + "logps/rejected": -0.2705972492694855, + "loss": 0.6890053749084473, + "loss/core": 0.6890053749084473, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.5264713764190674, + "rewards/margins": 1.7132002115249634, + "rewards/rejected": 1.8132712841033936, + "step": 355 + }, + { + "epoch": 0.34396273737011823, + "grad_norm": 0.921875, + "learning_rate": 3.75838779646545e-07, + "logits/chosen": -1.7900562286376953, + "logits/rejected": -1.820005178451538, + "logps/chosen": -0.2741461396217346, + "logps/rejected": -0.27509093284606934, + "loss": 0.6885601282119751, + "loss/core": 0.6885601282119751, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.2590813636779785, + "rewards/margins": 1.9007847309112549, + "rewards/rejected": 1.3582968711853027, + "step": 360 + }, + { + "epoch": 0.34873999761136987, + "grad_norm": 0.4140625, + "learning_rate": 3.7162620449218993e-07, + "logits/chosen": -1.9214216470718384, + "logits/rejected": -1.9572467803955078, + "logps/chosen": -0.30636709928512573, + "logps/rejected": -0.30526402592658997, + "loss": 0.6896119713783264, + "loss/core": 0.6896119713783264, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.470032215118408, + "rewards/margins": 1.4553961753845215, + "rewards/rejected": 1.0146360397338867, + "step": 365 + }, + { + "epoch": 0.3535172578526215, + "grad_norm": 0.1845703125, + "learning_rate": 3.673678906964727e-07, + "logits/chosen": -2.049393653869629, + "logits/rejected": -1.9477535486221313, + "logps/chosen": -0.2906716465950012, + "logps/rejected": -0.3007729649543762, + "loss": 0.6913913488388062, + "loss/core": 0.6913913488388062, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.4209463596343994, + "rewards/margins": 0.750097393989563, + "rewards/rejected": 1.6708488464355469, + "step": 370 + }, + { + "epoch": 0.35829451809387314, + "grad_norm": 0.1591796875, + "learning_rate": 3.6306543963705936e-07, + "logits/chosen": -1.738469123840332, + "logits/rejected": -1.6882975101470947, + "logps/chosen": -0.2867928743362427, + "logps/rejected": -0.28549641370773315, + "loss": 0.6875568628311157, + "loss/core": 0.6875568628311157, + "rewards/accuracies": 0.8125, + "rewards/chosen": 3.7997069358825684, + "rewards/margins": 2.3964896202087402, + "rewards/rejected": 1.4032175540924072, + "step": 375 + }, + { + "epoch": 0.36307177833512483, + "grad_norm": 0.74609375, + "learning_rate": 3.5872046928983623e-07, + "logits/chosen": -1.6411170959472656, + "logits/rejected": -1.74252450466156, + "logps/chosen": -0.28902941942214966, + "logps/rejected": -0.30169445276260376, + "loss": 0.6886734962463379, + "loss/core": 0.6886734962463379, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.515756607055664, + "rewards/margins": 1.8329532146453857, + "rewards/rejected": 1.6828029155731201, + "step": 380 + }, + { + "epoch": 0.36784903857637646, + "grad_norm": 1.734375, + "learning_rate": 3.5433461362045447e-07, + "logits/chosen": -1.9373714923858643, + "logits/rejected": -1.9126183986663818, + "logps/chosen": -0.30250269174575806, + "logps/rejected": -0.3157595694065094, + "loss": 0.6906894445419312, + "loss/core": 0.6906894445419312, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.1111555099487305, + "rewards/margins": 1.0718204975128174, + "rewards/rejected": 2.039334535598755, + "step": 385 + }, + { + "epoch": 0.3726262988176281, + "grad_norm": 0.1328125, + "learning_rate": 3.4990952196986305e-07, + "logits/chosen": -1.8262889385223389, + "logits/rejected": -1.826817274093628, + "logps/chosen": -0.29866036772727966, + "logps/rejected": -0.29913249611854553, + "loss": 0.6902146935462952, + "loss/core": 0.6902146935462952, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.0936100482940674, + "rewards/margins": 1.1971681118011475, + "rewards/rejected": 0.8964420557022095, + "step": 390 + }, + { + "epoch": 0.37740355905887973, + "grad_norm": 0.1572265625, + "learning_rate": 3.4544685843405875e-07, + "logits/chosen": -1.9917919635772705, + "logits/rejected": -2.0477356910705566, + "logps/chosen": -0.2745470404624939, + "logps/rejected": -0.26887795329093933, + "loss": 0.6905907392501831, + "loss/core": 0.6905907392501831, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.0287718772888184, + "rewards/margins": 1.0297727584838867, + "rewards/rejected": 0.998999297618866, + "step": 395 + }, + { + "epoch": 0.38218081930013137, + "grad_norm": 0.318359375, + "learning_rate": 3.4094830123828786e-07, + "logits/chosen": -1.9308719635009766, + "logits/rejected": -1.8756221532821655, + "logps/chosen": -0.2740345895290375, + "logps/rejected": -0.2740941047668457, + "loss": 0.6921342015266418, + "loss/core": 0.6921342015266418, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.0607590675354004, + "rewards/margins": 0.43019723892211914, + "rewards/rejected": 1.6305620670318604, + "step": 400 + }, + { + "epoch": 0.386958079541383, + "grad_norm": 1.25, + "learning_rate": 3.3641554210593414e-07, + "logits/chosen": -1.9905240535736084, + "logits/rejected": -2.03885817527771, + "logps/chosen": -0.265903115272522, + "logps/rejected": -0.26458966732025146, + "loss": 0.6899409294128418, + "loss/core": 0.6899409294128418, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.5544772148132324, + "rewards/margins": 1.3087366819381714, + "rewards/rejected": 1.2457404136657715, + "step": 405 + }, + { + "epoch": 0.39173533978263464, + "grad_norm": 0.244140625, + "learning_rate": 3.3185028562233107e-07, + "logits/chosen": -1.6438277959823608, + "logits/rejected": -1.7215983867645264, + "logps/chosen": -0.30950814485549927, + "logps/rejected": -0.30294355750083923, + "loss": 0.6917771697044373, + "loss/core": 0.6917771697044373, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 1.2238495349884033, + "rewards/margins": 0.5521842241287231, + "rewards/rejected": 0.6716650724411011, + "step": 410 + }, + { + "epoch": 0.3965126000238863, + "grad_norm": 0.12890625, + "learning_rate": 3.272542485937368e-07, + "logits/chosen": -1.944154143333435, + "logits/rejected": -1.9149882793426514, + "logps/chosen": -0.2952657639980316, + "logps/rejected": -0.28746429085731506, + "loss": 0.6903901696205139, + "loss/core": 0.6903901696205139, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.458098888397217, + "rewards/margins": 1.1424970626831055, + "rewards/rejected": 1.3156019449234009, + "step": 415 + }, + { + "epoch": 0.40128986026513797, + "grad_norm": 1.4375, + "learning_rate": 3.226291594017137e-07, + "logits/chosen": -1.852236032485962, + "logits/rejected": -1.9032227993011475, + "logps/chosen": -0.32367467880249023, + "logps/rejected": -0.3032284379005432, + "loss": 0.6876433491706848, + "loss/core": 0.6876433491706848, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.3354249000549316, + "rewards/margins": 2.3274753093719482, + "rewards/rejected": 1.007949709892273, + "step": 420 + }, + { + "epoch": 0.4060671205063896, + "grad_norm": 0.30859375, + "learning_rate": 3.1797675735315454e-07, + "logits/chosen": -1.7561454772949219, + "logits/rejected": -1.8125495910644531, + "logps/chosen": -0.3123387396335602, + "logps/rejected": -0.31348609924316406, + "loss": 0.6884433627128601, + "loss/core": 0.6884433627128601, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.085703134536743, + "rewards/margins": 1.986158013343811, + "rewards/rejected": 1.0995451211929321, + "step": 425 + }, + { + "epoch": 0.41084438074764124, + "grad_norm": 0.98828125, + "learning_rate": 3.1329879202620047e-07, + "logits/chosen": -1.8188142776489258, + "logits/rejected": -1.914798378944397, + "logps/chosen": -0.2659410238265991, + "logps/rejected": -0.27096885442733765, + "loss": 0.691714346408844, + "loss/core": 0.691714346408844, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.4955499172210693, + "rewards/margins": 0.5815773606300354, + "rewards/rejected": 0.9139726758003235, + "step": 430 + }, + { + "epoch": 0.41562164098889287, + "grad_norm": 2.078125, + "learning_rate": 3.0859702261229613e-07, + "logits/chosen": -1.769033670425415, + "logits/rejected": -1.7553457021713257, + "logps/chosen": -0.2833646237850189, + "logps/rejected": -0.26877862215042114, + "loss": 0.6868944764137268, + "loss/core": 0.6868944764137268, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.937399387359619, + "rewards/margins": 2.5866665840148926, + "rewards/rejected": 1.3507325649261475, + "step": 435 + }, + { + "epoch": 0.4203989012301445, + "grad_norm": 0.345703125, + "learning_rate": 3.0387321725463e-07, + "logits/chosen": -1.8661785125732422, + "logits/rejected": -1.8362553119659424, + "logps/chosen": -0.29633772373199463, + "logps/rejected": -0.2867165207862854, + "loss": 0.6908766031265259, + "loss/core": 0.6908766031265259, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.151101589202881, + "rewards/margins": 0.9135802388191223, + "rewards/rejected": 1.2375215291976929, + "step": 440 + }, + { + "epoch": 0.42517616147139614, + "grad_norm": 0.63671875, + "learning_rate": 2.991291523832075e-07, + "logits/chosen": -1.6893161535263062, + "logits/rejected": -1.7709661722183228, + "logps/chosen": -0.2839089035987854, + "logps/rejected": -0.27923282980918884, + "loss": 0.6885705590248108, + "loss/core": 0.6885705590248108, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.0020592212677, + "rewards/margins": 1.8747539520263672, + "rewards/rejected": 1.127305507659912, + "step": 445 + }, + { + "epoch": 0.4299534217126478, + "grad_norm": 3.0625, + "learning_rate": 2.943666120468088e-07, + "logits/chosen": -1.8707926273345947, + "logits/rejected": -1.9706052541732788, + "logps/chosen": -0.26380446553230286, + "logps/rejected": -0.2769017517566681, + "loss": 0.6897034645080566, + "loss/core": 0.6897034645080566, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.89481782913208, + "rewards/margins": 1.4606375694274902, + "rewards/rejected": 1.4341800212860107, + "step": 450 + }, + { + "epoch": 0.4347306819538994, + "grad_norm": 0.10302734375, + "learning_rate": 2.8958738724208073e-07, + "logits/chosen": -1.648950219154358, + "logits/rejected": -1.645956039428711, + "logps/chosen": -0.30575722455978394, + "logps/rejected": -0.30025702714920044, + "loss": 0.6909690499305725, + "loss/core": 0.6909690499305725, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.719103455543518, + "rewards/margins": 0.8803337216377258, + "rewards/rejected": 0.8387696146965027, + "step": 455 + }, + { + "epoch": 0.4395079421951511, + "grad_norm": 0.2001953125, + "learning_rate": 2.8479327524001633e-07, + "logits/chosen": -1.7238233089447021, + "logits/rejected": -1.8354371786117554, + "logps/chosen": -0.2844577133655548, + "logps/rejected": -0.2817632555961609, + "loss": 0.6913126111030579, + "loss/core": 0.6913126111030579, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.263275623321533, + "rewards/margins": 0.7599185109138489, + "rewards/rejected": 1.5033574104309082, + "step": 460 + }, + { + "epoch": 0.44428520243640274, + "grad_norm": 0.1796875, + "learning_rate": 2.7998607891007493e-07, + "logits/chosen": -1.869998574256897, + "logits/rejected": -1.823256492614746, + "logps/chosen": -0.27499106526374817, + "logps/rejected": -0.28210312128067017, + "loss": 0.6916349530220032, + "loss/core": 0.6916349530220032, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.8145194053649902, + "rewards/margins": 0.7166808843612671, + "rewards/rejected": 2.0978386402130127, + "step": 465 + }, + { + "epoch": 0.4490624626776544, + "grad_norm": 0.443359375, + "learning_rate": 2.7516760604219616e-07, + "logits/chosen": -1.927408218383789, + "logits/rejected": -1.9314380884170532, + "logps/chosen": -0.26992565393447876, + "logps/rejected": -0.28807181119918823, + "loss": 0.6917387247085571, + "loss/core": 0.6917387247085571, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.2258682250976562, + "rewards/margins": 0.6031666994094849, + "rewards/rejected": 1.6227014064788818, + "step": 470 + }, + { + "epoch": 0.453839722918906, + "grad_norm": 0.2578125, + "learning_rate": 2.703396686669646e-07, + "logits/chosen": -2.002894878387451, + "logits/rejected": -2.003776788711548, + "logps/chosen": -0.27657777070999146, + "logps/rejected": -0.2728041708469391, + "loss": 0.6912546753883362, + "loss/core": 0.6912546753883362, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.364955425262451, + "rewards/margins": 0.7624953985214233, + "rewards/rejected": 1.6024599075317383, + "step": 475 + }, + { + "epoch": 0.45861698316015764, + "grad_norm": 1.15625, + "learning_rate": 2.6550408237417884e-07, + "logits/chosen": -1.601522445678711, + "logits/rejected": -1.6724109649658203, + "logps/chosen": -0.279160737991333, + "logps/rejected": -0.2730087637901306, + "loss": 0.683729350566864, + "loss/core": 0.683729350566864, + "rewards/accuracies": 0.875, + "rewards/chosen": 5.24552059173584, + "rewards/margins": 4.030712127685547, + "rewards/rejected": 1.2148076295852661, + "step": 480 + }, + { + "epoch": 0.4633942434014093, + "grad_norm": 0.341796875, + "learning_rate": 2.6066266563008265e-07, + "logits/chosen": -1.8503739833831787, + "logits/rejected": -1.8288978338241577, + "logps/chosen": -0.28329378366470337, + "logps/rejected": -0.30308961868286133, + "loss": 0.6905882954597473, + "loss/core": 0.6905882954597473, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.0931591987609863, + "rewards/margins": 1.066819429397583, + "rewards/rejected": 1.0263397693634033, + "step": 485 + }, + { + "epoch": 0.4681715036426609, + "grad_norm": 1.15625, + "learning_rate": 2.5581723909351404e-07, + "logits/chosen": -1.9194366931915283, + "logits/rejected": -1.8568189144134521, + "logps/chosen": -0.28827863931655884, + "logps/rejected": -0.3087393343448639, + "loss": 0.6923280954360962, + "loss/core": 0.6923280954360962, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.665645956993103, + "rewards/margins": 0.3457864820957184, + "rewards/rejected": 1.3198593854904175, + "step": 490 + }, + { + "epoch": 0.47294876388391255, + "grad_norm": 2.453125, + "learning_rate": 2.509696249312301e-07, + "logits/chosen": -1.7917439937591553, + "logits/rejected": -1.7728865146636963, + "logps/chosen": -0.316561222076416, + "logps/rejected": -0.30622977018356323, + "loss": 0.6886661648750305, + "loss/core": 0.6886661648750305, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 3.6381404399871826, + "rewards/margins": 1.929628610610962, + "rewards/rejected": 1.7085119485855103, + "step": 495 + }, + { + "epoch": 0.47772602412516424, + "grad_norm": 1.890625, + "learning_rate": 2.461216461326642e-07, + "logits/chosen": -1.771681785583496, + "logits/rejected": -1.731675386428833, + "logps/chosen": -0.28252094984054565, + "logps/rejected": -0.26652899384498596, + "loss": 0.6878234148025513, + "loss/core": 0.6878234148025513, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.460325241088867, + "rewards/margins": 2.2089626789093018, + "rewards/rejected": 1.2513624429702759, + "step": 500 + }, + { + "epoch": 0.4825032843664159, + "grad_norm": 0.2109375, + "learning_rate": 2.412751258243748e-07, + "logits/chosen": -1.5761404037475586, + "logits/rejected": -1.5784422159194946, + "logps/chosen": -0.3069811463356018, + "logps/rejected": -0.3080241084098816, + "loss": 0.6885954737663269, + "loss/core": 0.6885954737663269, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.2021026611328125, + "rewards/margins": 1.8541738986968994, + "rewards/rejected": 1.347928762435913, + "step": 505 + }, + { + "epoch": 0.4872805446076675, + "grad_norm": 0.74609375, + "learning_rate": 2.3643188658444158e-07, + "logits/chosen": -1.889041543006897, + "logits/rejected": -1.9575011730194092, + "logps/chosen": -0.26323431730270386, + "logps/rejected": -0.2795872688293457, + "loss": 0.689049482345581, + "loss/core": 0.689049482345581, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.0836408138275146, + "rewards/margins": 1.6842677593231201, + "rewards/rejected": 1.399373173713684, + "step": 510 + }, + { + "epoch": 0.49205780484891914, + "grad_norm": 0.169921875, + "learning_rate": 2.315937497570688e-07, + "logits/chosen": -1.6385066509246826, + "logits/rejected": -1.6468448638916016, + "logps/chosen": -0.2980702519416809, + "logps/rejected": -0.3428047299385071, + "loss": 0.6906419992446899, + "loss/core": 0.6906419992446899, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.289318799972534, + "rewards/margins": 1.0524483919143677, + "rewards/rejected": 1.236870527267456, + "step": 515 + }, + { + "epoch": 0.4968350650901708, + "grad_norm": 0.08837890625, + "learning_rate": 2.2676253476765194e-07, + "logits/chosen": -1.5919002294540405, + "logits/rejected": -1.7469463348388672, + "logps/chosen": -0.3116455674171448, + "logps/rejected": -0.29385876655578613, + "loss": 0.6895068883895874, + "loss/core": 0.6895068883895874, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.45418643951416, + "rewards/margins": 1.5170066356658936, + "rewards/rejected": 0.937179446220398, + "step": 520 + }, + { + "epoch": 0.5016123253314224, + "grad_norm": 0.061279296875, + "learning_rate": 2.2194005843856633e-07, + "logits/chosen": -1.790310263633728, + "logits/rejected": -1.8553049564361572, + "logps/chosen": -0.27847370505332947, + "logps/rejected": -0.2731352150440216, + "loss": 0.688349723815918, + "loss/core": 0.688349723815918, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.402092456817627, + "rewards/margins": 1.995959997177124, + "rewards/rejected": 1.406132459640503, + "step": 525 + }, + { + "epoch": 0.5063895855726741, + "grad_norm": 0.126953125, + "learning_rate": 2.1712813430593434e-07, + "logits/chosen": -1.7230796813964844, + "logits/rejected": -1.798443078994751, + "logps/chosen": -0.29598766565322876, + "logps/rejected": -0.2965576946735382, + "loss": 0.6906968951225281, + "loss/core": 0.6906968951225281, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.8174304962158203, + "rewards/margins": 0.9899489283561707, + "rewards/rejected": 0.8274815678596497, + "step": 530 + }, + { + "epoch": 0.5111668458139257, + "grad_norm": 0.1904296875, + "learning_rate": 2.123285719376292e-07, + "logits/chosen": -1.7792017459869385, + "logits/rejected": -1.7719032764434814, + "logps/chosen": -0.28997790813446045, + "logps/rejected": -0.28154560923576355, + "loss": 0.6893754005432129, + "loss/core": 0.6893754005432129, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 3.301804304122925, + "rewards/margins": 1.5316702127456665, + "rewards/rejected": 1.7701339721679688, + "step": 535 + }, + { + "epoch": 0.5159441060551774, + "grad_norm": 0.5234375, + "learning_rate": 2.0754317625276982e-07, + "logits/chosen": -1.7738841772079468, + "logits/rejected": -1.7625110149383545, + "logps/chosen": -0.29042699933052063, + "logps/rejected": -0.28381985425949097, + "loss": 0.6905869245529175, + "loss/core": 0.6905869245529175, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9816631078720093, + "rewards/margins": 1.0423338413238525, + "rewards/rejected": 0.9393291473388672, + "step": 540 + }, + { + "epoch": 0.520721366296429, + "grad_norm": 0.71875, + "learning_rate": 2.0277374684296498e-07, + "logits/chosen": -1.829310417175293, + "logits/rejected": -1.8881504535675049, + "logps/chosen": -0.2865942716598511, + "logps/rejected": -0.2712253928184509, + "loss": 0.686186671257019, + "loss/core": 0.686186671257019, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.7003753185272217, + "rewards/margins": 2.869210720062256, + "rewards/rejected": 0.831164538860321, + "step": 545 + }, + { + "epoch": 0.5254986265376806, + "grad_norm": 0.2294921875, + "learning_rate": 1.980220772955602e-07, + "logits/chosen": -1.6887779235839844, + "logits/rejected": -1.7008845806121826, + "logps/chosen": -0.28278762102127075, + "logps/rejected": -0.2788473069667816, + "loss": 0.6903247237205505, + "loss/core": 0.6903247237205505, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.2481186389923096, + "rewards/margins": 1.135740041732788, + "rewards/rejected": 1.112378716468811, + "step": 550 + }, + { + "epoch": 0.5302758867789323, + "grad_norm": 0.2080078125, + "learning_rate": 1.932899545191433e-07, + "logits/chosen": -1.5623390674591064, + "logits/rejected": -1.6467643976211548, + "logps/chosen": -0.3176479935646057, + "logps/rejected": -0.29388684034347534, + "loss": 0.6890052556991577, + "loss/core": 0.6890052556991577, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.222991943359375, + "rewards/margins": 1.7007945775985718, + "rewards/rejected": 1.5221973657608032, + "step": 555 + }, + { + "epoch": 0.5350531470201839, + "grad_norm": 0.2099609375, + "learning_rate": 1.885791580715609e-07, + "logits/chosen": -1.622319221496582, + "logits/rejected": -1.63738214969635, + "logps/chosen": -0.3008866310119629, + "logps/rejected": -0.2952221930027008, + "loss": 0.6901183724403381, + "loss/core": 0.6901183724403381, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.5821938514709473, + "rewards/margins": 1.2760289907455444, + "rewards/rejected": 1.3061645030975342, + "step": 560 + }, + { + "epoch": 0.5398304072614356, + "grad_norm": 0.498046875, + "learning_rate": 1.8389145949069951e-07, + "logits/chosen": -1.7124159336090088, + "logits/rejected": -1.7910178899765015, + "logps/chosen": -0.27922117710113525, + "logps/rejected": -0.2886210083961487, + "loss": 0.6854382753372192, + "loss/core": 0.6854382753372192, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 5.158889293670654, + "rewards/margins": 3.2606253623962402, + "rewards/rejected": 1.8982641696929932, + "step": 565 + }, + { + "epoch": 0.5446076675026872, + "grad_norm": 0.7890625, + "learning_rate": 1.792286216282824e-07, + "logits/chosen": -1.8085368871688843, + "logits/rejected": -1.7837501764297485, + "logps/chosen": -0.31605297327041626, + "logps/rejected": -0.30117732286453247, + "loss": 0.692598283290863, + "loss/core": 0.692598283290863, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.354279041290283, + "rewards/margins": 0.28271275758743286, + "rewards/rejected": 2.071566343307495, + "step": 570 + }, + { + "epoch": 0.5493849277439389, + "grad_norm": 0.361328125, + "learning_rate": 1.745923979869336e-07, + "logits/chosen": -2.0771026611328125, + "logits/rejected": -2.058582305908203, + "logps/chosen": -0.28092527389526367, + "logps/rejected": -0.2910299301147461, + "loss": 0.6897355914115906, + "loss/core": 0.6897355914115906, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.395536422729492, + "rewards/margins": 1.4038153886795044, + "rewards/rejected": 0.9917212724685669, + "step": 575 + }, + { + "epoch": 0.5541621879851905, + "grad_norm": 0.169921875, + "learning_rate": 1.6998453206075708e-07, + "logits/chosen": -1.6925674676895142, + "logits/rejected": -1.7791401147842407, + "logps/chosen": -0.3049921989440918, + "logps/rejected": -0.3050501346588135, + "loss": 0.6895653605461121, + "loss/core": 0.6895653605461121, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.476440191268921, + "rewards/margins": 1.4556043148040771, + "rewards/rejected": 1.0208359956741333, + "step": 580 + }, + { + "epoch": 0.5589394482264421, + "grad_norm": 0.58203125, + "learning_rate": 1.6540675667967973e-07, + "logits/chosen": -1.778446912765503, + "logits/rejected": -1.8237922191619873, + "logps/chosen": -0.28048455715179443, + "logps/rejected": -0.28305408358573914, + "loss": 0.6914597153663635, + "loss/core": 0.6914597153663635, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.6184022426605225, + "rewards/margins": 0.748033344745636, + "rewards/rejected": 1.8703689575195312, + "step": 585 + }, + { + "epoch": 0.5637167084676937, + "grad_norm": 0.75390625, + "learning_rate": 1.60860793357805e-07, + "logits/chosen": -1.7187820672988892, + "logits/rejected": -1.6614373922348022, + "logps/chosen": -0.3019927144050598, + "logps/rejected": -0.29929202795028687, + "loss": 0.6907371282577515, + "loss/core": 0.6907371282577515, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.7743518352508545, + "rewards/margins": 1.0062803030014038, + "rewards/rejected": 1.7680715322494507, + "step": 590 + }, + { + "epoch": 0.5684939687089454, + "grad_norm": 0.1376953125, + "learning_rate": 1.5634835164602196e-07, + "logits/chosen": -1.7278330326080322, + "logits/rejected": -1.6942561864852905, + "logps/chosen": -0.27367183566093445, + "logps/rejected": -0.28410977125167847, + "loss": 0.6944975852966309, + "loss/core": 0.6944975852966309, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.822975516319275, + "rewards/margins": -0.4711933732032776, + "rewards/rejected": 2.294168710708618, + "step": 595 + }, + { + "epoch": 0.5732712289501971, + "grad_norm": 0.244140625, + "learning_rate": 1.518711284891132e-07, + "logits/chosen": -1.8173329830169678, + "logits/rejected": -1.748335599899292, + "logps/chosen": -0.29630497097969055, + "logps/rejected": -0.30823907256126404, + "loss": 0.6916288733482361, + "loss/core": 0.6916288733482361, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.783888816833496, + "rewards/margins": 0.6391955614089966, + "rewards/rejected": 1.14469313621521, + "step": 600 + }, + { + "epoch": 0.5780484891914487, + "grad_norm": 0.310546875, + "learning_rate": 1.47430807587603e-07, + "logits/chosen": -1.9312461614608765, + "logits/rejected": -1.9855235815048218, + "logps/chosen": -0.2915689945220947, + "logps/rejected": -0.3033396601676941, + "loss": 0.6898959279060364, + "loss/core": 0.6898959279060364, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.9401935338974, + "rewards/margins": 1.346240758895874, + "rewards/rejected": 0.5939527153968811, + "step": 605 + }, + { + "epoch": 0.5828257494327004, + "grad_norm": 0.1435546875, + "learning_rate": 1.430290587645865e-07, + "logits/chosen": -1.854900598526001, + "logits/rejected": -1.9152843952178955, + "logps/chosen": -0.30195021629333496, + "logps/rejected": -0.3078872561454773, + "loss": 0.6913703680038452, + "loss/core": 0.6913703680038452, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.3067500591278076, + "rewards/margins": 0.7192279696464539, + "rewards/rejected": 1.5875221490859985, + "step": 610 + }, + { + "epoch": 0.587603009673952, + "grad_norm": 0.07373046875, + "learning_rate": 1.3866753733777765e-07, + "logits/chosen": -1.8532918691635132, + "logits/rejected": -1.834735631942749, + "logps/chosen": -0.3193635642528534, + "logps/rejected": -0.3099674880504608, + "loss": 0.6915178894996643, + "loss/core": 0.6915178894996643, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.416029930114746, + "rewards/margins": 0.6589576005935669, + "rewards/rejected": 0.7570723295211792, + "step": 615 + }, + { + "epoch": 0.5923802699152036, + "grad_norm": 0.1806640625, + "learning_rate": 1.343478834970121e-07, + "logits/chosen": -1.9541394710540771, + "logits/rejected": -1.946727991104126, + "logps/chosen": -0.26166877150535583, + "logps/rejected": -0.2776499092578888, + "loss": 0.6918145418167114, + "loss/core": 0.6918145418167114, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.89385187625885, + "rewards/margins": 0.5465883612632751, + "rewards/rejected": 1.3472635746002197, + "step": 620 + }, + { + "epoch": 0.5971575301564552, + "grad_norm": 0.322265625, + "learning_rate": 1.3007172168743852e-07, + "logits/chosen": -1.755165696144104, + "logits/rejected": -1.7259944677352905, + "logps/chosen": -0.2936655580997467, + "logps/rejected": -0.298048198223114, + "loss": 0.6916288137435913, + "loss/core": 0.6916288137435913, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.5220264196395874, + "rewards/margins": 0.609094500541687, + "rewards/rejected": 0.9129319190979004, + "step": 625 + }, + { + "epoch": 0.6019347903977069, + "grad_norm": 0.408203125, + "learning_rate": 1.25840659998631e-07, + "logits/chosen": -1.9866920709609985, + "logits/rejected": -1.915894865989685, + "logps/chosen": -0.2786349654197693, + "logps/rejected": -0.2918104827404022, + "loss": 0.6923294067382812, + "loss/core": 0.6923294067382812, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.2574752569198608, + "rewards/margins": 0.3320327401161194, + "rewards/rejected": 0.9254426956176758, + "step": 630 + }, + { + "epoch": 0.6067120506389586, + "grad_norm": 0.060302734375, + "learning_rate": 1.2165628955985313e-07, + "logits/chosen": -1.8373003005981445, + "logits/rejected": -1.7937291860580444, + "logps/chosen": -0.3085290789604187, + "logps/rejected": -0.29286837577819824, + "loss": 0.6876273155212402, + "loss/core": 0.6876273155212402, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.1967198848724365, + "rewards/margins": 2.337005615234375, + "rewards/rejected": 0.8597143292427063, + "step": 635 + }, + { + "epoch": 0.6114893108802102, + "grad_norm": 0.7265625, + "learning_rate": 1.175201839416988e-07, + "logits/chosen": -1.621816873550415, + "logits/rejected": -1.822488784790039, + "logps/chosen": -0.2964431643486023, + "logps/rejected": -0.2829035520553589, + "loss": 0.6889544129371643, + "loss/core": 0.6889544129371643, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.3975021839141846, + "rewards/margins": 1.7253767251968384, + "rewards/rejected": 0.6721251606941223, + "step": 640 + }, + { + "epoch": 0.6162665711214619, + "grad_norm": 0.28515625, + "learning_rate": 1.1343389856433658e-07, + "logits/chosen": -1.8435766696929932, + "logits/rejected": -1.9550691843032837, + "logps/chosen": -0.28372493386268616, + "logps/rejected": -0.2617836594581604, + "loss": 0.6860982179641724, + "loss/core": 0.6860982179641724, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 4.548431873321533, + "rewards/margins": 2.9798524379730225, + "rewards/rejected": 1.5685795545578003, + "step": 645 + }, + { + "epoch": 0.6210438313627135, + "grad_norm": 0.1845703125, + "learning_rate": 1.0939897011258e-07, + "logits/chosen": -1.938122034072876, + "logits/rejected": -1.9469550848007202, + "logps/chosen": -0.29863983392715454, + "logps/rejected": -0.28583261370658875, + "loss": 0.6872848868370056, + "loss/core": 0.6872848868370056, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.3786215782165527, + "rewards/margins": 2.4727911949157715, + "rewards/rejected": 0.9058302044868469, + "step": 650 + }, + { + "epoch": 0.6258210916039652, + "grad_norm": 0.314453125, + "learning_rate": 1.0541691595800336e-07, + "logits/chosen": -1.600602149963379, + "logits/rejected": -1.5954480171203613, + "logps/chosen": -0.27909520268440247, + "logps/rejected": -0.30399781465530396, + "loss": 0.6903051733970642, + "loss/core": 0.6903051733970642, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.860823154449463, + "rewards/margins": 1.1584161520004272, + "rewards/rejected": 1.702406883239746, + "step": 655 + }, + { + "epoch": 0.6305983518452167, + "grad_norm": 0.2001953125, + "learning_rate": 1.0148923358832021e-07, + "logits/chosen": -2.0156798362731934, + "logits/rejected": -2.0649049282073975, + "logps/chosen": -0.2738130986690521, + "logps/rejected": -0.2789636552333832, + "loss": 0.6907733678817749, + "loss/core": 0.6907733678817749, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.0494155883789062, + "rewards/margins": 0.9572056531906128, + "rewards/rejected": 1.092210054397583, + "step": 660 + }, + { + "epoch": 0.6353756120864684, + "grad_norm": 0.2890625, + "learning_rate": 9.761740004423926e-08, + "logits/chosen": -1.7956726551055908, + "logits/rejected": -1.876203179359436, + "logps/chosen": -0.2982097268104553, + "logps/rejected": -0.30547016859054565, + "loss": 0.6916817426681519, + "loss/core": 0.6916817426681519, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.7903995513916016, + "rewards/margins": 0.6054316759109497, + "rewards/rejected": 1.1849677562713623, + "step": 665 + }, + { + "epoch": 0.64015287232772, + "grad_norm": 0.1650390625, + "learning_rate": 9.380287136400999e-08, + "logits/chosen": -1.754360556602478, + "logits/rejected": -1.8462966680526733, + "logps/chosen": -0.28413721919059753, + "logps/rejected": -0.29304325580596924, + "loss": 0.6911409497261047, + "loss/core": 0.6911409497261047, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.00300931930542, + "rewards/margins": 0.8140490651130676, + "rewards/rejected": 1.188960313796997, + "step": 670 + }, + { + "epoch": 0.6449301325689717, + "grad_norm": 0.1806640625, + "learning_rate": 9.004708203586628e-08, + "logits/chosen": -1.7482950687408447, + "logits/rejected": -1.8495619297027588, + "logps/chosen": -0.29564136266708374, + "logps/rejected": -0.2883380055427551, + "loss": 0.6910102367401123, + "loss/core": 0.6910102367401123, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.9195715188980103, + "rewards/margins": 0.8625022768974304, + "rewards/rejected": 1.0570690631866455, + "step": 675 + }, + { + "epoch": 0.6497073928102234, + "grad_norm": 2.09375, + "learning_rate": 8.635144445857407e-08, + "logits/chosen": -1.7818498611450195, + "logits/rejected": -1.819561243057251, + "logps/chosen": -0.2841910719871521, + "logps/rejected": -0.2850389778614044, + "loss": 0.693510890007019, + "loss/core": 0.693510890007019, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.2231769561767578, + "rewards/margins": -0.12083403766155243, + "rewards/rejected": 1.3440110683441162, + "step": 680 + }, + { + "epoch": 0.654484653051475, + "grad_norm": 0.62890625, + "learning_rate": 8.271734841028552e-08, + "logits/chosen": -1.8370325565338135, + "logits/rejected": -1.8255866765975952, + "logps/chosen": -0.29680460691452026, + "logps/rejected": -0.2980230748653412, + "loss": 0.6934405565261841, + "loss/core": 0.6934405565261841, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.4498194456100464, + "rewards/margins": -0.0671045333147049, + "rewards/rejected": 1.5169239044189453, + "step": 685 + }, + { + "epoch": 0.6592619132927267, + "grad_norm": 0.302734375, + "learning_rate": 7.91461605259007e-08, + "logits/chosen": -1.761301040649414, + "logits/rejected": -1.7649517059326172, + "logps/chosen": -0.26283127069473267, + "logps/rejected": -0.2820115387439728, + "loss": 0.6897014379501343, + "loss/core": 0.6897014379501343, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.0530502796173096, + "rewards/margins": 1.409324049949646, + "rewards/rejected": 1.6437263488769531, + "step": 690 + }, + { + "epoch": 0.6640391735339782, + "grad_norm": 0.15625, + "learning_rate": 7.563922378313217e-08, + "logits/chosen": -1.8872359991073608, + "logits/rejected": -1.8546419143676758, + "logps/chosen": -0.29460957646369934, + "logps/rejected": -0.30844348669052124, + "loss": 0.6914256811141968, + "loss/core": 0.6914256811141968, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.4490922689437866, + "rewards/margins": 0.6981182098388672, + "rewards/rejected": 0.750974178314209, + "step": 695 + }, + { + "epoch": 0.6688164337752299, + "grad_norm": 0.58984375, + "learning_rate": 7.219785699746572e-08, + "logits/chosen": -1.8494236469268799, + "logits/rejected": -1.9358432292938232, + "logps/chosen": -0.28461378812789917, + "logps/rejected": -0.26845136284828186, + "loss": 0.6891158819198608, + "loss/core": 0.6891158819198608, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.0717930793762207, + "rewards/margins": 1.6619094610214233, + "rewards/rejected": 1.409883737564087, + "step": 700 + }, + { + "epoch": 0.6735936940164815, + "grad_norm": 1.1328125, + "learning_rate": 6.882335432620779e-08, + "logits/chosen": -1.827734351158142, + "logits/rejected": -1.946860909461975, + "logps/chosen": -0.27680468559265137, + "logps/rejected": -0.275119811296463, + "loss": 0.6926502585411072, + "loss/core": 0.6926502585411072, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.4091708660125732, + "rewards/margins": 0.20720620453357697, + "rewards/rejected": 1.2019646167755127, + "step": 705 + }, + { + "epoch": 0.6783709542577332, + "grad_norm": 0.1279296875, + "learning_rate": 6.551698478180589e-08, + "logits/chosen": -1.6843173503875732, + "logits/rejected": -1.7385330200195312, + "logps/chosen": -0.28421929478645325, + "logps/rejected": -0.2876400053501129, + "loss": 0.69017493724823, + "loss/core": 0.69017493724823, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.562863826751709, + "rewards/margins": 1.2080352306365967, + "rewards/rejected": 1.3548285961151123, + "step": 710 + }, + { + "epoch": 0.6831482144989849, + "grad_norm": 0.0947265625, + "learning_rate": 6.22799917546252e-08, + "logits/chosen": -1.7102231979370117, + "logits/rejected": -1.7559185028076172, + "logps/chosen": -0.2854920029640198, + "logps/rejected": -0.29401493072509766, + "loss": 0.6895767450332642, + "loss/core": 0.6895767450332642, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.8428616523742676, + "rewards/margins": 1.45379638671875, + "rewards/rejected": 1.389065146446228, + "step": 715 + }, + { + "epoch": 0.6879254747402365, + "grad_norm": 0.11328125, + "learning_rate": 5.9113592545359944e-08, + "logits/chosen": -1.742907166481018, + "logits/rejected": -1.7834516763687134, + "logps/chosen": -0.2981284260749817, + "logps/rejected": -0.29392343759536743, + "loss": 0.6902607679367065, + "loss/core": 0.6902607679367065, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.236698627471924, + "rewards/margins": 1.1834802627563477, + "rewards/rejected": 1.053218126296997, + "step": 720 + }, + { + "epoch": 0.6927027349814882, + "grad_norm": 0.1552734375, + "learning_rate": 5.601897790725643e-08, + "logits/chosen": -1.7728607654571533, + "logits/rejected": -1.7845735549926758, + "logps/chosen": -0.3191523551940918, + "logps/rejected": -0.3145609498023987, + "loss": 0.6922028660774231, + "loss/core": 0.6922028660774231, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.1376005411148071, + "rewards/margins": 0.3802829086780548, + "rewards/rejected": 0.7573176622390747, + "step": 725 + }, + { + "epoch": 0.6974799952227397, + "grad_norm": 0.298828125, + "learning_rate": 5.299731159831952e-08, + "logits/chosen": -1.832451581954956, + "logits/rejected": -1.820267915725708, + "logps/chosen": -0.2969440519809723, + "logps/rejected": -0.30008524656295776, + "loss": 0.6890357732772827, + "loss/core": 0.6890357732772827, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.5468456745147705, + "rewards/margins": 1.685861349105835, + "rewards/rejected": 0.8609840273857117, + "step": 730 + }, + { + "epoch": 0.7022572554639914, + "grad_norm": 0.236328125, + "learning_rate": 5.0049729943671013e-08, + "logits/chosen": -1.8389718532562256, + "logits/rejected": -1.8534542322158813, + "logps/chosen": -0.2692399024963379, + "logps/rejected": -0.2879648208618164, + "loss": 0.6913638710975647, + "loss/core": 0.6913638710975647, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.036916971206665, + "rewards/margins": 0.7281818389892578, + "rewards/rejected": 1.3087352514266968, + "step": 735 + }, + { + "epoch": 0.707034515705243, + "grad_norm": 0.1708984375, + "learning_rate": 4.7177341408223994e-08, + "logits/chosen": -1.733703851699829, + "logits/rejected": -1.7701585292816162, + "logps/chosen": -0.26635271310806274, + "logps/rejected": -0.2835581600666046, + "loss": 0.6906028985977173, + "loss/core": 0.6906028985977173, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.2045063972473145, + "rewards/margins": 1.0283342599868774, + "rewards/rejected": 1.1761722564697266, + "step": 740 + }, + { + "epoch": 0.7118117759464947, + "grad_norm": 0.396484375, + "learning_rate": 4.438122617983442e-08, + "logits/chosen": -1.5449193716049194, + "logits/rejected": -1.5420639514923096, + "logps/chosen": -0.28244468569755554, + "logps/rejected": -0.26813778281211853, + "loss": 0.6907768249511719, + "loss/core": 0.6907768249511719, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.1589250564575195, + "rewards/margins": 1.085134744644165, + "rewards/rejected": 2.0737905502319336, + "step": 745 + }, + { + "epoch": 0.7165890361877463, + "grad_norm": 0.1708984375, + "learning_rate": 4.1662435763087114e-08, + "logits/chosen": -1.8882849216461182, + "logits/rejected": -1.9425636529922485, + "logps/chosen": -0.25149840116500854, + "logps/rejected": -0.2820515036582947, + "loss": 0.6905457973480225, + "loss/core": 0.6905457973480225, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.5088672637939453, + "rewards/margins": 1.0693387985229492, + "rewards/rejected": 1.4395287036895752, + "step": 750 + }, + { + "epoch": 0.721366296428998, + "grad_norm": 0.0771484375, + "learning_rate": 3.902199258386732e-08, + "logits/chosen": -1.9825334548950195, + "logits/rejected": -2.009000778198242, + "logps/chosen": -0.3606415390968323, + "logps/rejected": -0.28844404220581055, + "loss": 0.6889899969100952, + "loss/core": 0.6889899969100952, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.009084701538086, + "rewards/margins": 1.8813291788101196, + "rewards/rejected": 2.1277554035186768, + "step": 755 + }, + { + "epoch": 0.7261435566702497, + "grad_norm": 0.130859375, + "learning_rate": 3.646088960486862e-08, + "logits/chosen": -1.849302887916565, + "logits/rejected": -1.8440090417861938, + "logps/chosen": -0.29616793990135193, + "logps/rejected": -0.2985708713531494, + "loss": 0.6910385489463806, + "loss/core": 0.6910385489463806, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.5953813791275024, + "rewards/margins": 0.8663269281387329, + "rewards/rejected": 0.72905433177948, + "step": 760 + }, + { + "epoch": 0.7309208169115012, + "grad_norm": 0.1796875, + "learning_rate": 3.398008995217988e-08, + "logits/chosen": -2.037290096282959, + "logits/rejected": -2.077535390853882, + "logps/chosen": -0.27280181646347046, + "logps/rejected": -0.2824481725692749, + "loss": 0.6917025446891785, + "loss/core": 0.6917025446891785, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.4865820407867432, + "rewards/margins": 0.5794221758842468, + "rewards/rejected": 0.907159686088562, + "step": 765 + }, + { + "epoch": 0.7356980771527529, + "grad_norm": 0.453125, + "learning_rate": 3.1580526553093315e-08, + "logits/chosen": -1.942304253578186, + "logits/rejected": -1.9689109325408936, + "logps/chosen": -0.29773375391960144, + "logps/rejected": -0.2885454595088959, + "loss": 0.6907824277877808, + "loss/core": 0.6907824277877808, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.037071943283081, + "rewards/margins": 0.9551854133605957, + "rewards/rejected": 1.0818865299224854, + "step": 770 + }, + { + "epoch": 0.7404753373940045, + "grad_norm": 0.4921875, + "learning_rate": 2.9263101785268252e-08, + "logits/chosen": -1.9432833194732666, + "logits/rejected": -1.9501100778579712, + "logps/chosen": -0.26834672689437866, + "logps/rejected": -0.269378662109375, + "loss": 0.6902218461036682, + "loss/core": 0.6902218461036682, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 2.0740108489990234, + "rewards/margins": 1.1815000772476196, + "rewards/rejected": 0.8925108909606934, + "step": 775 + }, + { + "epoch": 0.7452525976352562, + "grad_norm": 0.349609375, + "learning_rate": 2.7028687137384264e-08, + "logits/chosen": -1.6919254064559937, + "logits/rejected": -1.7820037603378296, + "logps/chosen": -0.28077149391174316, + "logps/rejected": -0.279075562953949, + "loss": 0.6903265118598938, + "loss/core": 0.6903265118598938, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.868180513381958, + "rewards/margins": 1.1484382152557373, + "rewards/rejected": 1.7197420597076416, + "step": 780 + }, + { + "epoch": 0.7500298578765078, + "grad_norm": 0.38671875, + "learning_rate": 2.4878122881409447e-08, + "logits/chosen": -1.6962074041366577, + "logits/rejected": -1.6986114978790283, + "logps/chosen": -0.28555965423583984, + "logps/rejected": -0.2935461401939392, + "loss": 0.6910402178764343, + "loss/core": 0.6910402178764343, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.4226348400115967, + "rewards/margins": 0.897823691368103, + "rewards/rejected": 1.5248112678527832, + "step": 785 + }, + { + "epoch": 0.7548071181177595, + "grad_norm": 0.14453125, + "learning_rate": 2.2812217756608937e-08, + "logits/chosen": -1.844538927078247, + "logits/rejected": -1.9082237482070923, + "logps/chosen": -0.307687908411026, + "logps/rejected": -0.3395816683769226, + "loss": 0.6907016634941101, + "loss/core": 0.6907016634941101, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7747430801391602, + "rewards/margins": 0.9889305233955383, + "rewards/rejected": 0.7858127951622009, + "step": 790 + }, + { + "epoch": 0.7595843783590112, + "grad_norm": 1.015625, + "learning_rate": 2.0831748665410763e-08, + "logits/chosen": -1.975682020187378, + "logits/rejected": -1.9615514278411865, + "logps/chosen": -0.2598734200000763, + "logps/rejected": -0.27107658982276917, + "loss": 0.6921221613883972, + "loss/core": 0.6921221613883972, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.4536190032958984, + "rewards/margins": 0.45077699422836304, + "rewards/rejected": 2.0028419494628906, + "step": 795 + }, + { + "epoch": 0.7643616386002627, + "grad_norm": 0.2158203125, + "learning_rate": 1.8937460381244967e-08, + "logits/chosen": -1.632433533668518, + "logits/rejected": -1.7214616537094116, + "logps/chosen": -0.308523565530777, + "logps/rejected": -0.28264203667640686, + "loss": 0.6870818138122559, + "loss/core": 0.6870818138122559, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.5316734313964844, + "rewards/margins": 2.579163074493408, + "rewards/rejected": 0.9525102376937866, + "step": 800 + }, + { + "epoch": 0.7691388988415144, + "grad_norm": 0.341796875, + "learning_rate": 1.713006526846439e-08, + "logits/chosen": -1.7808490991592407, + "logits/rejected": -1.8207063674926758, + "logps/chosen": -0.2825844883918762, + "logps/rejected": -0.27922552824020386, + "loss": 0.6903389692306519, + "loss/core": 0.6903389692306519, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.0911669731140137, + "rewards/margins": 1.1479694843292236, + "rewards/rejected": 0.9431974291801453, + "step": 805 + }, + { + "epoch": 0.773916159082766, + "grad_norm": 0.33203125, + "learning_rate": 1.541024301445404e-08, + "logits/chosen": -2.021035671234131, + "logits/rejected": -2.0233137607574463, + "logps/chosen": -0.2777281403541565, + "logps/rejected": -0.2643401622772217, + "loss": 0.6903270483016968, + "loss/core": 0.6903270483016968, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.9988548755645752, + "rewards/margins": 1.1564974784851074, + "rewards/rejected": 0.8423575162887573, + "step": 810 + }, + { + "epoch": 0.7786934193240177, + "grad_norm": 1.8671875, + "learning_rate": 1.3778640374027983e-08, + "logits/chosen": -1.9014087915420532, + "logits/rejected": -1.8085308074951172, + "logps/chosen": -0.2891666293144226, + "logps/rejected": -0.30600792169570923, + "loss": 0.6911376118659973, + "loss/core": 0.6911376118659973, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.9695777893066406, + "rewards/margins": 0.8112414479255676, + "rewards/rejected": 1.1583364009857178, + "step": 815 + }, + { + "epoch": 0.7834706795652693, + "grad_norm": 0.458984375, + "learning_rate": 1.2235870926211616e-08, + "logits/chosen": -1.7752872705459595, + "logits/rejected": -1.7699896097183228, + "logps/chosen": -0.28837233781814575, + "logps/rejected": -0.2897118926048279, + "loss": 0.6898549199104309, + "loss/core": 0.6898549199104309, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.560025691986084, + "rewards/margins": 1.3317620754241943, + "rewards/rejected": 1.2282639741897583, + "step": 820 + }, + { + "epoch": 0.788247939806521, + "grad_norm": 0.1025390625, + "learning_rate": 1.0782514843499652e-08, + "logits/chosen": -1.6815214157104492, + "logits/rejected": -1.7447792291641235, + "logps/chosen": -0.30126506090164185, + "logps/rejected": -0.3436921536922455, + "loss": 0.6893914341926575, + "loss/core": 0.6893914341926575, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.2814574241638184, + "rewards/margins": 1.542607069015503, + "rewards/rejected": 0.738849937915802, + "step": 825 + }, + { + "epoch": 0.7930252000477725, + "grad_norm": 0.2451171875, + "learning_rate": 9.419118673676924e-09, + "logits/chosen": -1.8093611001968384, + "logits/rejected": -1.790492057800293, + "logps/chosen": -0.2983977794647217, + "logps/rejected": -0.28687578439712524, + "loss": 0.6904627680778503, + "loss/core": 0.6904627680778503, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8845103979110718, + "rewards/margins": 1.0901850461959839, + "rewards/rejected": 0.7943253517150879, + "step": 830 + }, + { + "epoch": 0.7978024602890242, + "grad_norm": 0.193359375, + "learning_rate": 8.14619513428405e-09, + "logits/chosen": -1.8425931930541992, + "logits/rejected": -1.853624939918518, + "logps/chosen": -0.2532172203063965, + "logps/rejected": -0.2740766704082489, + "loss": 0.6899970173835754, + "loss/core": 0.6899970173835754, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.117034673690796, + "rewards/margins": 1.288293480873108, + "rewards/rejected": 0.8287409543991089, + "step": 835 + }, + { + "epoch": 0.8025797205302759, + "grad_norm": 1.3671875, + "learning_rate": 6.96422291980539e-09, + "logits/chosen": -1.739816427230835, + "logits/rejected": -1.735145926475525, + "logps/chosen": -0.2897264361381531, + "logps/rejected": -0.2904459238052368, + "loss": 0.6874421834945679, + "loss/core": 0.6874421834945679, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.566680431365967, + "rewards/margins": 2.4187304973602295, + "rewards/rejected": 1.147949457168579, + "step": 840 + }, + { + "epoch": 0.8073569807715275, + "grad_norm": 0.4609375, + "learning_rate": 5.8736465216517594e-09, + "logits/chosen": -1.5910301208496094, + "logits/rejected": -1.6553596258163452, + "logps/chosen": -0.3020200729370117, + "logps/rejected": -0.3062165379524231, + "loss": 0.6924393773078918, + "loss/core": 0.6924393773078918, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.691601276397705, + "rewards/margins": 0.3428772985935211, + "rewards/rejected": 2.348723888397217, + "step": 845 + }, + { + "epoch": 0.8121342410127792, + "grad_norm": 0.2890625, + "learning_rate": 4.874876061005173e-09, + "logits/chosen": -1.8377548456192017, + "logits/rejected": -1.8844318389892578, + "logps/chosen": -0.2867121696472168, + "logps/rejected": -0.29250529408454895, + "loss": 0.6900558471679688, + "loss/core": 0.6900558471679688, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.395780086517334, + "rewards/margins": 1.2476097345352173, + "rewards/rejected": 1.1481703519821167, + "step": 850 + }, + { + "epoch": 0.8169115012540308, + "grad_norm": 0.302734375, + "learning_rate": 3.968287134589188e-09, + "logits/chosen": -1.7527811527252197, + "logits/rejected": -1.7593837976455688, + "logps/chosen": -0.2845557630062103, + "logps/rejected": -0.28174346685409546, + "loss": 0.690881609916687, + "loss/core": 0.690881609916687, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.1602466106414795, + "rewards/margins": 0.9179478883743286, + "rewards/rejected": 1.2422987222671509, + "step": 855 + }, + { + "epoch": 0.8216887614952825, + "grad_norm": 0.177734375, + "learning_rate": 3.154220673422192e-09, + "logits/chosen": -1.7969452142715454, + "logits/rejected": -1.8759748935699463, + "logps/chosen": -0.2859663963317871, + "logps/rejected": -0.33153361082077026, + "loss": 0.6913708448410034, + "loss/core": 0.6913708448410034, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.9288288354873657, + "rewards/margins": 0.7228490114212036, + "rewards/rejected": 1.205979585647583, + "step": 860 + }, + { + "epoch": 0.826466021736534, + "grad_norm": 0.91796875, + "learning_rate": 2.4329828146074096e-09, + "logits/chosen": -1.6951520442962646, + "logits/rejected": -1.7336347103118896, + "logps/chosen": -0.25996583700180054, + "logps/rejected": -0.29030436277389526, + "loss": 0.6907094120979309, + "loss/core": 0.6907094120979309, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.2786126136779785, + "rewards/margins": 0.9912854433059692, + "rewards/rejected": 1.2873269319534302, + "step": 865 + }, + { + "epoch": 0.8312432819777857, + "grad_norm": 1.3828125, + "learning_rate": 1.8048447862070714e-09, + "logits/chosen": -1.884939432144165, + "logits/rejected": -1.8319370746612549, + "logps/chosen": -0.30168288946151733, + "logps/rejected": -0.3016626834869385, + "loss": 0.690568745136261, + "loss/core": 0.690568745136261, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.4173693656921387, + "rewards/margins": 1.0916273593902588, + "rewards/rejected": 1.3257420063018799, + "step": 870 + }, + { + "epoch": 0.8360205422190374, + "grad_norm": 0.400390625, + "learning_rate": 1.2700428052447033e-09, + "logits/chosen": -1.948432207107544, + "logits/rejected": -1.7609401941299438, + "logps/chosen": -0.27557021379470825, + "logps/rejected": -0.29795706272125244, + "loss": 0.6937984228134155, + "loss/core": 0.6937984228134155, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.6998504400253296, + "rewards/margins": -0.23256230354309082, + "rewards/rejected": 1.9324127435684204, + "step": 875 + }, + { + "epoch": 0.840797802460289, + "grad_norm": 0.2255859375, + "learning_rate": 8.287779888734858e-10, + "logits/chosen": -1.8939878940582275, + "logits/rejected": -1.914552092552185, + "logps/chosen": -0.307658851146698, + "logps/rejected": -0.3075157403945923, + "loss": 0.6896311044692993, + "loss/core": 0.6896311044692993, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.8853094577789307, + "rewards/margins": 1.4759790897369385, + "rewards/rejected": 1.409330129623413, + "step": 880 + }, + { + "epoch": 0.8455750627015407, + "grad_norm": 0.166015625, + "learning_rate": 4.812162787445062e-10, + "logits/chosen": -1.8575178384780884, + "logits/rejected": -1.897189736366272, + "logps/chosen": -0.30906686186790466, + "logps/rejected": -0.29951316118240356, + "loss": 0.6915870904922485, + "loss/core": 0.6915870904922485, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.1715130805969238, + "rewards/margins": 0.6273240447044373, + "rewards/rejected": 0.5441890954971313, + "step": 885 + }, + { + "epoch": 0.8503523229427923, + "grad_norm": 0.76171875, + "learning_rate": 2.2748837860270265e-10, + "logits/chosen": -1.9069395065307617, + "logits/rejected": -1.957416296005249, + "logps/chosen": -0.3348519206047058, + "logps/rejected": -0.3277924954891205, + "loss": 0.6899693012237549, + "loss/core": 0.6899693012237549, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.047415256500244, + "rewards/margins": 1.303015947341919, + "rewards/rejected": 0.7443991899490356, + "step": 890 + }, + { + "epoch": 0.855129583184044, + "grad_norm": 0.23828125, + "learning_rate": 6.768970513457151e-11, + "logits/chosen": -1.7169866561889648, + "logits/rejected": -1.6992985010147095, + "logps/chosen": -0.2847689688205719, + "logps/rejected": -0.27514931559562683, + "loss": 0.6903687715530396, + "loss/core": 0.6903687715530396, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.195614814758301, + "rewards/margins": 1.1351045370101929, + "rewards/rejected": 1.060510516166687, + "step": 895 + }, + { + "epoch": 0.8599068434252956, + "grad_norm": 0.5625, + "learning_rate": 1.8803520859811406e-12, + "logits/chosen": -1.8456823825836182, + "logits/rejected": -1.849439024925232, + "logps/chosen": -0.31287848949432373, + "logps/rejected": -0.31777575612068176, + "loss": 0.6863356828689575, + "loss/core": 0.6863356828689575, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.444868087768555, + "rewards/margins": 2.881870985031128, + "rewards/rejected": 1.5629966259002686, + "step": 900 + }, + { + "epoch": 0.8599068434252956, + "step": 900, + "total_flos": 0.0, + "train_loss": 0.6902858657307095, + "train_runtime": 8330.5288, + "train_samples_per_second": 1.729, + "train_steps_per_second": 0.108 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..c458f12 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f587a2430e01b95ada56f200656c069969c4ecab8137e307029ea9a7c9de883f +size 6993