From 30c78df99f93f4e85d476c90903c9df8e66c078a Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Thu, 23 Jul 2026 14:54:10 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: promotion/qwen3-8b-aaai27-flagship-ht-mnpo-helpfulness-s44 Source: Original Platform --- .gitattributes | 36 + README.md | 24 + all_results.json | 9 + chat_template.jinja | 89 + config.json | 71 + config.yaml | 58 + generation_config.json | 12 + job_status.json | 23 + model.safetensors | 3 + objective_protocol.json | 69 + provenance.json | 14 + resource_profile.json | 21 + run_status.json | 15 + split_manifest.json | 23 + stability_gate.json | 38 + tokenizer.json | 3 + tokenizer_config.json | 30 + train_results.json | 9 + trainer_state.json | 3823 +++++++++++++++++++++++++++++++++++++++ training_args.bin | 3 + 20 files changed, 4373 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 config.yaml create mode 100644 generation_config.json create mode 100644 job_status.json create mode 100644 model.safetensors create mode 100644 objective_protocol.json create mode 100644 provenance.json create mode 100644 resource_profile.json create mode 100644 run_status.json create mode 100644 split_manifest.json create mode 100644 stability_gate.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..df93f97 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +--- +library_name: transformers +base_model: Qwen/Qwen3-8B +tags: +- ronpo +- mnpo +- preference-optimization +--- + +# qwen3-8b-aaai27-flagship-ht-mnpo-helpfulness-s44 + +Research checkpoint for the RONPO AAAI revision experiments. + +- Method: ht_mnpo_helpfulness +- Base model: `Qwen/Qwen3-8B` +- Seed: 44 +- Local source at upload time: `/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/full/ht_mnpo_helpfulness/seed44/attempt1` +- Uploaded at UTC: 2026-07-13T13:49:40Z +- Run status metadata: `{"attempt": 1, "effective_batch_size": 16, "method": "ht_mnpo_helpfulness", "objective_protocol": "objective_protocol.json", "optimizer_steps": 900, "resource_profile": "resource_profile.json", "scope_amendment": "scope_amendment_20260713.json", "seed": 44, "split_manifest": "split_manifest.json", "stability_gate": "stability_gate.json", "status": "completed", "wandb_run_id": "57260c985236", "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/57260c985236"}` + +AAAI-27 P1 matched budget: 900 optimizer steps, effective batch 16; passed non-thinking and collapse stability gates; sealed-test metrics are not used for checkpoint selection. + +Intended use: reproducibility and evaluation for the RONPO paper. This +checkpoint is not intended as a production assistant. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..9426da6 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8600609209819029, + "total_flos": 0.0, + "train_loss": 0.3132047750718064, + "train_runtime": 7069.0811, + "train_samples": 16743, + "train_samples_per_second": 2.037, + "train_steps_per_second": 0.127 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..03b100d --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +model_name_or_path: Qwen/Qwen3-8B +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/ht_mnpo_helpfulness: 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: ht_mnpo +eta: 0.0075 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.0 +preference_sft_weight: 0.0 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 5.0e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 44 +gradient_accumulation_steps: 16 +gradient_checkpointing: true +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 1 +per_device_eval_batch_size: 1 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/full/ht_mnpo_helpfulness/seed44/attempt1 +run_name: aaai27-flagship-full-ht_mnpo_helpfulness-s44-a1 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/job_status.json b/job_status.json new file mode 100644 index 0000000..7ee2580 --- /dev/null +++ b/job_status.json @@ -0,0 +1,23 @@ +{ + "status": "completed", + "stage": "full", + "method": "ht_mnpo_helpfulness", + "seed": 44, + "attempt": 1, + "gpu": 3, + "gpus": [ + 3 + ], + "returncode": 0, + "finite_metrics": true, + "oom": false, + "traceback": false, + "reason": "passed", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "57260c985236", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/57260c985236", + "output_dir": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/full/ht_mnpo_helpfulness/seed44/attempt1", + "log": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/logs/full_ht_mnpo_helpfulness_s44_a1.log", + "completed_at": "2026-07-13T13:45:59Z" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..0972366 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d347b2d7bba6acefc91c7515fd6f442f8681e66c6966b33572f2d5e5593b076f +size 16381517208 diff --git a/objective_protocol.json b/objective_protocol.json new file mode 100644 index 0000000..35b3b44 --- /dev/null +++ b/objective_protocol.json @@ -0,0 +1,69 @@ +{ + "schema_version": 1, + "frozen_at_kst": "2026-07-12T21:40:00+09:00", + "base_model": "Qwen/Qwen3-8B", + "primary_triple": { + "helpfulness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "ultrafeedback-helpfulness", + "transform": "identity" + }, + "safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "conciseness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "helpsteer-verbosity", + "transform": "negate" + } + }, + "gate_thresholds": { + "raw_score_std_strictly_greater_than": 0.01, + "aggregate_prompt_pair_spearman_median_strictly_less_than": 0.0, + "aggregate_pairwise_top1_mismatch_at_least": 0.5 + }, + "gate_pool": "source-train Qwen3-8B response pool split into deterministic train/validation; sealed source-test excluded", + "training_seeds": [42, 43, 44], + "optimizer_steps": 900, + "effective_batch_size": 16, + "decode": { + "enable_thinking": false, + "temperature": 0.7, + "top_p": 0.9, + "max_new_tokens": 2048, + "decode_seed": 42 + }, + "methods": [ + "ronpo_full_expect", + "ronpo_k_only", + "dpo", + "ipo", + "simpo", + "kto", + "sppo_avg", + "inpo_avg", + "ht_mnpo_helpfulness", + "ht_mnpo_safety", + "ht_mnpo_conciseness" + ], + "selection_policy": "No sealed-test selection. One preregistered configuration per method; failures use only the declared stability retry recipe, at most three attempts.", + "stability_gate": { + "validation_prompts": 128, + "min_candidate_base_mean_word_ratio": 0.33, + "max_candidate_base_mean_word_ratio": 2.0, + "max_consecutive_identical_word_run": 20, + "think_tag_count": 0, + "empty_response_count": 0, + "retry_recipe": "attempt 2 halves learning rate, enables gradient checkpointing, and adds reference=0.02/SFT=0.002 anchoring to unanchored unified losses; attempt 3 quarters learning rate and uses reference=0.05/SFT=0.005; RONPO uses reference=0.05/SFT=0.005 from attempt 1" + }, + "primary_metric": "mean prompt-level minimum of per-prompt min-max-normalized helpfulness, safety, conciseness", + "bootstrap": { + "unit": "prompt", + "paired": true, + "resamples": 2000, + "interval": "percentile_95", + "seed": 42 + } +} diff --git a/provenance.json b/provenance.json new file mode 100644 index 0000000..3505427 --- /dev/null +++ b/provenance.json @@ -0,0 +1,14 @@ +{ + "method": "ht_mnpo_helpfulness", + "seed": 44, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "57260c985236", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/57260c985236", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json", + "scope_amendment": "scope_amendment_20260713.json" +} diff --git a/resource_profile.json b/resource_profile.json new file mode 100644 index 0000000..18eec47 --- /dev/null +++ b/resource_profile.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "measured_at_kst": "2026-07-12T22:38:00+09:00", + "hardware": "NVIDIA B200 183359 MiB reported total; 178.35 GiB PyTorch-visible capacity", + "smoke_evidence": { + "attempted_profile": {"per_device_batch_size": 2, "gradient_accumulation_steps": 8, "gradient_checkpointing": false, "effective_batch_size": 16}, + "outcome": "OOM before completing the 20-step smoke across every launched unified baseline; representative process used 176.87 GiB and failed a 1.62 GiB allocation. KTO also OOMed at the optimizer step.", + "paper_metric": false + }, + "selected_full_training_profile": { + "per_device_batch_size": 2, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": false, + "attention_implementation": "PyTorch SDPA with cuDNN SDPA disabled", + "effective_batch_size": 16, + "applies_symmetrically_to_all_unified_pairwise_methods": true, + "kto_implementation_constraint": "TRL KTO requires actual batch size > 1. KTO uses batch 2 / accumulation 8 and identical AdamW with foreach=False. PyTorch SDPA is used with cuDNN SDPA disabled. The fixed base reference model is placed on a second B200 and its exact logits are returned to the policy device; the policy/optimizer remain on the first B200. This avoids a measured ~6-hour reference-precompute path while leaving the KTO loss, actual/effective batch, optimizer, and 900-step budget unchanged." + }, + "retry_semantics": "Resource selection is separated from method-stability attempts. Full attempt 1 uses the smoke-validated SDPA batch-2 profile and retains the preregistered method LR/anchor; attempts 2 and 3 fall back to batch 1 / accumulation 16 / gradient checkpointing and apply the declared LR/anchor stabilization recipe.", + "provenance_note": "No reward or sealed-evaluation result was available when this hardware profile was selected." +} diff --git a/run_status.json b/run_status.json new file mode 100644 index 0000000..b0d4012 --- /dev/null +++ b/run_status.json @@ -0,0 +1,15 @@ +{ + "method": "ht_mnpo_helpfulness", + "seed": 44, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "57260c985236", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/57260c985236", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json", + "scope_amendment": "scope_amendment_20260713.json", + "status": "completed" +} diff --git a/split_manifest.json b/split_manifest.json new file mode 100644 index 0000000..4fb96bd --- /dev/null +++ b/split_manifest.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "split_rule": "sort source-train prompts by sha256('flagship-v1||'+prompt_id); first round(10%) validation", + "validation_fraction": 0.1, + "source_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/avg_oracle", + "base_model": "Qwen/Qwen3-8B", + "counts": { + "train": 16780, + "validation": 1864, + "sealed_test": 604 + }, + "prompt_id_sha256": { + "train": "fcf6f2ea5a0b7bcd41179e6f5288f4d850b2d04bdb729da91907bd18bf733a69", + "validation": "a87ba6f519192164daa487b109a6547dd8cbde46a27a373d6da201ca2d061b8f", + "sealed_test": "b398ced6af6cf73d731e1a5f32a0631d8060323d51eda1e375695566bf5f1867" + }, + "overlaps": { + "train_validation": 0, + "train_sealed_test": 0, + "validation_sealed_test": 0 + }, + "sealed_test_policy": "prompts only until S4a; no objective scores or model generations used in S0-S3" +} diff --git a/stability_gate.json b/stability_gate.json new file mode 100644 index 0000000..9d60f84 --- /dev/null +++ b/stability_gate.json @@ -0,0 +1,38 @@ +{ + "status": "passed", + "passed": true, + "base": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 388.1484375, + "max_words": 1379, + "max_repeat_run": 4 + }, + "candidate": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 396.5, + "max_words": 1355, + "max_repeat_run": 6 + }, + "candidate_base_mean_word_ratio": 1.0215164140651731, + "thresholds": { + "min_length_ratio": 0.33, + "max_length_ratio": 2.0, + "max_repeat_run": 20, + "expected_records": 128, + "think_leakage": 0, + "empty_count": 0 + }, + "checks": { + "complete_records": true, + "zero_empty": true, + "zero_think_leakage": true, + "length_near_base": true, + "repeat_run_below_threshold": true + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ea3c9e2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..9426da6 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8600609209819029, + "total_flos": 0.0, + "train_loss": 0.3132047750718064, + "train_runtime": 7069.0811, + "train_samples": 16743, + "train_samples_per_second": 2.037, + "train_steps_per_second": 0.127 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..93e0883 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,3823 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8600609209819029, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.004778116227677238, + "grad_norm": 0.9296875, + "ht_mnpo/logit": 0.05088333040475845, + "ht_mnpo/residual": 0.04338332638144493, + "ht_mnpo/residual_abs": 0.19410482048988342, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.222222222222222e-08, + "logits/chosen": -2.322283983230591, + "logits/rejected": -2.095109224319458, + "logps/chosen": -0.3028138279914856, + "logps/rejected": -0.31887882947921753, + "loss": 0.7703814506530762, + "loss/core": 0.7703814506530762, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8105131387710571, + "rewards/margins": 0.5088331699371338, + "rewards/rejected": 1.3016799688339233, + "step": 5 + }, + { + "epoch": 0.009556232455354476, + "grad_norm": 104.0, + "ht_mnpo/logit": 0.030571451410651207, + "ht_mnpo/residual": 0.02307145670056343, + "ht_mnpo/residual_abs": 0.12138247489929199, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5e-08, + "logits/chosen": -2.109570026397705, + "logits/rejected": -1.8389272689819336, + "logps/chosen": -0.3074098527431488, + "logps/rejected": -0.32182925939559937, + "loss": 0.11068801581859589, + "loss/core": 0.11068801581859589, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": 2.1581244468688965, + "rewards/margins": 0.30571448802948, + "rewards/rejected": 1.8524096012115479, + "step": 10 + }, + { + "epoch": 0.014334348683031715, + "grad_norm": 63.25, + "ht_mnpo/logit": 0.03353700414299965, + "ht_mnpo/residual": 0.026036998257040977, + "ht_mnpo/residual_abs": 0.22524623572826385, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 7.777777777777778e-08, + "logits/chosen": -2.1958093643188477, + "logits/rejected": -1.9058806896209717, + "logps/chosen": -0.27820906043052673, + "logps/rejected": -0.2950650751590729, + "loss": 0.6725027561187744, + "loss/core": 0.6725027561187744, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 2.2042908668518066, + "rewards/margins": 0.33536994457244873, + "rewards/rejected": 1.8689210414886475, + "step": 15 + }, + { + "epoch": 0.019112464910708952, + "grad_norm": 2.203125, + "ht_mnpo/logit": 0.060489214956760406, + "ht_mnpo/residual": 0.05298921465873718, + "ht_mnpo/residual_abs": 0.06931939721107483, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0555555555555555e-07, + "logits/chosen": -1.9993699789047241, + "logits/rejected": -1.799395203590393, + "logps/chosen": -0.3109528720378876, + "logps/rejected": -0.31548988819122314, + "loss": 0.02609763666987419, + "loss/core": 0.02609763666987419, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.7414203882217407, + "rewards/margins": 0.6048921346664429, + "rewards/rejected": 1.1365283727645874, + "step": 20 + }, + { + "epoch": 0.023890581138386192, + "grad_norm": 114.5, + "ht_mnpo/logit": 0.09344500303268433, + "ht_mnpo/residual": 0.0859449952840805, + "ht_mnpo/residual_abs": 0.14693133533000946, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.3333333333333334e-07, + "logits/chosen": -2.228485345840454, + "logits/rejected": -1.9559946060180664, + "logps/chosen": -0.2875242233276367, + "logps/rejected": -0.2956809997558594, + "loss": 0.2483440339565277, + "loss/core": 0.2483440339565277, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.1028735637664795, + "rewards/margins": 0.9344498515129089, + "rewards/rejected": 1.1684236526489258, + "step": 25 + }, + { + "epoch": 0.02866869736606343, + "grad_norm": 5.4375, + "ht_mnpo/logit": 0.0791626125574112, + "ht_mnpo/residual": 0.07166260480880737, + "ht_mnpo/residual_abs": 0.16665640473365784, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.611111111111111e-07, + "logits/chosen": -2.1653201580047607, + "logits/rejected": -1.931023359298706, + "logps/chosen": -0.26770323514938354, + "logps/rejected": -0.28871315717697144, + "loss": 0.3910994529724121, + "loss/core": 0.3910994529724121, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 2.4544098377227783, + "rewards/margins": 0.7916260957717896, + "rewards/rejected": 1.6627838611602783, + "step": 30 + }, + { + "epoch": 0.03344681359374067, + "grad_norm": 56.25, + "ht_mnpo/logit": 0.09365877509117126, + "ht_mnpo/residual": 0.08615876734256744, + "ht_mnpo/residual_abs": 0.14225038886070251, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8888888888888888e-07, + "logits/chosen": -2.2201991081237793, + "logits/rejected": -1.9280380010604858, + "logps/chosen": -0.28348293900489807, + "logps/rejected": -0.28683170676231384, + "loss": 0.14075805246829987, + "loss/core": 0.14075805246829987, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.5023605823516846, + "rewards/margins": 0.9365876317024231, + "rewards/rejected": 1.5657732486724854, + "step": 35 + }, + { + "epoch": 0.038224929821417904, + "grad_norm": 3.6875, + "ht_mnpo/logit": 0.023442532867193222, + "ht_mnpo/residual": 0.015942532569169998, + "ht_mnpo/residual_abs": 0.15496210753917694, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.1666666666666667e-07, + "logits/chosen": -2.354321002960205, + "logits/rejected": -1.9975166320800781, + "logps/chosen": -0.2890976667404175, + "logps/rejected": -0.30953389406204224, + "loss": 0.2759542167186737, + "loss/core": 0.2759542167186737, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 1.8200693130493164, + "rewards/margins": 0.23442523181438446, + "rewards/rejected": 1.585644006729126, + "step": 40 + }, + { + "epoch": 0.04300304604909514, + "grad_norm": 1.9140625, + "ht_mnpo/logit": 0.1023062914609909, + "ht_mnpo/residual": 0.09480629861354828, + "ht_mnpo/residual_abs": 0.10691990703344345, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4444444444444445e-07, + "logits/chosen": -2.300888776779175, + "logits/rejected": -2.1800649166107178, + "logps/chosen": -0.294630229473114, + "logps/rejected": -0.2884281277656555, + "loss": 0.06364284455776215, + "loss/core": 0.06364284455776215, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.0192909240722656, + "rewards/margins": 1.0230629444122314, + "rewards/rejected": 0.9962279200553894, + "step": 45 + }, + { + "epoch": 0.047781162276772385, + "grad_norm": 0.93359375, + "ht_mnpo/logit": 0.05063255876302719, + "ht_mnpo/residual": 0.04313255473971367, + "ht_mnpo/residual_abs": 0.056837163865566254, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.7222222222222216e-07, + "logits/chosen": -2.179457902908325, + "logits/rejected": -1.9664961099624634, + "logps/chosen": -0.2966025769710541, + "logps/rejected": -0.30326494574546814, + "loss": 0.02287248522043228, + "loss/core": 0.02287248522043228, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.278033971786499, + "rewards/margins": 0.5063256025314331, + "rewards/rejected": 0.771708607673645, + "step": 50 + }, + { + "epoch": 0.05255927850444962, + "grad_norm": 0.90234375, + "ht_mnpo/logit": 0.10425949096679688, + "ht_mnpo/residual": 0.09675948321819305, + "ht_mnpo/residual_abs": 0.1356755793094635, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3e-07, + "logits/chosen": -2.292445182800293, + "logits/rejected": -2.0353176593780518, + "logps/chosen": -0.29430681467056274, + "logps/rejected": -0.28913360834121704, + "loss": 0.3527126908302307, + "loss/core": 0.3527126908302307, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.0845179557800293, + "rewards/margins": 1.0425949096679688, + "rewards/rejected": 1.04192316532135, + "step": 55 + }, + { + "epoch": 0.05733739473212686, + "grad_norm": 13.3125, + "ht_mnpo/logit": 0.17591974139213562, + "ht_mnpo/residual": 0.168419748544693, + "ht_mnpo/residual_abs": 0.17273147404193878, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.2777777777777776e-07, + "logits/chosen": -1.9740009307861328, + "logits/rejected": -1.7925126552581787, + "logps/chosen": -0.27490484714508057, + "logps/rejected": -0.29008954763412476, + "loss": 0.24429097771644592, + "loss/core": 0.24429097771644592, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.951267719268799, + "rewards/margins": 1.759197473526001, + "rewards/rejected": 1.1920703649520874, + "step": 60 + }, + { + "epoch": 0.062115510959804096, + "grad_norm": 87.0, + "ht_mnpo/logit": -0.027438348159193993, + "ht_mnpo/residual": -0.03493834659457207, + "ht_mnpo/residual_abs": 0.11112980544567108, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.5555555555555553e-07, + "logits/chosen": -2.2002429962158203, + "logits/rejected": -2.0315022468566895, + "logps/chosen": -0.28836536407470703, + "logps/rejected": -0.29485026001930237, + "loss": 0.34929710626602173, + "loss/core": 0.34929710626602173, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.0422712564468384, + "rewards/margins": -0.2743833363056183, + "rewards/rejected": 1.3166545629501343, + "step": 65 + }, + { + "epoch": 0.06689362718748133, + "grad_norm": 1.8125, + "ht_mnpo/logit": -0.009448112919926643, + "ht_mnpo/residual": -0.01694810949265957, + "ht_mnpo/residual_abs": 0.10872050374746323, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.8333333333333335e-07, + "logits/chosen": -2.333392858505249, + "logits/rejected": -2.1000077724456787, + "logps/chosen": -0.2741190791130066, + "logps/rejected": -0.2725319266319275, + "loss": 0.1955917775630951, + "loss/core": 0.1955917775630951, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 1.531646490097046, + "rewards/margins": -0.09448114782571793, + "rewards/rejected": 1.6261276006698608, + "step": 70 + }, + { + "epoch": 0.07167174341515857, + "grad_norm": 79.5, + "ht_mnpo/logit": 0.09921145439147949, + "ht_mnpo/residual": 0.09171145409345627, + "ht_mnpo/residual_abs": 0.1735668033361435, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.1111111111111107e-07, + "logits/chosen": -2.177030324935913, + "logits/rejected": -1.9694257974624634, + "logps/chosen": -0.30770665407180786, + "logps/rejected": -0.30710548162460327, + "loss": 0.26593199372291565, + "loss/core": 0.26593199372291565, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.6823678016662598, + "rewards/margins": 0.9921146631240845, + "rewards/rejected": 1.6902530193328857, + "step": 75 + }, + { + "epoch": 0.07644985964283581, + "grad_norm": 15.5625, + "ht_mnpo/logit": 0.028042754158377647, + "ht_mnpo/residual": 0.02054273523390293, + "ht_mnpo/residual_abs": 0.27549320459365845, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.3888888888888884e-07, + "logits/chosen": -2.109937906265259, + "logits/rejected": -1.8973028659820557, + "logps/chosen": -0.3108934760093689, + "logps/rejected": -0.3061710000038147, + "loss": 1.2482848167419434, + "loss/core": 1.2482848167419434, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 2.4044840335845947, + "rewards/margins": 0.28042739629745483, + "rewards/rejected": 2.124056577682495, + "step": 80 + }, + { + "epoch": 0.08122797587051304, + "grad_norm": 2.625, + "ht_mnpo/logit": 0.04282355681061745, + "ht_mnpo/residual": 0.03532355651259422, + "ht_mnpo/residual_abs": 0.05199320241808891, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.6666666666666666e-07, + "logits/chosen": -2.1569108963012695, + "logits/rejected": -1.9936288595199585, + "logps/chosen": -0.29405292868614197, + "logps/rejected": -0.2927798628807068, + "loss": 0.015568750910460949, + "loss/core": 0.015568750910460949, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.2469490766525269, + "rewards/margins": 0.42823559045791626, + "rewards/rejected": 0.8187135457992554, + "step": 85 + }, + { + "epoch": 0.08600609209819028, + "grad_norm": 21.25, + "ht_mnpo/logit": 0.09608729183673859, + "ht_mnpo/residual": 0.08858727663755417, + "ht_mnpo/residual_abs": 0.12235699594020844, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.944444444444445e-07, + "logits/chosen": -2.227970600128174, + "logits/rejected": -2.119290351867676, + "logps/chosen": -0.26927095651626587, + "logps/rejected": -0.2766638994216919, + "loss": 0.09564952552318573, + "loss/core": 0.09564952552318573, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.044128179550171, + "rewards/margins": 0.9608729481697083, + "rewards/rejected": 1.0832551717758179, + "step": 90 + }, + { + "epoch": 0.09078420832586753, + "grad_norm": 8.6875, + "ht_mnpo/logit": -0.01110445149242878, + "ht_mnpo/residual": -0.018604444339871407, + "ht_mnpo/residual_abs": 0.14139024913311005, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.999699149323369e-07, + "logits/chosen": -2.525456190109253, + "logits/rejected": -2.102797746658325, + "logps/chosen": -0.27081310749053955, + "logps/rejected": -0.271461546421051, + "loss": 0.31182369589805603, + "loss/core": 0.31182369589805603, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 1.4296600818634033, + "rewards/margins": -0.11104442924261093, + "rewards/rejected": 1.540704607963562, + "step": 95 + }, + { + "epoch": 0.09556232455354477, + "grad_norm": 222.0, + "ht_mnpo/logit": -0.07714849710464478, + "ht_mnpo/residual": -0.0846485048532486, + "ht_mnpo/residual_abs": 0.18699464201927185, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.998477067547739e-07, + "logits/chosen": -2.0692691802978516, + "logits/rejected": -1.7881990671157837, + "logps/chosen": -0.27515479922294617, + "logps/rejected": -0.30582091212272644, + "loss": 0.3758378028869629, + "loss/core": 0.3758378028869629, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 1.941310167312622, + "rewards/margins": -0.7714849710464478, + "rewards/rejected": 2.7127950191497803, + "step": 100 + }, + { + "epoch": 0.100340440781222, + "grad_norm": 1.6640625, + "ht_mnpo/logit": 0.08836665749549866, + "ht_mnpo/residual": 0.08086666464805603, + "ht_mnpo/residual_abs": 0.1014769896864891, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.996315410727229e-07, + "logits/chosen": -2.1733365058898926, + "logits/rejected": -1.8746709823608398, + "logps/chosen": -0.29187583923339844, + "logps/rejected": -0.2903345227241516, + "loss": 0.12463197857141495, + "loss/core": 0.12463197857141495, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": 2.0642318725585938, + "rewards/margins": 0.8836666941642761, + "rewards/rejected": 1.1805652379989624, + "step": 105 + }, + { + "epoch": 0.10511855700889924, + "grad_norm": 19.125, + "ht_mnpo/logit": 0.08112769573926926, + "ht_mnpo/residual": 0.07362769544124603, + "ht_mnpo/residual_abs": 0.09633813798427582, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.993214991772562e-07, + "logits/chosen": -2.219769239425659, + "logits/rejected": -1.847038984298706, + "logps/chosen": -0.27057093381881714, + "logps/rejected": -0.27792128920555115, + "loss": 0.04170053079724312, + "loss/core": 0.04170053079724312, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 2.387577772140503, + "rewards/margins": 0.8112770318984985, + "rewards/rejected": 1.576300859451294, + "step": 110 + }, + { + "epoch": 0.10989667323657648, + "grad_norm": 24.875, + "ht_mnpo/logit": 0.07629410922527313, + "ht_mnpo/residual": 0.06879410147666931, + "ht_mnpo/residual_abs": 0.17269201576709747, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.989176976624511e-07, + "logits/chosen": -2.1905808448791504, + "logits/rejected": -1.9591457843780518, + "logps/chosen": -0.2870820164680481, + "logps/rejected": -0.29649895429611206, + "loss": 0.2637905478477478, + "loss/core": 0.2637905478477478, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.930060863494873, + "rewards/margins": 0.7629408836364746, + "rewards/rejected": 2.1671202182769775, + "step": 115 + }, + { + "epoch": 0.11467478946425372, + "grad_norm": 193.0, + "ht_mnpo/logit": -0.001218314515426755, + "ht_mnpo/residual": -0.00871831364929676, + "ht_mnpo/residual_abs": 0.10168764740228653, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.984202883815428e-07, + "logits/chosen": -2.316711664199829, + "logits/rejected": -2.1473333835601807, + "logps/chosen": -0.28898805379867554, + "logps/rejected": -0.2972700595855713, + "loss": 0.08895155042409897, + "loss/core": 0.08895155042409897, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.3815490007400513, + "rewards/margins": -0.012183129787445068, + "rewards/rejected": 1.3937320709228516, + "step": 120 + }, + { + "epoch": 0.11945290569193096, + "grad_norm": 8.375, + "ht_mnpo/logit": 0.03216388076543808, + "ht_mnpo/residual": 0.024663884192705154, + "ht_mnpo/residual_abs": 0.04822679981589317, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.978294583898195e-07, + "logits/chosen": -2.516610622406006, + "logits/rejected": -2.1666038036346436, + "logps/chosen": -0.2692713737487793, + "logps/rejected": -0.27771317958831787, + "loss": 0.009285898879170418, + "loss/core": 0.009285898879170418, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 1.463373064994812, + "rewards/margins": 0.32163888216018677, + "rewards/rejected": 1.1417343616485596, + "step": 125 + }, + { + "epoch": 0.12423102191960819, + "grad_norm": 13.0, + "ht_mnpo/logit": 0.028603965416550636, + "ht_mnpo/residual": 0.021103959530591965, + "ht_mnpo/residual_abs": 0.10350266844034195, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.971454298742779e-07, + "logits/chosen": -2.189800977706909, + "logits/rejected": -1.9241540431976318, + "logps/chosen": -0.28411176800727844, + "logps/rejected": -0.2913844585418701, + "loss": 0.07367049157619476, + "loss/core": 0.07367049157619476, + "rewards/accuracies": 0.75, + "rewards/chosen": 1.4351568222045898, + "rewards/margins": 0.2860395610332489, + "rewards/rejected": 1.149117350578308, + "step": 130 + }, + { + "epoch": 0.12900913814728543, + "grad_norm": 102.0, + "ht_mnpo/logit": 0.07047243416309357, + "ht_mnpo/residual": 0.06297242641448975, + "ht_mnpo/residual_abs": 0.17785733938217163, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.963684600700678e-07, + "logits/chosen": -2.143766403198242, + "logits/rejected": -1.9228655099868774, + "logps/chosen": -0.26399558782577515, + "logps/rejected": -0.27883967757225037, + "loss": 0.3106807768344879, + "loss/core": 0.3106807768344879, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.427741765975952, + "rewards/margins": 0.7047242522239685, + "rewards/rejected": 1.723017692565918, + "step": 135 + }, + { + "epoch": 0.13378725437496267, + "grad_norm": 17.875, + "ht_mnpo/logit": 0.12879535555839539, + "ht_mnpo/residual": 0.12129535526037216, + "ht_mnpo/residual_abs": 0.12546807527542114, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.954988411637571e-07, + "logits/chosen": -2.2192699909210205, + "logits/rejected": -2.05328369140625, + "logps/chosen": -0.28640010952949524, + "logps/rejected": -0.2853923439979553, + "loss": 0.17789167165756226, + "loss/core": 0.17789167165756226, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.192383289337158, + "rewards/margins": 1.2879533767700195, + "rewards/rejected": 0.9044297337532043, + "step": 140 + }, + { + "epoch": 0.1385653706026399, + "grad_norm": 13.9375, + "ht_mnpo/logit": 0.09813486784696579, + "ht_mnpo/residual": 0.09063487499952316, + "ht_mnpo/residual_abs": 0.16477659344673157, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.945369001834514e-07, + "logits/chosen": -2.2470571994781494, + "logits/rejected": -1.975572943687439, + "logps/chosen": -0.26441556215286255, + "logps/rejected": -0.25484567880630493, + "loss": 0.24567930400371552, + "loss/core": 0.24567930400371552, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.544703960418701, + "rewards/margins": 0.9813488125801086, + "rewards/rejected": 1.5633552074432373, + "step": 145 + }, + { + "epoch": 0.14334348683031714, + "grad_norm": 15.25, + "ht_mnpo/logit": 0.13078755140304565, + "ht_mnpo/residual": 0.12328755855560303, + "ht_mnpo/residual_abs": 0.14581966400146484, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.93482998875813e-07, + "logits/chosen": -2.325977325439453, + "logits/rejected": -2.084429979324341, + "logps/chosen": -0.29626038670539856, + "logps/rejected": -0.3051699995994568, + "loss": 0.28483083844184875, + "loss/core": 0.28483083844184875, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.152726888656616, + "rewards/margins": 1.307875394821167, + "rewards/rejected": 0.8448513746261597, + "step": 150 + }, + { + "epoch": 0.14812160305799438, + "grad_norm": 6.1875, + "ht_mnpo/logit": 0.0646345466375351, + "ht_mnpo/residual": 0.05713455006480217, + "ht_mnpo/residual_abs": 0.13885614275932312, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.923375335700223e-07, + "logits/chosen": -2.069967269897461, + "logits/rejected": -1.802931547164917, + "logps/chosen": -0.26899904012680054, + "logps/rejected": -0.27681413292884827, + "loss": 0.12919534742832184, + "loss/core": 0.12919534742832184, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.119464874267578, + "rewards/margins": 0.6463454961776733, + "rewards/rejected": 1.4731194972991943, + "step": 155 + }, + { + "epoch": 0.15289971928567161, + "grad_norm": 282.0, + "ht_mnpo/logit": 0.07195184379816055, + "ht_mnpo/residual": 0.06445185840129852, + "ht_mnpo/residual_abs": 0.16485223174095154, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.911009350287347e-07, + "logits/chosen": -2.126732110977173, + "logits/rejected": -1.9010658264160156, + "logps/chosen": -0.3052712678909302, + "logps/rejected": -0.3145846426486969, + "loss": 0.3342359960079193, + "loss/core": 0.3342359960079193, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.308896541595459, + "rewards/margins": 0.7195185422897339, + "rewards/rejected": 1.589377760887146, + "step": 160 + }, + { + "epoch": 0.15767783551334885, + "grad_norm": 16.25, + "ht_mnpo/logit": 0.13453207910060883, + "ht_mnpo/residual": 0.1270320862531662, + "ht_mnpo/residual_abs": 0.17115262150764465, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.897736682860885e-07, + "logits/chosen": -2.2729172706604004, + "logits/rejected": -1.9023488759994507, + "logps/chosen": -0.26925191283226013, + "logps/rejected": -0.28397589921951294, + "loss": 0.7371556162834167, + "loss/core": 0.7371556162834167, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.71228289604187, + "rewards/margins": 1.3453209400177002, + "rewards/rejected": 1.3669623136520386, + "step": 165 + }, + { + "epoch": 0.1624559517410261, + "grad_norm": 26.25, + "ht_mnpo/logit": 0.11945419013500214, + "ht_mnpo/residual": 0.11195418983697891, + "ht_mnpo/residual_abs": 0.14006376266479492, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.883562324728241e-07, + "logits/chosen": -2.131185531616211, + "logits/rejected": -1.9534437656402588, + "logps/chosen": -0.28382784128189087, + "logps/rejected": -0.2792891263961792, + "loss": 0.2930060029029846, + "loss/core": 0.2930060029029846, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.2859272956848145, + "rewards/margins": 1.1945419311523438, + "rewards/rejected": 1.0913852453231812, + "step": 170 + }, + { + "epoch": 0.16723406796870333, + "grad_norm": 5.625, + "ht_mnpo/logit": 0.039273008704185486, + "ht_mnpo/residual": 0.03177300840616226, + "ht_mnpo/residual_abs": 0.06933975219726562, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.868491606285823e-07, + "logits/chosen": -2.2991464138031006, + "logits/rejected": -2.073848247528076, + "logps/chosen": -0.2741314768791199, + "logps/rejected": -0.276893675327301, + "loss": 0.032467905431985855, + "loss/core": 0.032467905431985855, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.6739985942840576, + "rewards/margins": 0.39273008704185486, + "rewards/rejected": 1.2812687158584595, + "step": 175 + }, + { + "epoch": 0.17201218419638056, + "grad_norm": 17.75, + "ht_mnpo/logit": 0.09057626128196716, + "ht_mnpo/residual": 0.08307625353336334, + "ht_mnpo/residual_abs": 0.18893863260746002, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.852530195014489e-07, + "logits/chosen": -2.2697224617004395, + "logits/rejected": -2.026869058609009, + "logps/chosen": -0.2926976680755615, + "logps/rejected": -0.3049258589744568, + "loss": 0.3669869601726532, + "loss/core": 0.3669869601726532, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 2.28019380569458, + "rewards/margins": 0.9057625532150269, + "rewards/rejected": 1.3744308948516846, + "step": 180 + }, + { + "epoch": 0.17679030042405783, + "grad_norm": 0.76171875, + "ht_mnpo/logit": 0.036733776330947876, + "ht_mnpo/residual": 0.02923377975821495, + "ht_mnpo/residual_abs": 0.051221203058958054, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.835684093348244e-07, + "logits/chosen": -2.279662847518921, + "logits/rejected": -2.0322604179382324, + "logps/chosen": -0.3091420829296112, + "logps/rejected": -0.325103759765625, + "loss": 0.01263344008475542, + "loss/core": 0.01263344008475542, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 1.3614156246185303, + "rewards/margins": 0.36733779311180115, + "rewards/rejected": 0.9940778613090515, + "step": 185 + }, + { + "epoch": 0.18156841665173507, + "grad_norm": 38.25, + "ht_mnpo/logit": 0.15630559623241425, + "ht_mnpo/residual": 0.14880560338497162, + "ht_mnpo/residual_abs": 0.1863393634557724, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.817959636416969e-07, + "logits/chosen": -2.1544783115386963, + "logits/rejected": -1.9566150903701782, + "logps/chosen": -0.28683212399482727, + "logps/rejected": -0.2882298231124878, + "loss": 0.36742496490478516, + "loss/core": 0.36742496490478516, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.6428446769714355, + "rewards/margins": 1.5630559921264648, + "rewards/rejected": 1.0797886848449707, + "step": 190 + }, + { + "epoch": 0.1863465328794123, + "grad_norm": 16.75, + "ht_mnpo/logit": 0.06805960834026337, + "ht_mnpo/residual": 0.06055961176753044, + "ht_mnpo/residual_abs": 0.09312574565410614, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.799363489664039e-07, + "logits/chosen": -2.1282095909118652, + "logits/rejected": -1.9009084701538086, + "logps/chosen": -0.30094921588897705, + "logps/rejected": -0.28223350644111633, + "loss": 0.059374578297138214, + "loss/core": 0.059374578297138214, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 2.1673290729522705, + "rewards/margins": 0.6805960536003113, + "rewards/rejected": 1.4867329597473145, + "step": 195 + }, + { + "epoch": 0.19112464910708954, + "grad_norm": 144.0, + "ht_mnpo/logit": 0.10441961139440536, + "ht_mnpo/residual": 0.09691962599754333, + "ht_mnpo/residual_abs": 0.16735684871673584, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.779902646339721e-07, + "logits/chosen": -2.2485835552215576, + "logits/rejected": -2.009777545928955, + "logps/chosen": -0.2734311521053314, + "logps/rejected": -0.2917312681674957, + "loss": 0.35473141074180603, + "loss/core": 0.35473141074180603, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.6118438243865967, + "rewards/margins": 1.0441962480545044, + "rewards/rejected": 1.5676480531692505, + "step": 200 + }, + { + "epoch": 0.19590276533476678, + "grad_norm": 14.375, + "ht_mnpo/logit": 0.11748448759317398, + "ht_mnpo/residual": 0.10998449474573135, + "ht_mnpo/residual_abs": 0.1774880737066269, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.759584424871301e-07, + "logits/chosen": -2.0733883380889893, + "logits/rejected": -1.8939872980117798, + "logps/chosen": -0.32192596793174744, + "logps/rejected": -0.316753089427948, + "loss": 0.2780437469482422, + "loss/core": 0.2780437469482422, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.506334066390991, + "rewards/margins": 1.1748448610305786, + "rewards/rejected": 1.3314893245697021, + "step": 205 + }, + { + "epoch": 0.200680881562444, + "grad_norm": 4.09375, + "ht_mnpo/logit": -0.01023789867758751, + "ht_mnpo/residual": -0.017737898975610733, + "ht_mnpo/residual_abs": 0.08284340798854828, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.738416466110917e-07, + "logits/chosen": -2.1783037185668945, + "logits/rejected": -1.9191534519195557, + "logps/chosen": -0.3014274537563324, + "logps/rejected": -0.30610206723213196, + "loss": 0.0633908361196518, + "loss/core": 0.0633908361196518, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.9192719459533691, + "rewards/margins": -0.10237900167703629, + "rewards/rejected": 1.021651029586792, + "step": 210 + }, + { + "epoch": 0.20545899779012125, + "grad_norm": 20.25, + "ht_mnpo/logit": 0.04202476888895035, + "ht_mnpo/residual": 0.03452477231621742, + "ht_mnpo/residual_abs": 0.07536228746175766, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.716406730462153e-07, + "logits/chosen": -2.296433210372925, + "logits/rejected": -2.0917699337005615, + "logps/chosen": -0.28844669461250305, + "logps/rejected": -0.30069172382354736, + "loss": 0.03301898017525673, + "loss/core": 0.03301898017525673, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.5373624563217163, + "rewards/margins": 0.4202477037906647, + "rewards/rejected": 1.117114782333374, + "step": 215 + }, + { + "epoch": 0.2102371140177985, + "grad_norm": 241.0, + "ht_mnpo/logit": 0.13244636356830597, + "ht_mnpo/residual": 0.12494637072086334, + "ht_mnpo/residual_abs": 0.161330446600914, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.693563494886454e-07, + "logits/chosen": -2.25962495803833, + "logits/rejected": -2.10876727104187, + "logps/chosen": -0.3020437955856323, + "logps/rejected": -0.30034393072128296, + "loss": 0.20618359744548798, + "loss/core": 0.20618359744548798, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.1069769859313965, + "rewards/margins": 1.3244636058807373, + "rewards/rejected": 1.7825132608413696, + "step": 220 + }, + { + "epoch": 0.21501523024547572, + "grad_norm": 0.1259765625, + "ht_mnpo/logit": 0.03315456956624985, + "ht_mnpo/residual": 0.025654572993516922, + "ht_mnpo/residual_abs": 0.04697638005018234, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.6698953497905016e-07, + "logits/chosen": -2.288386583328247, + "logits/rejected": -2.069204807281494, + "logps/chosen": -0.308716356754303, + "logps/rejected": -0.3130949139595032, + "loss": 0.011724801734089851, + "loss/core": 0.011724801734089851, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.28463613986969, + "rewards/margins": 0.33154574036598206, + "rewards/rejected": 0.9530903697013855, + "step": 225 + }, + { + "epoch": 0.21979334647315296, + "grad_norm": 2.8125, + "ht_mnpo/logit": 0.0781010240316391, + "ht_mnpo/residual": 0.07060102373361588, + "ht_mnpo/residual_abs": 0.14166823029518127, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.645411195795709e-07, + "logits/chosen": -2.4182097911834717, + "logits/rejected": -2.011842966079712, + "logps/chosen": -0.2857847809791565, + "logps/rejected": -0.2783315181732178, + "loss": 0.1992371380329132, + "loss/core": 0.1992371380329132, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 2.4587364196777344, + "rewards/margins": 0.7810102105140686, + "rewards/rejected": 1.6777263879776, + "step": 230 + }, + { + "epoch": 0.2245714627008302, + "grad_norm": 3.3125, + "ht_mnpo/logit": 0.13024064898490906, + "ht_mnpo/residual": 0.12274064868688583, + "ht_mnpo/residual_abs": 0.15162988007068634, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.6201202403910643e-07, + "logits/chosen": -2.106680393218994, + "logits/rejected": -1.7685272693634033, + "logps/chosen": -0.3049893379211426, + "logps/rejected": -0.3265388607978821, + "loss": 0.2962871193885803, + "loss/core": 0.2962871193885803, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 2.6692185401916504, + "rewards/margins": 1.3024064302444458, + "rewards/rejected": 1.366811990737915, + "step": 235 + }, + { + "epoch": 0.22934957892850744, + "grad_norm": 1.015625, + "ht_mnpo/logit": -0.012782636098563671, + "ht_mnpo/residual": -0.02028263546526432, + "ht_mnpo/residual_abs": 0.10213587433099747, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.594031994470573e-07, + "logits/chosen": -2.342169761657715, + "logits/rejected": -1.9928566217422485, + "logps/chosen": -0.2670109272003174, + "logps/rejected": -0.27326709032058716, + "loss": 0.09348958730697632, + "loss/core": 0.09348958730697632, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 1.772402048110962, + "rewards/margins": -0.12782634794712067, + "rewards/rejected": 1.900228500366211, + "step": 240 + }, + { + "epoch": 0.23412769515618467, + "grad_norm": 0.66015625, + "ht_mnpo/logit": 0.038539618253707886, + "ht_mnpo/residual": 0.03103962168097496, + "ht_mnpo/residual_abs": 0.07602906227111816, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.567156268756593e-07, + "logits/chosen": -2.4512276649475098, + "logits/rejected": -2.143380880355835, + "logps/chosen": -0.2683179974555969, + "logps/rejected": -0.2903439998626709, + "loss": 0.0462082214653492, + "loss/core": 0.0462082214653492, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.954590082168579, + "rewards/margins": 0.38539624214172363, + "rewards/rejected": 1.5691938400268555, + "step": 245 + }, + { + "epoch": 0.2389058113838619, + "grad_norm": 428.0, + "ht_mnpo/logit": 0.01622030697762966, + "ht_mnpo/residual": 0.008720312267541885, + "ht_mnpo/residual_abs": 0.15026597678661346, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.5395031701104303e-07, + "logits/chosen": -2.188828706741333, + "logits/rejected": -1.8585360050201416, + "logps/chosen": -0.3078772723674774, + "logps/rejected": -0.311143159866333, + "loss": 0.2350800484418869, + "loss/core": 0.2350800484418869, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": 1.6038856506347656, + "rewards/margins": 0.1622031033039093, + "rewards/rejected": 1.4416824579238892, + "step": 250 + }, + { + "epoch": 0.24368392761153915, + "grad_norm": 2.234375, + "ht_mnpo/logit": 0.04869508370757103, + "ht_mnpo/residual": 0.041195083409547806, + "ht_mnpo/residual_abs": 0.08923383802175522, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.511083097731555e-07, + "logits/chosen": -2.215907096862793, + "logits/rejected": -1.984976053237915, + "logps/chosen": -0.2803708612918854, + "logps/rejected": -0.30261167883872986, + "loss": 0.08057855069637299, + "loss/core": 0.08057855069637299, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 1.605520248413086, + "rewards/margins": 0.4869508147239685, + "rewards/rejected": 1.1185696125030518, + "step": 255 + }, + { + "epoch": 0.24846204383921638, + "grad_norm": 6.5625, + "ht_mnpo/logit": 0.06030106544494629, + "ht_mnpo/residual": 0.05280106142163277, + "ht_mnpo/residual_abs": 0.29066184163093567, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.481906739246894e-07, + "logits/chosen": -2.153928279876709, + "logits/rejected": -1.8087905645370483, + "logps/chosen": -0.2784316837787628, + "logps/rejected": -0.2740718424320221, + "loss": 0.6678136587142944, + "loss/core": 0.6678136587142944, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 3.0530622005462646, + "rewards/margins": 0.6030107736587524, + "rewards/rejected": 2.450051784515381, + "step": 260 + }, + { + "epoch": 0.25324016006689365, + "grad_norm": 61.25, + "ht_mnpo/logit": 0.0862557515501976, + "ht_mnpo/residual": 0.07875575870275497, + "ht_mnpo/residual_abs": 0.16950246691703796, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.451985066691648e-07, + "logits/chosen": -2.1912569999694824, + "logits/rejected": -1.9450289011001587, + "logps/chosen": -0.2741378843784332, + "logps/rejected": -0.31863006949424744, + "loss": 0.3198677599430084, + "loss/core": 0.3198677599430084, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 2.4567675590515137, + "rewards/margins": 0.8625577092170715, + "rewards/rejected": 1.5942096710205078, + "step": 265 + }, + { + "epoch": 0.25801827629457086, + "grad_norm": 79.5, + "ht_mnpo/logit": 0.04499917849898338, + "ht_mnpo/residual": 0.03749917820096016, + "ht_mnpo/residual_abs": 0.08561719208955765, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.421329332383158e-07, + "logits/chosen": -2.34749698638916, + "logits/rejected": -2.015653371810913, + "logps/chosen": -0.26775068044662476, + "logps/rejected": -0.2911599576473236, + "loss": 0.04689795523881912, + "loss/core": 0.04689795523881912, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 1.4798815250396729, + "rewards/margins": 0.44999170303344727, + "rewards/rejected": 1.0298898220062256, + "step": 270 + }, + { + "epoch": 0.2627963925222481, + "grad_norm": 62.0, + "ht_mnpo/logit": 0.07096774876117706, + "ht_mnpo/residual": 0.06346776336431503, + "ht_mnpo/residual_abs": 0.1662285178899765, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.3899510646893696e-07, + "logits/chosen": -2.1291439533233643, + "logits/rejected": -1.8797943592071533, + "logps/chosen": -0.2925297021865845, + "logps/rejected": -0.33546027541160583, + "loss": 0.3264152407646179, + "loss/core": 0.3264152407646179, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.4164235591888428, + "rewards/margins": 0.7096775770187378, + "rewards/rejected": 1.7067458629608154, + "step": 275 + }, + { + "epoch": 0.26757450874992533, + "grad_norm": 3.3125, + "ht_mnpo/logit": 0.05690809339284897, + "ht_mnpo/residual": 0.04940810054540634, + "ht_mnpo/residual_abs": 0.07465264946222305, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.357862063693485e-07, + "logits/chosen": -2.1697328090667725, + "logits/rejected": -1.9812034368515015, + "logps/chosen": -0.2690741717815399, + "logps/rejected": -0.27334779500961304, + "loss": 0.031530968844890594, + "loss/core": 0.031530968844890594, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.834280014038086, + "rewards/margins": 0.5690810084342957, + "rewards/rejected": 1.2651989459991455, + "step": 280 + }, + { + "epoch": 0.2723526249776026, + "grad_norm": 15.375, + "ht_mnpo/logit": 0.05523994565010071, + "ht_mnpo/residual": 0.047739945352077484, + "ht_mnpo/residual_abs": 0.05985153466463089, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.3250743967564364e-07, + "logits/chosen": -2.1334269046783447, + "logits/rejected": -2.0200657844543457, + "logps/chosen": -0.3056999146938324, + "logps/rejected": -0.3090917766094208, + "loss": 0.02557971142232418, + "loss/core": 0.02557971142232418, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.5456047058105469, + "rewards/margins": 0.5523993372917175, + "rewards/rejected": 0.9932052493095398, + "step": 285 + }, + { + "epoch": 0.2771307412052798, + "grad_norm": 5.875, + "ht_mnpo/logit": 0.03501705452799797, + "ht_mnpo/residual": 0.027517059817910194, + "ht_mnpo/residual_abs": 0.06787494570016861, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.29160039397884e-07, + "logits/chosen": -2.2637059688568115, + "logits/rejected": -2.0740151405334473, + "logps/chosen": -0.3052566349506378, + "logps/rejected": -0.30728772282600403, + "loss": 0.02679051086306572, + "loss/core": 0.02679051086306572, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 1.183258056640625, + "rewards/margins": 0.3501706123352051, + "rewards/rejected": 0.8330875635147095, + "step": 290 + }, + { + "epoch": 0.28190885743295707, + "grad_norm": 28.5, + "ht_mnpo/logit": 0.14480124413967133, + "ht_mnpo/residual": 0.1373012363910675, + "ht_mnpo/residual_abs": 0.14611732959747314, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.2574526435641546e-07, + "logits/chosen": -2.285439968109131, + "logits/rejected": -2.081019878387451, + "logps/chosen": -0.3001931607723236, + "logps/rejected": -0.30784377455711365, + "loss": 0.6604355573654175, + "loss/core": 0.6604355573654175, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.0780365467071533, + "rewards/margins": 1.4480124711990356, + "rewards/rejected": 0.6300240755081177, + "step": 295 + }, + { + "epoch": 0.2866869736606343, + "grad_norm": 4.90625, + "ht_mnpo/logit": 0.06456714868545532, + "ht_mnpo/residual": 0.0570671446621418, + "ht_mnpo/residual_abs": 0.14857563376426697, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.22264398708477e-07, + "logits/chosen": -2.2999801635742188, + "logits/rejected": -1.9922819137573242, + "logps/chosen": -0.28616541624069214, + "logps/rejected": -0.2884747087955475, + "loss": 0.24228546023368835, + "loss/core": 0.24228546023368835, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 2.1733169555664062, + "rewards/margins": 0.6456714868545532, + "rewards/rejected": 1.5276457071304321, + "step": 300 + }, + { + "epoch": 0.29146508988831155, + "grad_norm": 3.8125, + "ht_mnpo/logit": 0.04495188593864441, + "ht_mnpo/residual": 0.037451885640621185, + "ht_mnpo/residual_abs": 0.20770028233528137, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.187187514652819e-07, + "logits/chosen": -2.157289981842041, + "logits/rejected": -1.8468250036239624, + "logps/chosen": -0.28001922369003296, + "logps/rejected": -0.29186612367630005, + "loss": 0.4534494876861572, + "loss/core": 0.4534494876861572, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": 2.569370746612549, + "rewards/margins": 0.4495188593864441, + "rewards/rejected": 2.119851589202881, + "step": 305 + }, + { + "epoch": 0.29624320611598876, + "grad_norm": 7.28125, + "ht_mnpo/logit": 0.024377625435590744, + "ht_mnpo/residual": 0.01687762141227722, + "ht_mnpo/residual_abs": 0.08526398241519928, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.151096559997519e-07, + "logits/chosen": -2.399658441543579, + "logits/rejected": -2.0769169330596924, + "logps/chosen": -0.2818771004676819, + "logps/rejected": -0.3100544810295105, + "loss": 0.04117077589035034, + "loss/core": 0.04117077589035034, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.4926480054855347, + "rewards/margins": 0.24377623200416565, + "rewards/rejected": 1.2488718032836914, + "step": 310 + }, + { + "epoch": 0.301021322343666, + "grad_norm": 1.4609375, + "ht_mnpo/logit": 0.1227768212556839, + "ht_mnpo/residual": 0.11527681350708008, + "ht_mnpo/residual_abs": 0.14000196754932404, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.114384695450905e-07, + "logits/chosen": -2.276912212371826, + "logits/rejected": -2.0756640434265137, + "logps/chosen": -0.29674750566482544, + "logps/rejected": -0.30335813760757446, + "loss": 0.5935509204864502, + "loss/core": 0.5935509204864502, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.0897817611694336, + "rewards/margins": 1.2277681827545166, + "rewards/rejected": 0.8620136380195618, + "step": 315 + }, + { + "epoch": 0.30579943857134323, + "grad_norm": 240.0, + "ht_mnpo/logit": 0.22828857600688934, + "ht_mnpo/residual": 0.22078856825828552, + "ht_mnpo/residual_abs": 0.23243160545825958, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.077065726843828e-07, + "logits/chosen": -2.360725164413452, + "logits/rejected": -2.132331371307373, + "logps/chosen": -0.28701794147491455, + "logps/rejected": -0.28683117032051086, + "loss": 1.1871272325515747, + "loss/core": 1.1871272325515747, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 3.358243465423584, + "rewards/margins": 2.2828855514526367, + "rewards/rejected": 1.0753579139709473, + "step": 320 + }, + { + "epoch": 0.3105775547990205, + "grad_norm": 1.4921875, + "ht_mnpo/logit": 0.06212912127375603, + "ht_mnpo/residual": 0.0546291284263134, + "ht_mnpo/residual_abs": 0.20826132595539093, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.039153688314145e-07, + "logits/chosen": -2.275365114212036, + "logits/rejected": -2.0728507041931152, + "logps/chosen": -0.26758795976638794, + "logps/rejected": -0.27879461646080017, + "loss": 0.41284671425819397, + "loss/core": 0.41284671425819397, + "rewards/accuracies": 0.75, + "rewards/chosen": 2.737895965576172, + "rewards/margins": 0.6212912797927856, + "rewards/rejected": 2.116604804992676, + "step": 325 + }, + { + "epoch": 0.3153556710266977, + "grad_norm": 0.81640625, + "ht_mnpo/logit": 0.06797120720148087, + "ht_mnpo/residual": 0.06047120690345764, + "ht_mnpo/residual_abs": 0.07851966470479965, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.0006628370290613e-07, + "logits/chosen": -2.2605273723602295, + "logits/rejected": -2.134922504425049, + "logps/chosen": -0.2739706039428711, + "logps/rejected": -0.2764506936073303, + "loss": 0.05020144581794739, + "loss/core": 0.05020144581794739, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.6947124004364014, + "rewards/margins": 0.6797120571136475, + "rewards/rejected": 1.0150004625320435, + "step": 330 + }, + { + "epoch": 0.32013378725437497, + "grad_norm": 0.75390625, + "ht_mnpo/logit": 0.19798977673053741, + "ht_mnpo/residual": 0.1904897689819336, + "ht_mnpo/residual_abs": 0.21845956146717072, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.9616076478235826e-07, + "logits/chosen": -2.047623872756958, + "logits/rejected": -1.9703845977783203, + "logps/chosen": -0.29154783487319946, + "logps/rejected": -0.3145486116409302, + "loss": 0.6114920973777771, + "loss/core": 0.6114920973777771, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.0861105918884277, + "rewards/margins": 1.9798977375030518, + "rewards/rejected": 1.106212854385376, + "step": 335 + }, + { + "epoch": 0.3249119034820522, + "grad_norm": 41.5, + "ht_mnpo/logit": 0.11068092286586761, + "ht_mnpo/residual": 0.10318093001842499, + "ht_mnpo/residual_abs": 0.25756824016571045, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.922002807757129e-07, + "logits/chosen": -1.9539859294891357, + "logits/rejected": -1.8528114557266235, + "logps/chosen": -0.2756306827068329, + "logps/rejected": -0.30442994832992554, + "loss": 0.9918975830078125, + "loss/core": 0.9918975830078125, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.8355953693389893, + "rewards/margins": 1.106809377670288, + "rewards/rejected": 1.7287861108779907, + "step": 340 + }, + { + "epoch": 0.32969001970972944, + "grad_norm": 0.703125, + "ht_mnpo/logit": 0.015581438317894936, + "ht_mnpo/residual": 0.008081444539129734, + "ht_mnpo/residual_abs": 0.10090570151805878, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.8818632105903315e-07, + "logits/chosen": -2.3269078731536865, + "logits/rejected": -2.0692667961120605, + "logps/chosen": -0.2906230390071869, + "logps/rejected": -0.29532790184020996, + "loss": 0.09663812071084976, + "loss/core": 0.09663812071084976, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.2236311435699463, + "rewards/margins": 0.15581440925598145, + "rewards/rejected": 1.0678166151046753, + "step": 345 + }, + { + "epoch": 0.33446813593740665, + "grad_norm": 1.4296875, + "ht_mnpo/logit": 0.07701550424098969, + "ht_mnpo/residual": 0.06951549649238586, + "ht_mnpo/residual_abs": 0.08727418631315231, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.841203951184094e-07, + "logits/chosen": -2.214233875274658, + "logits/rejected": -1.9848973751068115, + "logps/chosen": -0.3011823892593384, + "logps/rejected": -0.294931560754776, + "loss": 0.11329641193151474, + "loss/core": 0.11329641193151474, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.9033756256103516, + "rewards/margins": 0.7701549530029297, + "rewards/rejected": 1.133220911026001, + "step": 350 + }, + { + "epoch": 0.3392462521650839, + "grad_norm": 416.0, + "ht_mnpo/logit": 0.0463683120906353, + "ht_mnpo/residual": 0.03886830434203148, + "ht_mnpo/residual_abs": 0.14574840664863586, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.800040319823038e-07, + "logits/chosen": -2.264720916748047, + "logits/rejected": -1.9842700958251953, + "logps/chosen": -0.2861303985118866, + "logps/rejected": -0.2938997149467468, + "loss": 0.17697656154632568, + "loss/core": 0.17697656154632568, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.538461685180664, + "rewards/margins": 0.46368303894996643, + "rewards/rejected": 2.0747783184051514, + "step": 355 + }, + { + "epoch": 0.3440243683927611, + "grad_norm": 494.0, + "ht_mnpo/logit": 0.12127704918384552, + "ht_mnpo/residual": 0.1137770414352417, + "ht_mnpo/residual_abs": 0.1548972874879837, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.75838779646545e-07, + "logits/chosen": -2.3002190589904785, + "logits/rejected": -2.164665460586548, + "logps/chosen": -0.2934378981590271, + "logps/rejected": -0.2929591238498688, + "loss": 0.482862651348114, + "loss/core": 0.482862651348114, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.9623796939849854, + "rewards/margins": 1.2127704620361328, + "rewards/rejected": 0.7496089339256287, + "step": 360 + }, + { + "epoch": 0.3488024846204384, + "grad_norm": 424.0, + "ht_mnpo/logit": 0.010891343466937542, + "ht_mnpo/residual": 0.00339134712703526, + "ht_mnpo/residual_abs": 0.08606524765491486, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.7162620449218993e-07, + "logits/chosen": -2.2042396068573, + "logits/rejected": -1.8955295085906982, + "logps/chosen": -0.31718710064888, + "logps/rejected": -0.3305971920490265, + "loss": 0.12844952940940857, + "loss/core": 0.12844952940940857, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 1.4607499837875366, + "rewards/margins": 0.10891339927911758, + "rewards/rejected": 1.3518364429473877, + "step": 365 + }, + { + "epoch": 0.35358060084811566, + "grad_norm": 251.0, + "ht_mnpo/logit": 0.11479277908802032, + "ht_mnpo/residual": 0.10729275643825531, + "ht_mnpo/residual_abs": 0.11697915941476822, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.673678906964727e-07, + "logits/chosen": -2.4074511528015137, + "logits/rejected": -2.101377487182617, + "logps/chosen": -0.2906573712825775, + "logps/rejected": -0.29892730712890625, + "loss": 0.11968455463647842, + "loss/core": 0.11968455463647842, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.02966046333313, + "rewards/margins": 1.1479276418685913, + "rewards/rejected": 0.8817326426506042, + "step": 370 + }, + { + "epoch": 0.35835871707579287, + "grad_norm": 68.0, + "ht_mnpo/logit": 0.09350790083408356, + "ht_mnpo/residual": 0.08600790798664093, + "ht_mnpo/residual_abs": 0.1067660003900528, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.6306543963705936e-07, + "logits/chosen": -2.127977132797241, + "logits/rejected": -1.9483768939971924, + "logps/chosen": -0.26925674080848694, + "logps/rejected": -0.2880834937095642, + "loss": 0.106331467628479, + "loss/core": 0.106331467628479, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.7844947576522827, + "rewards/margins": 0.9350789785385132, + "rewards/rejected": 0.8494156002998352, + "step": 375 + }, + { + "epoch": 0.36313683330347013, + "grad_norm": 1.265625, + "ht_mnpo/logit": 0.004885909613221884, + "ht_mnpo/residual": -0.002614091383293271, + "ht_mnpo/residual_abs": 0.10483495146036148, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.5872046928983623e-07, + "logits/chosen": -2.168792724609375, + "logits/rejected": -1.950922966003418, + "logps/chosen": -0.28580015897750854, + "logps/rejected": -0.30009040236473083, + "loss": 0.0811811238527298, + "loss/core": 0.0811811238527298, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.6719993352890015, + "rewards/margins": 0.04885910823941231, + "rewards/rejected": 1.6231400966644287, + "step": 380 + }, + { + "epoch": 0.36791494953114734, + "grad_norm": 22.625, + "ht_mnpo/logit": 0.1782342940568924, + "ht_mnpo/residual": 0.17073427140712738, + "ht_mnpo/residual_abs": 0.20563189685344696, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.5433461362045447e-07, + "logits/chosen": -2.0228257179260254, + "logits/rejected": -1.7602002620697021, + "logps/chosen": -0.28422048687934875, + "logps/rejected": -0.282476007938385, + "loss": 0.31415554881095886, + "loss/core": 0.31415554881095886, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.310380458831787, + "rewards/margins": 1.7823426723480225, + "rewards/rejected": 1.5280377864837646, + "step": 385 + }, + { + "epoch": 0.3726930657588246, + "grad_norm": 19.875, + "ht_mnpo/logit": 0.18474507331848145, + "ht_mnpo/residual": 0.17724506556987762, + "ht_mnpo/residual_abs": 0.24547035992145538, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.4990952196986305e-07, + "logits/chosen": -2.205859422683716, + "logits/rejected": -1.906855821609497, + "logps/chosen": -0.2814488708972931, + "logps/rejected": -0.28038260340690613, + "loss": 0.5938295125961304, + "loss/core": 0.5938295125961304, + "rewards/accuracies": 0.8125, + "rewards/chosen": 3.566596269607544, + "rewards/margins": 1.847450852394104, + "rewards/rejected": 1.7191451787948608, + "step": 390 + }, + { + "epoch": 0.3774711819865018, + "grad_norm": 528.0, + "ht_mnpo/logit": 0.11254217475652695, + "ht_mnpo/residual": 0.10504218190908432, + "ht_mnpo/residual_abs": 0.1626928150653839, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.4544685843405875e-07, + "logits/chosen": -2.29341983795166, + "logits/rejected": -1.993295431137085, + "logps/chosen": -0.27144044637680054, + "logps/rejected": -0.29052314162254333, + "loss": 0.3823533058166504, + "loss/core": 0.3823533058166504, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.345606565475464, + "rewards/margins": 1.1254217624664307, + "rewards/rejected": 1.2201849222183228, + "step": 395 + }, + { + "epoch": 0.3822492982141791, + "grad_norm": 1.53125, + "ht_mnpo/logit": -0.04141603037714958, + "ht_mnpo/residual": -0.0489160381257534, + "ht_mnpo/residual_abs": 0.11461643129587173, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.4094830123828786e-07, + "logits/chosen": -2.3432741165161133, + "logits/rejected": -1.979839563369751, + "logps/chosen": -0.26395756006240845, + "logps/rejected": -0.2680288255214691, + "loss": 0.25311708450317383, + "loss/core": 0.25311708450317383, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.3107450008392334, + "rewards/margins": -0.41416025161743164, + "rewards/rejected": 1.724905252456665, + "step": 400 + }, + { + "epoch": 0.3870274144418563, + "grad_norm": 796.0, + "ht_mnpo/logit": 0.0915079265832901, + "ht_mnpo/residual": 0.08400792628526688, + "ht_mnpo/residual_abs": 0.20653638243675232, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.3641554210593414e-07, + "logits/chosen": -2.0984344482421875, + "logits/rejected": -1.869283676147461, + "logps/chosen": -0.26586979627609253, + "logps/rejected": -0.2844234108924866, + "loss": 0.7313702702522278, + "loss/core": 0.7313702702522278, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.7626240253448486, + "rewards/margins": 0.9150789976119995, + "rewards/rejected": 1.8475449085235596, + "step": 405 + }, + { + "epoch": 0.39180553066953355, + "grad_norm": 5.09375, + "ht_mnpo/logit": 0.07326017320156097, + "ht_mnpo/residual": 0.06576018035411835, + "ht_mnpo/residual_abs": 0.08752019703388214, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.3185028562233107e-07, + "logits/chosen": -2.194375514984131, + "logits/rejected": -2.0342750549316406, + "logps/chosen": -0.281860888004303, + "logps/rejected": -0.3041122555732727, + "loss": 0.044236548244953156, + "loss/core": 0.044236548244953156, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.6896095275878906, + "rewards/margins": 0.7326017618179321, + "rewards/rejected": 0.9570077061653137, + "step": 410 + }, + { + "epoch": 0.39658364689721076, + "grad_norm": 640.0, + "ht_mnpo/logit": -0.07659461349248886, + "ht_mnpo/residual": -0.08409460633993149, + "ht_mnpo/residual_abs": 0.14910180866718292, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.272542485937368e-07, + "logits/chosen": -2.4320578575134277, + "logits/rejected": -2.1560490131378174, + "logps/chosen": -0.2776671051979065, + "logps/rejected": -0.293962299823761, + "loss": 0.5594010949134827, + "loss/core": 0.5594010949134827, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.1026519536972046, + "rewards/margins": -0.7659462690353394, + "rewards/rejected": 1.868598222732544, + "step": 415 + }, + { + "epoch": 0.401361763124888, + "grad_norm": 185.0, + "ht_mnpo/logit": 0.076129250228405, + "ht_mnpo/residual": 0.06862926483154297, + "ht_mnpo/residual_abs": 0.1663965880870819, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.226291594017137e-07, + "logits/chosen": -2.326869249343872, + "logits/rejected": -2.0747084617614746, + "logps/chosen": -0.2761536240577698, + "logps/rejected": -0.28266650438308716, + "loss": 0.34604477882385254, + "loss/core": 0.34604477882385254, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.188345432281494, + "rewards/margins": 0.761292576789856, + "rewards/rejected": 1.4270528554916382, + "step": 420 + }, + { + "epoch": 0.40613987935256524, + "grad_norm": 0.42578125, + "ht_mnpo/logit": 0.07638172805309296, + "ht_mnpo/residual": 0.06888173520565033, + "ht_mnpo/residual_abs": 0.11476260423660278, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.1797675735315454e-07, + "logits/chosen": -2.2939515113830566, + "logits/rejected": -2.034564971923828, + "logps/chosen": -0.27834567427635193, + "logps/rejected": -0.3104603886604309, + "loss": 0.17839036881923676, + "loss/core": 0.17839036881923676, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.17610239982605, + "rewards/margins": 0.7638173699378967, + "rewards/rejected": 1.4122850894927979, + "step": 425 + }, + { + "epoch": 0.4109179955802425, + "grad_norm": 3.15625, + "ht_mnpo/logit": -0.03292038291692734, + "ht_mnpo/residual": -0.04042038321495056, + "ht_mnpo/residual_abs": 0.18147443234920502, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.1329879202620047e-07, + "logits/chosen": -2.2801480293273926, + "logits/rejected": -2.009410858154297, + "logps/chosen": -0.2517848610877991, + "logps/rejected": -0.28678059577941895, + "loss": 0.35211431980133057, + "loss/core": 0.35211431980133057, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 2.0473055839538574, + "rewards/margins": -0.3292037844657898, + "rewards/rejected": 2.376509428024292, + "step": 430 + }, + { + "epoch": 0.4156961118079197, + "grad_norm": 38.5, + "ht_mnpo/logit": 0.06122002750635147, + "ht_mnpo/residual": 0.05372002720832825, + "ht_mnpo/residual_abs": 0.08932183682918549, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.0859702261229613e-07, + "logits/chosen": -2.134705066680908, + "logits/rejected": -1.9749943017959595, + "logps/chosen": -0.3081589341163635, + "logps/rejected": -0.31096822023391724, + "loss": 0.053012412041425705, + "loss/core": 0.053012412041425705, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 1.6446977853775024, + "rewards/margins": 0.6122003197669983, + "rewards/rejected": 1.0324976444244385, + "step": 435 + }, + { + "epoch": 0.420474228035597, + "grad_norm": 2.875, + "ht_mnpo/logit": 0.09818646311759949, + "ht_mnpo/residual": 0.09068647027015686, + "ht_mnpo/residual_abs": 0.12812697887420654, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.0387321725463e-07, + "logits/chosen": -2.1747286319732666, + "logits/rejected": -1.9857454299926758, + "logps/chosen": -0.2803365886211395, + "logps/rejected": -0.2985735535621643, + "loss": 0.1355080008506775, + "loss/core": 0.1355080008506775, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.7439346313476562, + "rewards/margins": 0.9818647503852844, + "rewards/rejected": 1.7620700597763062, + "step": 440 + }, + { + "epoch": 0.4252523442632742, + "grad_norm": 28.875, + "ht_mnpo/logit": 0.07558500021696091, + "ht_mnpo/residual": 0.06808499991893768, + "ht_mnpo/residual_abs": 0.1335534304380417, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.991291523832075e-07, + "logits/chosen": -2.0056984424591064, + "logits/rejected": -1.829700231552124, + "logps/chosen": -0.2837182879447937, + "logps/rejected": -0.301053524017334, + "loss": 0.12916430830955505, + "loss/core": 0.12916430830955505, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.1971168518066406, + "rewards/margins": 0.7558499574661255, + "rewards/rejected": 1.4412667751312256, + "step": 445 + }, + { + "epoch": 0.43003046049095145, + "grad_norm": 26.125, + "ht_mnpo/logit": 0.0845903530716896, + "ht_mnpo/residual": 0.07709035277366638, + "ht_mnpo/residual_abs": 0.26011234521865845, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.943666120468088e-07, + "logits/chosen": -2.2318408489227295, + "logits/rejected": -1.978154182434082, + "logps/chosen": -0.26634055376052856, + "logps/rejected": -0.28922945261001587, + "loss": 0.7810267210006714, + "loss/core": 0.7810267210006714, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.7954800128936768, + "rewards/margins": 0.8459035754203796, + "rewards/rejected": 1.9495761394500732, + "step": 450 + }, + { + "epoch": 0.43480857671862866, + "grad_norm": 2.734375, + "ht_mnpo/logit": 0.025683075189590454, + "ht_mnpo/residual": 0.01818307302892208, + "ht_mnpo/residual_abs": 0.08304707705974579, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.8958738724208073e-07, + "logits/chosen": -2.370572328567505, + "logits/rejected": -2.1037092208862305, + "logps/chosen": -0.26799899339675903, + "logps/rejected": -0.27521270513534546, + "loss": 0.059945620596408844, + "loss/core": 0.059945620596408844, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.4931560754776, + "rewards/margins": 0.25683075189590454, + "rewards/rejected": 1.2363255023956299, + "step": 455 + }, + { + "epoch": 0.4395866929463059, + "grad_norm": 740.0, + "ht_mnpo/logit": 0.1160498708486557, + "ht_mnpo/residual": 0.10854987800121307, + "ht_mnpo/residual_abs": 0.13926836848258972, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.8479327524001633e-07, + "logits/chosen": -2.282945156097412, + "logits/rejected": -2.147752285003662, + "logps/chosen": -0.29756516218185425, + "logps/rejected": -0.28938478231430054, + "loss": 0.4518727660179138, + "loss/core": 0.4518727660179138, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.2263054847717285, + "rewards/margins": 1.1604987382888794, + "rewards/rejected": 1.0658067464828491, + "step": 460 + }, + { + "epoch": 0.44436480917398313, + "grad_norm": 5.875, + "ht_mnpo/logit": 0.09502001106739044, + "ht_mnpo/residual": 0.08752001821994781, + "ht_mnpo/residual_abs": 0.1076284870505333, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.7998607891007493e-07, + "logits/chosen": -2.368155002593994, + "logits/rejected": -2.1031692028045654, + "logps/chosen": -0.28982120752334595, + "logps/rejected": -0.3006080985069275, + "loss": 0.20630280673503876, + "loss/core": 0.20630280673503876, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.638486623764038, + "rewards/margins": 0.9502002596855164, + "rewards/rejected": 0.6882864236831665, + "step": 465 + }, + { + "epoch": 0.4491429254016604, + "grad_norm": 20.75, + "ht_mnpo/logit": 0.11079178750514984, + "ht_mnpo/residual": 0.10329178720712662, + "ht_mnpo/residual_abs": 0.11315907537937164, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.7516760604219616e-07, + "logits/chosen": -2.191417694091797, + "logits/rejected": -2.082339286804199, + "logps/chosen": -0.25483283400535583, + "logps/rejected": -0.2595880925655365, + "loss": 0.08468286693096161, + "loss/core": 0.08468286693096161, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.0527219772338867, + "rewards/margins": 1.1079180240631104, + "rewards/rejected": 0.9448040723800659, + "step": 470 + }, + { + "epoch": 0.4539210416293376, + "grad_norm": 1.5546875, + "ht_mnpo/logit": 0.041524358093738556, + "ht_mnpo/residual": 0.03402435779571533, + "ht_mnpo/residual_abs": 0.1066092848777771, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.703396686669646e-07, + "logits/chosen": -2.2947616577148438, + "logits/rejected": -2.062729597091675, + "logps/chosen": -0.2770024240016937, + "logps/rejected": -0.28233516216278076, + "loss": 0.07373562455177307, + "loss/core": 0.07373562455177307, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.652806043624878, + "rewards/margins": 0.41524356603622437, + "rewards/rejected": 1.237562656402588, + "step": 475 + }, + { + "epoch": 0.45869915785701487, + "grad_norm": 109.0, + "ht_mnpo/logit": 0.12709861993789673, + "ht_mnpo/residual": 0.11959858983755112, + "ht_mnpo/residual_abs": 0.2165347784757614, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.6550408237417884e-07, + "logits/chosen": -2.141500234603882, + "logits/rejected": -1.88735032081604, + "logps/chosen": -0.32017025351524353, + "logps/rejected": -0.33095675706863403, + "loss": 0.7335715889930725, + "loss/core": 0.7335715889930725, + "rewards/accuracies": 0.75, + "rewards/chosen": 3.164018154144287, + "rewards/margins": 1.2709861993789673, + "rewards/rejected": 1.8930323123931885, + "step": 480 + }, + { + "epoch": 0.46347727408469214, + "grad_norm": 2.84375, + "ht_mnpo/logit": 0.03613976389169693, + "ht_mnpo/residual": 0.028639763593673706, + "ht_mnpo/residual_abs": 0.10108862817287445, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.6066266563008265e-07, + "logits/chosen": -2.169957399368286, + "logits/rejected": -1.9736477136611938, + "logps/chosen": -0.2766631245613098, + "logps/rejected": -0.2809513807296753, + "loss": 0.07384424656629562, + "loss/core": 0.07384424656629562, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.7005043029785156, + "rewards/margins": 0.3613975942134857, + "rewards/rejected": 1.339106559753418, + "step": 485 + }, + { + "epoch": 0.46825539031236935, + "grad_norm": 8.875, + "ht_mnpo/logit": 0.053298842161893845, + "ht_mnpo/residual": 0.045798830687999725, + "ht_mnpo/residual_abs": 0.2681800127029419, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.5581723909351404e-07, + "logits/chosen": -2.0894553661346436, + "logits/rejected": -1.8972949981689453, + "logps/chosen": -0.3042840361595154, + "logps/rejected": -0.33394938707351685, + "loss": 0.8056422472000122, + "loss/core": 0.8056422472000122, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.712467908859253, + "rewards/margins": 0.5329883694648743, + "rewards/rejected": 2.1794795989990234, + "step": 490 + }, + { + "epoch": 0.4730335065400466, + "grad_norm": 20.375, + "ht_mnpo/logit": 0.06616373360157013, + "ht_mnpo/residual": 0.058663733303546906, + "ht_mnpo/residual_abs": 0.07755361497402191, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.509696249312301e-07, + "logits/chosen": -2.2147159576416016, + "logits/rejected": -2.0647025108337402, + "logps/chosen": -0.26900672912597656, + "logps/rejected": -0.27678027749061584, + "loss": 0.034580301493406296, + "loss/core": 0.034580301493406296, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.6060407161712646, + "rewards/margins": 0.6616373062133789, + "rewards/rejected": 0.9444035291671753, + "step": 495 + }, + { + "epoch": 0.4778116227677238, + "grad_norm": 1.171875, + "ht_mnpo/logit": 0.15142087638378143, + "ht_mnpo/residual": 0.14392085373401642, + "ht_mnpo/residual_abs": 0.2897629141807556, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.461216461326642e-07, + "logits/chosen": -1.9548299312591553, + "logits/rejected": -1.732587456703186, + "logps/chosen": -0.2988852262496948, + "logps/rejected": -0.3125118017196655, + "loss": 0.9474697113037109, + "loss/core": 0.9474697113037109, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 3.979719638824463, + "rewards/margins": 1.5142085552215576, + "rewards/rejected": 2.465510845184326, + "step": 500 + }, + { + "epoch": 0.4825897389954011, + "grad_norm": 3.03125, + "ht_mnpo/logit": 0.06197315454483032, + "ht_mnpo/residual": 0.0544731430709362, + "ht_mnpo/residual_abs": 0.146002858877182, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.412751258243748e-07, + "logits/chosen": -2.1246144771575928, + "logits/rejected": -1.9374281167984009, + "logps/chosen": -0.24218598008155823, + "logps/rejected": -0.2681754529476166, + "loss": 0.19025397300720215, + "loss/core": 0.19025397300720215, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.2516086101531982, + "rewards/margins": 0.6197315454483032, + "rewards/rejected": 1.6318775415420532, + "step": 505 + }, + { + "epoch": 0.4873678552230783, + "grad_norm": 63.5, + "ht_mnpo/logit": 0.07161416858434677, + "ht_mnpo/residual": 0.06411416828632355, + "ht_mnpo/residual_abs": 0.11509843915700912, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.3643188658444158e-07, + "logits/chosen": -2.34808611869812, + "logits/rejected": -2.110934019088745, + "logps/chosen": -0.2828849256038666, + "logps/rejected": -0.29181432723999023, + "loss": 0.08646970242261887, + "loss/core": 0.08646970242261887, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.3318309783935547, + "rewards/margins": 0.7161417603492737, + "rewards/rejected": 1.6156890392303467, + "step": 510 + }, + { + "epoch": 0.49214597145075556, + "grad_norm": 59.0, + "ht_mnpo/logit": 0.07742169499397278, + "ht_mnpo/residual": 0.06992168724536896, + "ht_mnpo/residual_abs": 0.07737835496664047, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.315937497570688e-07, + "logits/chosen": -2.3272345066070557, + "logits/rejected": -2.0595836639404297, + "logps/chosen": -0.2829161584377289, + "logps/rejected": -0.2974115014076233, + "loss": 0.04383247345685959, + "loss/core": 0.04383247345685959, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.7030824422836304, + "rewards/margins": 0.7742169499397278, + "rewards/rejected": 0.9288653135299683, + "step": 515 + }, + { + "epoch": 0.49692408767843277, + "grad_norm": 0.2099609375, + "ht_mnpo/logit": 0.0885908305644989, + "ht_mnpo/residual": 0.08109083026647568, + "ht_mnpo/residual_abs": 0.12812694907188416, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.2676253476765194e-07, + "logits/chosen": -2.1603314876556396, + "logits/rejected": -2.0189948081970215, + "logps/chosen": -0.28825613856315613, + "logps/rejected": -0.30118125677108765, + "loss": 0.17555531859397888, + "loss/core": 0.17555531859397888, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.3579986095428467, + "rewards/margins": 0.885908305644989, + "rewards/rejected": 1.472090244293213, + "step": 520 + }, + { + "epoch": 0.50170220390611, + "grad_norm": 12.3125, + "ht_mnpo/logit": 0.047115158289670944, + "ht_mnpo/residual": 0.039615172892808914, + "ht_mnpo/residual_abs": 0.1817067414522171, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.2194005843856633e-07, + "logits/chosen": -2.351896047592163, + "logits/rejected": -2.1078438758850098, + "logps/chosen": -0.2944217622280121, + "logps/rejected": -0.28825289011001587, + "loss": 0.6127477884292603, + "loss/core": 0.6127477884292603, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.208118438720703, + "rewards/margins": 0.47115176916122437, + "rewards/rejected": 1.7369667291641235, + "step": 525 + }, + { + "epoch": 0.5064803201337873, + "grad_norm": 1000.0, + "ht_mnpo/logit": 0.06004665046930313, + "ht_mnpo/residual": 0.05254665017127991, + "ht_mnpo/residual_abs": 0.23322728276252747, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.1712813430593434e-07, + "logits/chosen": -2.1865041255950928, + "logits/rejected": -1.8540855646133423, + "logps/chosen": -0.2854662537574768, + "logps/rejected": -0.32376745343208313, + "loss": 0.5732264518737793, + "loss/core": 0.5732264518737793, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 3.1418910026550293, + "rewards/margins": 0.6004664301872253, + "rewards/rejected": 2.5414247512817383, + "step": 530 + }, + { + "epoch": 0.5112584363614645, + "grad_norm": 27.0, + "ht_mnpo/logit": 0.13712215423583984, + "ht_mnpo/residual": 0.1296221762895584, + "ht_mnpo/residual_abs": 0.17209194600582123, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.123285719376292e-07, + "logits/chosen": -2.173382043838501, + "logits/rejected": -1.8262176513671875, + "logps/chosen": -0.2807655930519104, + "logps/rejected": -0.2747199237346649, + "loss": 0.29922977089881897, + "loss/core": 0.29922977089881897, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 2.9412617683410645, + "rewards/margins": 1.3712217807769775, + "rewards/rejected": 1.570040225982666, + "step": 535 + }, + { + "epoch": 0.5160365525891417, + "grad_norm": 0.578125, + "ht_mnpo/logit": 0.061078108847141266, + "ht_mnpo/residual": 0.05357811599969864, + "ht_mnpo/residual_abs": 0.14000064134597778, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.0754317625276982e-07, + "logits/chosen": -2.1986470222473145, + "logits/rejected": -2.0166780948638916, + "logps/chosen": -0.2934110760688782, + "logps/rejected": -0.31152665615081787, + "loss": 0.2973403334617615, + "loss/core": 0.2973403334617615, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.6136223077774048, + "rewards/margins": 0.6107810735702515, + "rewards/rejected": 1.0028412342071533, + "step": 540 + }, + { + "epoch": 0.5208146688168189, + "grad_norm": 30.875, + "ht_mnpo/logit": 0.09199979156255722, + "ht_mnpo/residual": 0.084499791264534, + "ht_mnpo/residual_abs": 0.1785065233707428, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.0277374684296498e-07, + "logits/chosen": -2.1413793563842773, + "logits/rejected": -1.9582176208496094, + "logps/chosen": -0.26738229393959045, + "logps/rejected": -0.2807932198047638, + "loss": 0.19445274770259857, + "loss/core": 0.19445274770259857, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.202378511428833, + "rewards/margins": 0.919998049736023, + "rewards/rejected": 2.2823805809020996, + "step": 545 + }, + { + "epoch": 0.5255927850444962, + "grad_norm": 624.0, + "ht_mnpo/logit": 0.048402734100818634, + "ht_mnpo/residual": 0.04090272635221481, + "ht_mnpo/residual_abs": 0.20708465576171875, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.980220772955602e-07, + "logits/chosen": -2.335815191268921, + "logits/rejected": -2.078155994415283, + "logps/chosen": -0.33746832609176636, + "logps/rejected": -0.307259738445282, + "loss": 0.6829358339309692, + "loss/core": 0.6829358339309692, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.586956024169922, + "rewards/margins": 0.48402705788612366, + "rewards/rejected": 2.102928876876831, + "step": 550 + }, + { + "epoch": 0.5303709012721735, + "grad_norm": 504.0, + "ht_mnpo/logit": 0.17546024918556213, + "ht_mnpo/residual": 0.1679602712392807, + "ht_mnpo/residual_abs": 0.21153728663921356, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.932899545191433e-07, + "logits/chosen": -2.0990540981292725, + "logits/rejected": -1.8069511651992798, + "logps/chosen": -0.29029348492622375, + "logps/rejected": -0.284585177898407, + "loss": 0.5383775234222412, + "loss/core": 0.5383775234222412, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 2.9643406867980957, + "rewards/margins": 1.7546026706695557, + "rewards/rejected": 1.2097381353378296, + "step": 555 + }, + { + "epoch": 0.5351490174998507, + "grad_norm": 12.3125, + "ht_mnpo/logit": 0.1519746482372284, + "ht_mnpo/residual": 0.14447465538978577, + "ht_mnpo/residual_abs": 0.17106272280216217, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.885791580715609e-07, + "logits/chosen": -2.2186014652252197, + "logits/rejected": -2.0292179584503174, + "logps/chosen": -0.2765801250934601, + "logps/rejected": -0.26710256934165955, + "loss": 0.511273205280304, + "loss/core": 0.511273205280304, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.9151830673217773, + "rewards/margins": 1.5197465419769287, + "rewards/rejected": 1.3954365253448486, + "step": 560 + }, + { + "epoch": 0.5399271337275279, + "grad_norm": 2.375, + "ht_mnpo/logit": -0.013432973995804787, + "ht_mnpo/residual": -0.020932968705892563, + "ht_mnpo/residual_abs": 0.15191642940044403, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8389145949069951e-07, + "logits/chosen": -2.257132053375244, + "logits/rejected": -2.013878345489502, + "logps/chosen": -0.2700672447681427, + "logps/rejected": -0.2897999882698059, + "loss": 0.5020833015441895, + "loss/core": 0.5020833015441895, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 1.8380361795425415, + "rewards/margins": -0.1343296468257904, + "rewards/rejected": 1.9723659753799438, + "step": 565 + }, + { + "epoch": 0.5447052499552052, + "grad_norm": 32.25, + "ht_mnpo/logit": 0.10750623792409897, + "ht_mnpo/residual": 0.10000623762607574, + "ht_mnpo/residual_abs": 0.11906830966472626, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.792286216282824e-07, + "logits/chosen": -2.1714699268341064, + "logits/rejected": -1.9297101497650146, + "logps/chosen": -0.29392221570014954, + "logps/rejected": -0.2952617406845093, + "loss": 0.24284780025482178, + "loss/core": 0.24284780025482178, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 2.1015772819519043, + "rewards/margins": 1.0750623941421509, + "rewards/rejected": 1.0265148878097534, + "step": 570 + }, + { + "epoch": 0.5494833661828824, + "grad_norm": 350.0, + "ht_mnpo/logit": 0.1779872626066208, + "ht_mnpo/residual": 0.17048725485801697, + "ht_mnpo/residual_abs": 0.19057273864746094, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.745923979869336e-07, + "logits/chosen": -2.037717342376709, + "logits/rejected": -1.7374416589736938, + "logps/chosen": -0.2867358326911926, + "logps/rejected": -0.28076958656311035, + "loss": 0.4524991512298584, + "loss/core": 0.4524991512298584, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": 2.9693145751953125, + "rewards/margins": 1.7798726558685303, + "rewards/rejected": 1.1894419193267822, + "step": 575 + }, + { + "epoch": 0.5542614824105596, + "grad_norm": 52.5, + "ht_mnpo/logit": 0.07053961604833603, + "ht_mnpo/residual": 0.0630396157503128, + "ht_mnpo/residual_abs": 0.16084904968738556, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.6998453206075708e-07, + "logits/chosen": -2.151390790939331, + "logits/rejected": -1.9337005615234375, + "logps/chosen": -0.28611868619918823, + "logps/rejected": -0.2922998368740082, + "loss": 0.2780520021915436, + "loss/core": 0.2780520021915436, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.186372756958008, + "rewards/margins": 0.7053961753845215, + "rewards/rejected": 1.4809764623641968, + "step": 580 + }, + { + "epoch": 0.5590395986382368, + "grad_norm": 159.0, + "ht_mnpo/logit": 0.073150135576725, + "ht_mnpo/residual": 0.06565012782812119, + "ht_mnpo/residual_abs": 0.17466747760772705, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.6540675667967973e-07, + "logits/chosen": -2.1101319789886475, + "logits/rejected": -1.8840051889419556, + "logps/chosen": -0.2872307598590851, + "logps/rejected": -0.3082466125488281, + "loss": 0.2757607400417328, + "loss/core": 0.2757607400417328, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.8962421417236328, + "rewards/margins": 0.7315012812614441, + "rewards/rejected": 1.1647409200668335, + "step": 585 + }, + { + "epoch": 0.5638177148659141, + "grad_norm": 1.609375, + "ht_mnpo/logit": 0.057926855981349945, + "ht_mnpo/residual": 0.05042685940861702, + "ht_mnpo/residual_abs": 0.12516820430755615, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.60860793357805e-07, + "logits/chosen": -2.291469097137451, + "logits/rejected": -2.0484461784362793, + "logps/chosen": -0.3092489242553711, + "logps/rejected": -0.3214951455593109, + "loss": 0.18674132227897644, + "loss/core": 0.18674132227897644, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.030933380126953, + "rewards/margins": 0.5792685747146606, + "rewards/rejected": 1.451664686203003, + "step": 590 + }, + { + "epoch": 0.5685958310935914, + "grad_norm": 3.640625, + "ht_mnpo/logit": 0.034106455743312836, + "ht_mnpo/residual": 0.02660645917057991, + "ht_mnpo/residual_abs": 0.06431782990694046, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.5634835164602196e-07, + "logits/chosen": -2.3490476608276367, + "logits/rejected": -2.112255096435547, + "logps/chosen": -0.2819194793701172, + "logps/rejected": -0.2832038104534149, + "loss": 0.019458934664726257, + "loss/core": 0.019458934664726257, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.5581998825073242, + "rewards/margins": 0.34106460213661194, + "rewards/rejected": 1.2171355485916138, + "step": 595 + }, + { + "epoch": 0.5733739473212686, + "grad_norm": 1.4296875, + "ht_mnpo/logit": 0.1805572807788849, + "ht_mnpo/residual": 0.17305730283260345, + "ht_mnpo/residual_abs": 0.18950334191322327, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.518711284891132e-07, + "logits/chosen": -2.1632628440856934, + "logits/rejected": -1.9910504817962646, + "logps/chosen": -0.2667522430419922, + "logps/rejected": -0.29135292768478394, + "loss": 0.4613543152809143, + "loss/core": 0.4613543152809143, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.934541702270508, + "rewards/margins": 1.8055731058120728, + "rewards/rejected": 1.1289689540863037, + "step": 600 + }, + { + "epoch": 0.5781520635489459, + "grad_norm": 1.6015625, + "ht_mnpo/logit": 0.048975661396980286, + "ht_mnpo/residual": 0.04147566109895706, + "ht_mnpo/residual_abs": 0.1043766587972641, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.47430807587603e-07, + "logits/chosen": -2.1373214721679688, + "logits/rejected": -1.9456546306610107, + "logps/chosen": -0.2933582365512848, + "logps/rejected": -0.29404282569885254, + "loss": 0.10271594673395157, + "loss/core": 0.10271594673395157, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.032723903656006, + "rewards/margins": 0.4897565245628357, + "rewards/rejected": 1.5429675579071045, + "step": 605 + }, + { + "epoch": 0.5829301797766231, + "grad_norm": 8.3125, + "ht_mnpo/logit": 0.06926552206277847, + "ht_mnpo/residual": 0.06176552176475525, + "ht_mnpo/residual_abs": 0.12705007195472717, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.430290587645865e-07, + "logits/chosen": -2.3621904850006104, + "logits/rejected": -2.105658531188965, + "logps/chosen": -0.29055026173591614, + "logps/rejected": -0.2932811677455902, + "loss": 0.21939155459403992, + "loss/core": 0.21939155459403992, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.4611763954162598, + "rewards/margins": 0.6926552057266235, + "rewards/rejected": 1.7685215473175049, + "step": 610 + }, + { + "epoch": 0.5877082960043003, + "grad_norm": 6.90625, + "ht_mnpo/logit": 0.05083639174699783, + "ht_mnpo/residual": 0.04333639144897461, + "ht_mnpo/residual_abs": 0.09056956321001053, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.3866753733777765e-07, + "logits/chosen": -2.2035889625549316, + "logits/rejected": -2.006361484527588, + "logps/chosen": -0.27977409958839417, + "logps/rejected": -0.29095375537872314, + "loss": 0.05176713317632675, + "loss/core": 0.05176713317632675, + "rewards/accuracies": 0.75, + "rewards/chosen": 1.685443639755249, + "rewards/margins": 0.5083640217781067, + "rewards/rejected": 1.1770797967910767, + "step": 615 + }, + { + "epoch": 0.5924864122319775, + "grad_norm": 2.171875, + "ht_mnpo/logit": 0.1474437713623047, + "ht_mnpo/residual": 0.13994379341602325, + "ht_mnpo/residual_abs": 0.16675330698490143, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.343478834970121e-07, + "logits/chosen": -2.2853574752807617, + "logits/rejected": -1.9415066242218018, + "logps/chosen": -0.291348934173584, + "logps/rejected": -0.30028507113456726, + "loss": 0.7507929801940918, + "loss/core": 0.7507929801940918, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.765855312347412, + "rewards/margins": 1.4744378328323364, + "rewards/rejected": 1.2914174795150757, + "step": 620 + }, + { + "epoch": 0.5972645284596548, + "grad_norm": 206.0, + "ht_mnpo/logit": 0.14434558153152466, + "ht_mnpo/residual": 0.13684558868408203, + "ht_mnpo/residual_abs": 0.2338484227657318, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.3007172168743852e-07, + "logits/chosen": -2.187645673751831, + "logits/rejected": -1.9885934591293335, + "logps/chosen": -0.27156367897987366, + "logps/rejected": -0.28177887201309204, + "loss": 0.889374852180481, + "loss/core": 0.889374852180481, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.960251569747925, + "rewards/margins": 1.4434558153152466, + "rewards/rejected": 1.5167958736419678, + "step": 625 + }, + { + "epoch": 0.602042644687332, + "grad_norm": 242.0, + "ht_mnpo/logit": 0.11875858157873154, + "ht_mnpo/residual": 0.11125858873128891, + "ht_mnpo/residual_abs": 0.1652863770723343, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.25840659998631e-07, + "logits/chosen": -2.1578569412231445, + "logits/rejected": -1.9522138833999634, + "logps/chosen": -0.3227333724498749, + "logps/rejected": -0.32840460538864136, + "loss": 0.2998679280281067, + "loss/core": 0.2998679280281067, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.60225510597229, + "rewards/margins": 1.187585711479187, + "rewards/rejected": 1.414669156074524, + "step": 630 + }, + { + "epoch": 0.6068207609150092, + "grad_norm": 4.34375, + "ht_mnpo/logit": 0.09825298935174942, + "ht_mnpo/residual": 0.0907529965043068, + "ht_mnpo/residual_abs": 0.15308210253715515, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.2165628955985313e-07, + "logits/chosen": -2.2780983448028564, + "logits/rejected": -2.0277860164642334, + "logps/chosen": -0.31773704290390015, + "logps/rejected": -0.3203713297843933, + "loss": 0.22551850974559784, + "loss/core": 0.22551850974559784, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.1558048725128174, + "rewards/margins": 0.9825299382209778, + "rewards/rejected": 1.1732749938964844, + "step": 635 + }, + { + "epoch": 0.6115988771426865, + "grad_norm": 0.90625, + "ht_mnpo/logit": 0.030277207493782043, + "ht_mnpo/residual": 0.02277720905840397, + "ht_mnpo/residual_abs": 0.129721999168396, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.175201839416988e-07, + "logits/chosen": -2.434414863586426, + "logits/rejected": -2.18612003326416, + "logps/chosen": -0.2906445264816284, + "logps/rejected": -0.2876962721347809, + "loss": 0.2192559689283371, + "loss/core": 0.2192559689283371, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.5939425230026245, + "rewards/margins": 0.30277204513549805, + "rewards/rejected": 1.2911707162857056, + "step": 640 + }, + { + "epoch": 0.6163769933703638, + "grad_norm": 106.0, + "ht_mnpo/logit": 0.03566845878958702, + "ht_mnpo/residual": 0.028168460354208946, + "ht_mnpo/residual_abs": 0.08532391488552094, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.1343389856433658e-07, + "logits/chosen": -2.1786632537841797, + "logits/rejected": -1.9741607904434204, + "logps/chosen": -0.27927350997924805, + "logps/rejected": -0.2722075581550598, + "loss": 0.03750378638505936, + "loss/core": 0.03750378638505936, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.107971668243408, + "rewards/margins": 0.3566845953464508, + "rewards/rejected": 1.7512871026992798, + "step": 645 + }, + { + "epoch": 0.621155109598041, + "grad_norm": 3.203125, + "ht_mnpo/logit": 0.05108143016695976, + "ht_mnpo/residual": 0.04358143359422684, + "ht_mnpo/residual_abs": 0.056876249611377716, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0939897011258e-07, + "logits/chosen": -2.1610920429229736, + "logits/rejected": -1.9564762115478516, + "logps/chosen": -0.28243714570999146, + "logps/rejected": -0.28669795393943787, + "loss": 0.015014773234724998, + "loss/core": 0.015014773234724998, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.650695562362671, + "rewards/margins": 0.510814368724823, + "rewards/rejected": 1.1398811340332031, + "step": 650 + }, + { + "epoch": 0.6259332258257182, + "grad_norm": 19.625, + "ht_mnpo/logit": 0.15237808227539062, + "ht_mnpo/residual": 0.1448780596256256, + "ht_mnpo/residual_abs": 0.1591993272304535, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0541691595800336e-07, + "logits/chosen": -2.39475679397583, + "logits/rejected": -2.1787381172180176, + "logps/chosen": -0.28849363327026367, + "logps/rejected": -0.29972606897354126, + "loss": 0.4378318190574646, + "loss/core": 0.4378318190574646, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.363157272338867, + "rewards/margins": 1.5237809419631958, + "rewards/rejected": 0.8393763303756714, + "step": 655 + }, + { + "epoch": 0.6307113420533954, + "grad_norm": 256.0, + "ht_mnpo/logit": 0.019719239324331284, + "ht_mnpo/residual": 0.012219244614243507, + "ht_mnpo/residual_abs": 0.1551227569580078, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0148923358832021e-07, + "logits/chosen": -2.1171586513519287, + "logits/rejected": -1.8910051584243774, + "logps/chosen": -0.2649215757846832, + "logps/rejected": -0.2766799330711365, + "loss": 0.2119613140821457, + "loss/core": 0.2119613140821457, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.211346387863159, + "rewards/margins": 0.19719253480434418, + "rewards/rejected": 2.0141539573669434, + "step": 660 + }, + { + "epoch": 0.6354894582810727, + "grad_norm": 74.5, + "ht_mnpo/logit": 0.10560693591833115, + "ht_mnpo/residual": 0.09810693562030792, + "ht_mnpo/residual_abs": 0.13916924595832825, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.761740004423926e-08, + "logits/chosen": -2.324584484100342, + "logits/rejected": -2.0863864421844482, + "logps/chosen": -0.25897565484046936, + "logps/rejected": -0.27116915583610535, + "loss": 0.12608017027378082, + "loss/core": 0.12608017027378082, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.370238780975342, + "rewards/margins": 1.056069254875183, + "rewards/rejected": 1.3141696453094482, + "step": 665 + }, + { + "epoch": 0.6402675745087499, + "grad_norm": 213.0, + "ht_mnpo/logit": 0.055773068219423294, + "ht_mnpo/residual": 0.04827307164669037, + "ht_mnpo/residual_abs": 0.09928829967975616, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.380287136400999e-08, + "logits/chosen": -2.439232349395752, + "logits/rejected": -2.102266788482666, + "logps/chosen": -0.2713320255279541, + "logps/rejected": -0.3047031760215759, + "loss": 0.08617246150970459, + "loss/core": 0.08617246150970459, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.5870898962020874, + "rewards/margins": 0.5577307343482971, + "rewards/rejected": 1.0293591022491455, + "step": 670 + }, + { + "epoch": 0.6450456907364271, + "grad_norm": 13.1875, + "ht_mnpo/logit": -0.06488653272390366, + "ht_mnpo/residual": -0.07238653302192688, + "ht_mnpo/residual_abs": 0.12053285539150238, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.004708203586628e-08, + "logits/chosen": -2.3484644889831543, + "logits/rejected": -2.097277879714966, + "logps/chosen": -0.2894912660121918, + "logps/rejected": -0.29819828271865845, + "loss": 0.35253381729125977, + "loss/core": 0.35253381729125977, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.9312378764152527, + "rewards/margins": -0.6488652229309082, + "rewards/rejected": 1.5801031589508057, + "step": 675 + }, + { + "epoch": 0.6498238069641044, + "grad_norm": 3.390625, + "ht_mnpo/logit": 0.08097716420888901, + "ht_mnpo/residual": 0.07347716391086578, + "ht_mnpo/residual_abs": 0.09472920745611191, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.635144445857407e-08, + "logits/chosen": -2.3550667762756348, + "logits/rejected": -2.101799964904785, + "logps/chosen": -0.27208298444747925, + "logps/rejected": -0.28226494789123535, + "loss": 0.09052277356386185, + "loss/core": 0.09052277356386185, + "rewards/accuracies": 0.75, + "rewards/chosen": 2.032668113708496, + "rewards/margins": 0.809771716594696, + "rewards/rejected": 1.2228964567184448, + "step": 680 + }, + { + "epoch": 0.6546019231917817, + "grad_norm": 5.09375, + "ht_mnpo/logit": -0.013841989450156689, + "ht_mnpo/residual": -0.02134198695421219, + "ht_mnpo/residual_abs": 0.12373582273721695, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.271734841028552e-08, + "logits/chosen": -2.093864679336548, + "logits/rejected": -1.9131803512573242, + "logps/chosen": -0.27449527382850647, + "logps/rejected": -0.2798653542995453, + "loss": 0.2753180265426636, + "loss/core": 0.2753180265426636, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.514013648033142, + "rewards/margins": -0.13841985166072845, + "rewards/rejected": 1.6524333953857422, + "step": 685 + }, + { + "epoch": 0.6593800394194589, + "grad_norm": 81.5, + "ht_mnpo/logit": 0.10633112490177155, + "ht_mnpo/residual": 0.09883112460374832, + "ht_mnpo/residual_abs": 0.16929301619529724, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 7.91461605259007e-08, + "logits/chosen": -2.175103187561035, + "logits/rejected": -2.002682685852051, + "logps/chosen": -0.28046196699142456, + "logps/rejected": -0.302472323179245, + "loss": 0.4346230924129486, + "loss/core": 0.4346230924129486, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.207476854324341, + "rewards/margins": 1.063311219215393, + "rewards/rejected": 2.144165515899658, + "step": 690 + }, + { + "epoch": 0.6641581556471361, + "grad_norm": 3.96875, + "ht_mnpo/logit": 0.037042342126369476, + "ht_mnpo/residual": 0.02954234555363655, + "ht_mnpo/residual_abs": 0.07325103133916855, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 7.563922378313217e-08, + "logits/chosen": -2.2522799968719482, + "logits/rejected": -2.0153305530548096, + "logps/chosen": -0.2950429916381836, + "logps/rejected": -0.29979586601257324, + "loss": 0.030671823769807816, + "loss/core": 0.030671823769807816, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.3066444396972656, + "rewards/margins": 0.37042343616485596, + "rewards/rejected": 0.9362211227416992, + "step": 695 + }, + { + "epoch": 0.6689362718748133, + "grad_norm": 176.0, + "ht_mnpo/logit": -0.02812144160270691, + "ht_mnpo/residual": -0.035621438175439835, + "ht_mnpo/residual_abs": 0.12590409815311432, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 7.219785699746572e-08, + "logits/chosen": -2.437011241912842, + "logits/rejected": -2.2066168785095215, + "logps/chosen": -0.2685781419277191, + "logps/rejected": -0.2836915850639343, + "loss": 0.22989264130592346, + "loss/core": 0.22989264130592346, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.5847865343093872, + "rewards/margins": -0.28121432662010193, + "rewards/rejected": 1.8660008907318115, + "step": 700 + }, + { + "epoch": 0.6737143881024906, + "grad_norm": 8.0625, + "ht_mnpo/logit": 0.09928493201732635, + "ht_mnpo/residual": 0.09178493171930313, + "ht_mnpo/residual_abs": 0.13132280111312866, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.882335432620779e-08, + "logits/chosen": -2.0853044986724854, + "logits/rejected": -1.8013473749160767, + "logps/chosen": -0.2867644727230072, + "logps/rejected": -0.31725770235061646, + "loss": 0.159874826669693, + "loss/core": 0.159874826669693, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.3235867023468018, + "rewards/margins": 0.9928493499755859, + "rewards/rejected": 1.3307373523712158, + "step": 705 + }, + { + "epoch": 0.6784925043301678, + "grad_norm": 1.71875, + "ht_mnpo/logit": 0.19838109612464905, + "ht_mnpo/residual": 0.19088108837604523, + "ht_mnpo/residual_abs": 0.19749945402145386, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.551698478180589e-08, + "logits/chosen": -2.0413479804992676, + "logits/rejected": -1.9397804737091064, + "logps/chosen": -0.2884281873703003, + "logps/rejected": -0.29842838644981384, + "loss": 0.7291596531867981, + "loss/core": 0.7291596531867981, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.16792631149292, + "rewards/margins": 1.9838106632232666, + "rewards/rejected": 1.1841155290603638, + "step": 710 + }, + { + "epoch": 0.683270620557845, + "grad_norm": 11.75, + "ht_mnpo/logit": 0.0204590056091547, + "ht_mnpo/residual": 0.012959008105099201, + "ht_mnpo/residual_abs": 0.06075044348835945, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.22799917546252e-08, + "logits/chosen": -2.1333656311035156, + "logits/rejected": -1.8975515365600586, + "logps/chosen": -0.2886390686035156, + "logps/rejected": -0.294972687959671, + "loss": 0.032932449132204056, + "loss/core": 0.032932449132204056, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 1.033470630645752, + "rewards/margins": 0.2045900523662567, + "rewards/rejected": 0.8288804888725281, + "step": 715 + }, + { + "epoch": 0.6880487367855223, + "grad_norm": 7.0625, + "ht_mnpo/logit": 0.07070360332727432, + "ht_mnpo/residual": 0.0632036104798317, + "ht_mnpo/residual_abs": 0.178388774394989, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.9113592545359944e-08, + "logits/chosen": -2.3582139015197754, + "logits/rejected": -2.154236316680908, + "logps/chosen": -0.27584514021873474, + "logps/rejected": -0.2994580864906311, + "loss": 0.5289813280105591, + "loss/core": 0.5289813280105591, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.8845659494400024, + "rewards/margins": 0.7070361375808716, + "rewards/rejected": 1.1775298118591309, + "step": 720 + }, + { + "epoch": 0.6928268530131996, + "grad_norm": 22.875, + "ht_mnpo/logit": 0.11204788833856583, + "ht_mnpo/residual": 0.1045479029417038, + "ht_mnpo/residual_abs": 0.23556935787200928, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.601897790725643e-08, + "logits/chosen": -2.127692699432373, + "logits/rejected": -1.873258352279663, + "logps/chosen": -0.2844100296497345, + "logps/rejected": -0.34677594900131226, + "loss": 0.522869884967804, + "loss/core": 0.522869884967804, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 3.1595566272735596, + "rewards/margins": 1.1204789876937866, + "rewards/rejected": 2.0390777587890625, + "step": 725 + }, + { + "epoch": 0.6976049692408768, + "grad_norm": 34.25, + "ht_mnpo/logit": 0.0576045997440815, + "ht_mnpo/residual": 0.05010459944605827, + "ht_mnpo/residual_abs": 0.14411763846874237, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.299731159831952e-08, + "logits/chosen": -2.3395884037017822, + "logits/rejected": -2.088940143585205, + "logps/chosen": -0.2896740436553955, + "logps/rejected": -0.2858702540397644, + "loss": 0.16655156016349792, + "loss/core": 0.16655156016349792, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.1040432453155518, + "rewards/margins": 0.5760459899902344, + "rewards/rejected": 1.5279972553253174, + "step": 730 + }, + { + "epoch": 0.702383085468554, + "grad_norm": 3.40625, + "ht_mnpo/logit": 0.04949961230158806, + "ht_mnpo/residual": 0.04199961572885513, + "ht_mnpo/residual_abs": 0.04821108654141426, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.0049729943671013e-08, + "logits/chosen": -2.2227725982666016, + "logits/rejected": -2.019742250442505, + "logps/chosen": -0.28382936120033264, + "logps/rejected": -0.2969701290130615, + "loss": 0.010463174432516098, + "loss/core": 0.010463174432516098, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 1.1914409399032593, + "rewards/margins": 0.4949961304664612, + "rewards/rejected": 0.6964449286460876, + "step": 735 + }, + { + "epoch": 0.7071612016962313, + "grad_norm": 1.71875, + "ht_mnpo/logit": 0.20136702060699463, + "ht_mnpo/residual": 0.193867027759552, + "ht_mnpo/residual_abs": 0.28303730487823486, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.7177341408223994e-08, + "logits/chosen": -2.123994827270508, + "logits/rejected": -1.8912299871444702, + "logps/chosen": -0.31486859917640686, + "logps/rejected": -0.25920960307121277, + "loss": 1.0651627779006958, + "loss/core": 1.0651627779006958, + "rewards/accuracies": 0.8125, + "rewards/chosen": 3.329946517944336, + "rewards/margins": 2.013669967651367, + "rewards/rejected": 1.3162766695022583, + "step": 740 + }, + { + "epoch": 0.7119393179239085, + "grad_norm": 57.0, + "ht_mnpo/logit": 0.07492740452289581, + "ht_mnpo/residual": 0.06742740422487259, + "ht_mnpo/residual_abs": 0.09558016806840897, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.438122617983442e-08, + "logits/chosen": -2.3298563957214355, + "logits/rejected": -2.0255560874938965, + "logps/chosen": -0.29623550176620483, + "logps/rejected": -0.29635828733444214, + "loss": 0.0679873377084732, + "loss/core": 0.0679873377084732, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.242774248123169, + "rewards/margins": 0.7492740750312805, + "rewards/rejected": 1.493499994277954, + "step": 745 + }, + { + "epoch": 0.7167174341515857, + "grad_norm": 4.71875, + "ht_mnpo/logit": 0.02682856284081936, + "ht_mnpo/residual": 0.019328564405441284, + "ht_mnpo/residual_abs": 0.10245944559574127, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.1662435763087114e-08, + "logits/chosen": -2.2698183059692383, + "logits/rejected": -2.0956759452819824, + "logps/chosen": -0.2996281385421753, + "logps/rejected": -0.3088133931159973, + "loss": 0.11470890045166016, + "loss/core": 0.11470890045166016, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.4476314783096313, + "rewards/margins": 0.2682856619358063, + "rewards/rejected": 1.179345726966858, + "step": 750 + }, + { + "epoch": 0.7214955503792629, + "grad_norm": 368.0, + "ht_mnpo/logit": 0.11839409917593002, + "ht_mnpo/residual": 0.1108940988779068, + "ht_mnpo/residual_abs": 0.19081872701644897, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.902199258386732e-08, + "logits/chosen": -2.1703250408172607, + "logits/rejected": -1.846046805381775, + "logps/chosen": -0.2736147940158844, + "logps/rejected": -0.27697068452835083, + "loss": 0.3880789875984192, + "loss/core": 0.3880789875984192, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": 3.0956952571868896, + "rewards/margins": 1.1839410066604614, + "rewards/rejected": 1.9117543697357178, + "step": 755 + }, + { + "epoch": 0.7262736666069403, + "grad_norm": 4.9375, + "ht_mnpo/logit": 0.1197742447257042, + "ht_mnpo/residual": 0.11227424442768097, + "ht_mnpo/residual_abs": 0.14611002802848816, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.646088960486862e-08, + "logits/chosen": -2.176957607269287, + "logits/rejected": -1.8870060443878174, + "logps/chosen": -0.30091139674186707, + "logps/rejected": -0.3017504811286926, + "loss": 0.22610747814178467, + "loss/core": 0.22610747814178467, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.655430793762207, + "rewards/margins": 1.1977424621582031, + "rewards/rejected": 1.457688570022583, + "step": 760 + }, + { + "epoch": 0.7310517828346175, + "grad_norm": 27.75, + "ht_mnpo/logit": 0.07762764394283295, + "ht_mnpo/residual": 0.07012763619422913, + "ht_mnpo/residual_abs": 0.12880326807498932, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.398008995217988e-08, + "logits/chosen": -1.9855482578277588, + "logits/rejected": -1.878286361694336, + "logps/chosen": -0.27580446004867554, + "logps/rejected": -0.27556952834129333, + "loss": 0.1310841143131256, + "loss/core": 0.1310841143131256, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.5765607357025146, + "rewards/margins": 0.7762764692306519, + "rewards/rejected": 1.8002843856811523, + "step": 765 + }, + { + "epoch": 0.7358298990622947, + "grad_norm": 6.0, + "ht_mnpo/logit": 0.10170654207468033, + "ht_mnpo/residual": 0.0942065417766571, + "ht_mnpo/residual_abs": 0.16815176606178284, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.1580526553093315e-08, + "logits/chosen": -2.3226284980773926, + "logits/rejected": -2.1349689960479736, + "logps/chosen": -0.29955098032951355, + "logps/rejected": -0.2934722900390625, + "loss": 0.691552460193634, + "loss/core": 0.691552460193634, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 2.19858980178833, + "rewards/margins": 1.0170655250549316, + "rewards/rejected": 1.1815242767333984, + "step": 770 + }, + { + "epoch": 0.7406080152899719, + "grad_norm": 7.53125, + "ht_mnpo/logit": 0.04109996557235718, + "ht_mnpo/residual": 0.033599965274333954, + "ht_mnpo/residual_abs": 0.19126072525978088, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.9263101785268252e-08, + "logits/chosen": -2.1104021072387695, + "logits/rejected": -1.9156906604766846, + "logps/chosen": -0.2990146577358246, + "logps/rejected": -0.3021544814109802, + "loss": 0.3742893636226654, + "loss/core": 0.3742893636226654, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.993298053741455, + "rewards/margins": 0.410999596118927, + "rewards/rejected": 2.5822982788085938, + "step": 775 + }, + { + "epoch": 0.7453861315176492, + "grad_norm": 17.5, + "ht_mnpo/logit": 0.12976470589637756, + "ht_mnpo/residual": 0.12226470559835434, + "ht_mnpo/residual_abs": 0.14971300959587097, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.7028687137384264e-08, + "logits/chosen": -2.1731667518615723, + "logits/rejected": -2.022315263748169, + "logps/chosen": -0.2888116240501404, + "logps/rejected": -0.30564987659454346, + "loss": 0.3799237906932831, + "loss/core": 0.3799237906932831, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.4324753284454346, + "rewards/margins": 1.2976469993591309, + "rewards/rejected": 1.1348284482955933, + "step": 780 + }, + { + "epoch": 0.7501642477453264, + "grad_norm": 896.0, + "ht_mnpo/logit": 0.2294764518737793, + "ht_mnpo/residual": 0.22197644412517548, + "ht_mnpo/residual_abs": 0.27920347452163696, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4878122881409447e-08, + "logits/chosen": -1.785697340965271, + "logits/rejected": -1.6078464984893799, + "logps/chosen": -0.30665987730026245, + "logps/rejected": -0.31216126680374146, + "loss": 0.8990079164505005, + "loss/core": 0.8990079164505005, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.655916690826416, + "rewards/margins": 2.294764757156372, + "rewards/rejected": 1.361151933670044, + "step": 785 + }, + { + "epoch": 0.7549423639730036, + "grad_norm": 4.96875, + "ht_mnpo/logit": 0.0998556837439537, + "ht_mnpo/residual": 0.09235568344593048, + "ht_mnpo/residual_abs": 0.1307271420955658, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.2812217756608937e-08, + "logits/chosen": -2.0636184215545654, + "logits/rejected": -1.8661634922027588, + "logps/chosen": -0.2779093384742737, + "logps/rejected": -0.27925077080726624, + "loss": 0.07385687530040741, + "loss/core": 0.07385687530040741, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 3.1228384971618652, + "rewards/margins": 0.998556911945343, + "rewards/rejected": 2.124281644821167, + "step": 790 + }, + { + "epoch": 0.7597204802006808, + "grad_norm": 1608.0, + "ht_mnpo/logit": 0.07121960818767548, + "ht_mnpo/residual": 0.06371960043907166, + "ht_mnpo/residual_abs": 0.16604584455490112, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.0831748665410763e-08, + "logits/chosen": -2.2949540615081787, + "logits/rejected": -2.0313305854797363, + "logps/chosen": -0.3159402906894684, + "logps/rejected": -0.31764060258865356, + "loss": 0.4635314345359802, + "loss/core": 0.4635314345359802, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.100029230117798, + "rewards/margins": 0.7121961116790771, + "rewards/rejected": 1.3878331184387207, + "step": 795 + }, + { + "epoch": 0.7644985964283582, + "grad_norm": 292.0, + "ht_mnpo/logit": 0.012357105500996113, + "ht_mnpo/residual": 0.004857116844505072, + "ht_mnpo/residual_abs": 0.20216397941112518, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8937460381244967e-08, + "logits/chosen": -2.1980741024017334, + "logits/rejected": -1.9934120178222656, + "logps/chosen": -0.24281847476959229, + "logps/rejected": -0.2602716088294983, + "loss": 0.5272241234779358, + "loss/core": 0.5272241234779358, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.3460819721221924, + "rewards/margins": 0.12357117235660553, + "rewards/rejected": 2.222511053085327, + "step": 800 + }, + { + "epoch": 0.7692767126560354, + "grad_norm": 508.0, + "ht_mnpo/logit": -0.049846429377794266, + "ht_mnpo/residual": -0.05734643340110779, + "ht_mnpo/residual_abs": 0.2035159170627594, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.713006526846439e-08, + "logits/chosen": -2.2257566452026367, + "logits/rejected": -1.9966598749160767, + "logps/chosen": -0.29499825835227966, + "logps/rejected": -0.32620564103126526, + "loss": 0.7258022427558899, + "loss/core": 0.7258022427558899, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.3188430070877075, + "rewards/margins": -0.4984644055366516, + "rewards/rejected": 1.8173072338104248, + "step": 805 + }, + { + "epoch": 0.7740548288837126, + "grad_norm": 1.203125, + "ht_mnpo/logit": -0.05915957689285278, + "ht_mnpo/residual": -0.06665956974029541, + "ht_mnpo/residual_abs": 0.12409082800149918, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.541024301445404e-08, + "logits/chosen": -2.266317844390869, + "logits/rejected": -1.98003351688385, + "logps/chosen": -0.29652053117752075, + "logps/rejected": -0.2886161804199219, + "loss": 0.35598623752593994, + "loss/core": 0.35598623752593994, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 1.1817328929901123, + "rewards/margins": -0.5915957689285278, + "rewards/rejected": 1.7733285427093506, + "step": 810 + }, + { + "epoch": 0.7788329451113898, + "grad_norm": 4.53125, + "ht_mnpo/logit": -0.044502951204776764, + "ht_mnpo/residual": -0.052002958953380585, + "ht_mnpo/residual_abs": 0.22781427204608917, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.3778640374027983e-08, + "logits/chosen": -2.026960849761963, + "logits/rejected": -1.768654227256775, + "logps/chosen": -0.2787315249443054, + "logps/rejected": -0.32032424211502075, + "loss": 0.4553064703941345, + "loss/core": 0.4553064703941345, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.9568564891815186, + "rewards/margins": -0.44502943754196167, + "rewards/rejected": 2.401885747909546, + "step": 815 + }, + { + "epoch": 0.7836110613390671, + "grad_norm": 178.0, + "ht_mnpo/logit": 0.08812891691923141, + "ht_mnpo/residual": 0.0806289091706276, + "ht_mnpo/residual_abs": 0.19937752187252045, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.2235870926211616e-08, + "logits/chosen": -2.264019012451172, + "logits/rejected": -2.017019271850586, + "logps/chosen": -0.2904714047908783, + "logps/rejected": -0.2898346483707428, + "loss": 0.6707652807235718, + "loss/core": 0.6707652807235718, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.9655208587646484, + "rewards/margins": 0.8812893033027649, + "rewards/rejected": 2.0842318534851074, + "step": 820 + }, + { + "epoch": 0.7883891775667443, + "grad_norm": 102.5, + "ht_mnpo/logit": 0.14062213897705078, + "ht_mnpo/residual": 0.13312214612960815, + "ht_mnpo/residual_abs": 0.18947193026542664, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0782514843499652e-08, + "logits/chosen": -2.1509218215942383, + "logits/rejected": -1.9344791173934937, + "logps/chosen": -0.30122148990631104, + "logps/rejected": -0.30858007073402405, + "loss": 0.46916574239730835, + "loss/core": 0.46916574239730835, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.392636299133301, + "rewards/margins": 1.4062215089797974, + "rewards/rejected": 0.986414909362793, + "step": 825 + }, + { + "epoch": 0.7931672937944215, + "grad_norm": 368.0, + "ht_mnpo/logit": 0.21851785480976105, + "ht_mnpo/residual": 0.21101781725883484, + "ht_mnpo/residual_abs": 0.30629879236221313, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.419118673676924e-09, + "logits/chosen": -2.0330405235290527, + "logits/rejected": -1.8500254154205322, + "logps/chosen": -0.2705790102481842, + "logps/rejected": -0.3051673471927643, + "loss": 0.8804137110710144, + "loss/core": 0.8804137110710144, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.997424602508545, + "rewards/margins": 2.185178279876709, + "rewards/rejected": 1.8122460842132568, + "step": 830 + }, + { + "epoch": 0.7979454100220987, + "grad_norm": 16.75, + "ht_mnpo/logit": 0.06802691519260406, + "ht_mnpo/residual": 0.06052691861987114, + "ht_mnpo/residual_abs": 0.10366304218769073, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.14619513428405e-09, + "logits/chosen": -2.204408884048462, + "logits/rejected": -2.0475637912750244, + "logps/chosen": -0.2984268367290497, + "logps/rejected": -0.3082461953163147, + "loss": 0.08803679049015045, + "loss/core": 0.08803679049015045, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.8154687881469727, + "rewards/margins": 0.6802691221237183, + "rewards/rejected": 1.1351996660232544, + "step": 835 + }, + { + "epoch": 0.802723526249776, + "grad_norm": 94.5, + "ht_mnpo/logit": 0.1807180941104889, + "ht_mnpo/residual": 0.17321810126304626, + "ht_mnpo/residual_abs": 0.2219761312007904, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.96422291980539e-09, + "logits/chosen": -2.1479125022888184, + "logits/rejected": -1.9582202434539795, + "logps/chosen": -0.3181340992450714, + "logps/rejected": -0.2776985466480255, + "loss": 0.5196969509124756, + "loss/core": 0.5196969509124756, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 3.3365378379821777, + "rewards/margins": 1.8071807622909546, + "rewards/rejected": 1.5293573141098022, + "step": 840 + }, + { + "epoch": 0.8075016424774533, + "grad_norm": 3.1875, + "ht_mnpo/logit": 0.021912937983870506, + "ht_mnpo/residual": 0.014412937685847282, + "ht_mnpo/residual_abs": 0.0572718009352684, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.8736465216517594e-09, + "logits/chosen": -2.237180709838867, + "logits/rejected": -1.9485756158828735, + "logps/chosen": -0.30848002433776855, + "logps/rejected": -0.3073003888130188, + "loss": 0.02010386623442173, + "loss/core": 0.02010386623442173, + "rewards/accuracies": 0.75, + "rewards/chosen": 1.0542051792144775, + "rewards/margins": 0.21912936866283417, + "rewards/rejected": 0.8350757360458374, + "step": 845 + }, + { + "epoch": 0.8122797587051305, + "grad_norm": 524.0, + "ht_mnpo/logit": 0.1235678419470787, + "ht_mnpo/residual": 0.11606784909963608, + "ht_mnpo/residual_abs": 0.15284273028373718, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.874876061005173e-09, + "logits/chosen": -2.293239116668701, + "logits/rejected": -2.054229974746704, + "logps/chosen": -0.3143402934074402, + "logps/rejected": -0.31967416405677795, + "loss": 0.381173700094223, + "loss/core": 0.381173700094223, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.127030849456787, + "rewards/margins": 1.2356785535812378, + "rewards/rejected": 0.8913521766662598, + "step": 850 + }, + { + "epoch": 0.8170578749328078, + "grad_norm": 10.9375, + "ht_mnpo/logit": 0.14451901614665985, + "ht_mnpo/residual": 0.13701900839805603, + "ht_mnpo/residual_abs": 0.28897708654403687, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.968287134589188e-09, + "logits/chosen": -2.1433966159820557, + "logits/rejected": -1.8137801885604858, + "logps/chosen": -0.2558881938457489, + "logps/rejected": -0.2788417935371399, + "loss": 1.2627418041229248, + "loss/core": 1.2627418041229248, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 3.3266005516052246, + "rewards/margins": 1.445190191268921, + "rewards/rejected": 1.8814102411270142, + "step": 855 + }, + { + "epoch": 0.821835991160485, + "grad_norm": 47.0, + "ht_mnpo/logit": 0.06459364295005798, + "ht_mnpo/residual": 0.05709363892674446, + "ht_mnpo/residual_abs": 0.1335936039686203, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.154220673422192e-09, + "logits/chosen": -2.194952964782715, + "logits/rejected": -2.061500310897827, + "logps/chosen": -0.3744109570980072, + "logps/rejected": -0.30319473147392273, + "loss": 0.15060871839523315, + "loss/core": 0.15060871839523315, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.501992702484131, + "rewards/margins": 0.6459364295005798, + "rewards/rejected": 1.8560559749603271, + "step": 860 + }, + { + "epoch": 0.8266141073881622, + "grad_norm": 1.6015625, + "ht_mnpo/logit": 0.14823171496391296, + "ht_mnpo/residual": 0.14073172211647034, + "ht_mnpo/residual_abs": 0.17731209099292755, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4329828146074096e-09, + "logits/chosen": -2.154472827911377, + "logits/rejected": -1.924818992614746, + "logps/chosen": -0.3006120026111603, + "logps/rejected": -0.309847354888916, + "loss": 0.4654766023159027, + "loss/core": 0.4654766023159027, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.900441884994507, + "rewards/margins": 1.4823172092437744, + "rewards/rejected": 1.418124794960022, + "step": 865 + }, + { + "epoch": 0.8313922236158394, + "grad_norm": 1.65625, + "ht_mnpo/logit": 0.08181913942098618, + "ht_mnpo/residual": 0.07431914657354355, + "ht_mnpo/residual_abs": 0.10339722782373428, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8048447862070714e-09, + "logits/chosen": -2.1827683448791504, + "logits/rejected": -1.9179191589355469, + "logps/chosen": -0.28617894649505615, + "logps/rejected": -0.2885843515396118, + "loss": 0.17244181036949158, + "loss/core": 0.17244181036949158, + "rewards/accuracies": 0.6875, + "rewards/chosen": 2.336758852005005, + "rewards/margins": 0.8181915283203125, + "rewards/rejected": 1.5185672044754028, + "step": 870 + }, + { + "epoch": 0.8361703398435167, + "grad_norm": 11.25, + "ht_mnpo/logit": 0.05557389184832573, + "ht_mnpo/residual": 0.04807388782501221, + "ht_mnpo/residual_abs": 0.18917661905288696, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.2700428052447033e-09, + "logits/chosen": -2.171966075897217, + "logits/rejected": -1.9326622486114502, + "logps/chosen": -0.2600437104701996, + "logps/rejected": -0.2582940459251404, + "loss": 0.24842104315757751, + "loss/core": 0.24842104315757751, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.4055025577545166, + "rewards/margins": 0.5557388067245483, + "rewards/rejected": 1.8497635126113892, + "step": 875 + }, + { + "epoch": 0.840948456071194, + "grad_norm": 15.9375, + "ht_mnpo/logit": 0.0831659659743309, + "ht_mnpo/residual": 0.07566596567630768, + "ht_mnpo/residual_abs": 0.11803103983402252, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.287779888734858e-10, + "logits/chosen": -2.134854555130005, + "logits/rejected": -1.8504482507705688, + "logps/chosen": -0.25479409098625183, + "logps/rejected": -0.27924638986587524, + "loss": 0.12039796262979507, + "loss/core": 0.12039796262979507, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.021028757095337, + "rewards/margins": 0.8316596150398254, + "rewards/rejected": 1.1893693208694458, + "step": 880 + }, + { + "epoch": 0.8457265722988712, + "grad_norm": 19.625, + "ht_mnpo/logit": 0.0474698506295681, + "ht_mnpo/residual": 0.039969850331544876, + "ht_mnpo/residual_abs": 0.05242999643087387, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.812162787445062e-10, + "logits/chosen": -2.2313616275787354, + "logits/rejected": -2.0359046459198, + "logps/chosen": -0.328023761510849, + "logps/rejected": -0.3314855694770813, + "loss": 0.025688689202070236, + "loss/core": 0.025688689202070236, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.1934434175491333, + "rewards/margins": 0.474698543548584, + "rewards/rejected": 0.7187449336051941, + "step": 885 + }, + { + "epoch": 0.8505046885265484, + "grad_norm": 207.0, + "ht_mnpo/logit": 0.13611260056495667, + "ht_mnpo/residual": 0.12861260771751404, + "ht_mnpo/residual_abs": 0.22040756046772003, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.2748837860270265e-10, + "logits/chosen": -2.1162843704223633, + "logits/rejected": -1.941434621810913, + "logps/chosen": -0.31317272782325745, + "logps/rejected": -0.3353462815284729, + "loss": 0.7812239527702332, + "loss/core": 0.7812239527702332, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.680392265319824, + "rewards/margins": 1.361126184463501, + "rewards/rejected": 1.3192662000656128, + "step": 890 + }, + { + "epoch": 0.8552828047542257, + "grad_norm": 588.0, + "ht_mnpo/logit": -0.056675732135772705, + "ht_mnpo/residual": -0.06417573988437653, + "ht_mnpo/residual_abs": 0.21688473224639893, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.768970513457151e-11, + "logits/chosen": -2.205193042755127, + "logits/rejected": -1.8673622608184814, + "logps/chosen": -0.29924076795578003, + "logps/rejected": -0.3369528651237488, + "loss": 0.5442713499069214, + "loss/core": 0.5442713499069214, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.8029203414916992, + "rewards/margins": -0.566757321357727, + "rewards/rejected": 2.369677782058716, + "step": 895 + }, + { + "epoch": 0.8600609209819029, + "grad_norm": 1.1796875, + "ht_mnpo/logit": -0.059881217777729034, + "ht_mnpo/residual": -0.06738121807575226, + "ht_mnpo/residual_abs": 0.10789163410663605, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8803520859811406e-12, + "logits/chosen": -2.390237331390381, + "logits/rejected": -2.1097664833068848, + "logps/chosen": -0.30715036392211914, + "logps/rejected": -0.3250555992126465, + "loss": 0.425441175699234, + "loss/core": 0.425441175699234, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 0.8894049525260925, + "rewards/margins": -0.5988121628761292, + "rewards/rejected": 1.4882172346115112, + "step": 900 + }, + { + "epoch": 0.8600609209819029, + "step": 900, + "total_flos": 0.0, + "train_loss": 0.3132047750718064, + "train_runtime": 7069.0811, + "train_samples_per_second": 2.037, + "train_steps_per_second": 0.127 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..b4b56ee --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6f007c24fdb5dfd39d3e2ab3491211022ed5157d1bb3f7e775837999652cd44 +size 7057