From c1dfc4f2fafed2afe71592ac3d859fe4494d014a Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Thu, 23 Jul 2026 15:34:10 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: promotion/qwen3-8b-aaai27-flagship-ht-mnpo-helpfulness-s43 Source: Original Platform --- .gitattributes | 36 + README.md | 24 + all_results.json | 9 + chat_template.jinja | 89 + config.json | 71 + config.yaml | 58 + generation_config.json | 12 + job_status.json | 23 + model.safetensors | 3 + objective_protocol.json | 69 + provenance.json | 14 + resource_profile.json | 21 + run_status.json | 15 + split_manifest.json | 23 + stability_gate.json | 38 + tokenizer.json | 3 + tokenizer_config.json | 30 + train_results.json | 9 + trainer_state.json | 3823 +++++++++++++++++++++++++++++++++++++++ training_args.bin | 3 + 20 files changed, 4373 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 config.yaml create mode 100644 generation_config.json create mode 100644 job_status.json create mode 100644 model.safetensors create mode 100644 objective_protocol.json create mode 100644 provenance.json create mode 100644 resource_profile.json create mode 100644 run_status.json create mode 100644 split_manifest.json create mode 100644 stability_gate.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..f0f3600 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +--- +library_name: transformers +base_model: Qwen/Qwen3-8B +tags: +- ronpo +- mnpo +- preference-optimization +--- + +# qwen3-8b-aaai27-flagship-ht-mnpo-helpfulness-s43 + +Research checkpoint for the RONPO AAAI revision experiments. + +- Method: ht_mnpo_helpfulness +- Base model: `Qwen/Qwen3-8B` +- Seed: 43 +- Local source at upload time: `/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/full/ht_mnpo_helpfulness/seed43/attempt1` +- Uploaded at UTC: 2026-07-13T11:40:06Z +- Run status metadata: `{"attempt": 1, "effective_batch_size": 16, "method": "ht_mnpo_helpfulness", "objective_protocol": "objective_protocol.json", "optimizer_steps": 900, "resource_profile": "resource_profile.json", "scope_amendment": "scope_amendment_20260713.json", "seed": 43, "split_manifest": "split_manifest.json", "stability_gate": "stability_gate.json", "status": "completed", "wandb_run_id": "14a1a6f2db76", "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/14a1a6f2db76"}` + +AAAI-27 P1 matched budget: 900 optimizer steps, effective batch 16; passed non-thinking and collapse stability gates; sealed-test metrics are not used for checkpoint selection. + +Intended use: reproducibility and evaluation for the RONPO paper. This +checkpoint is not intended as a production assistant. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..d528ce7 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8600609209819029, + "total_flos": 0.0, + "train_loss": 0.3149479303053684, + "train_runtime": 7160.3559, + "train_samples": 16743, + "train_samples_per_second": 2.011, + "train_steps_per_second": 0.126 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..7bfb577 --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +model_name_or_path: Qwen/Qwen3-8B +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/ht_mnpo_helpfulness: 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: ht_mnpo +eta: 0.0075 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.0 +preference_sft_weight: 0.0 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 5.0e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 43 +gradient_accumulation_steps: 16 +gradient_checkpointing: true +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 1 +per_device_eval_batch_size: 1 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/full/ht_mnpo_helpfulness/seed43/attempt1 +run_name: aaai27-flagship-full-ht_mnpo_helpfulness-s43-a1 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/job_status.json b/job_status.json new file mode 100644 index 0000000..584578f --- /dev/null +++ b/job_status.json @@ -0,0 +1,23 @@ +{ + "status": "completed", + "stage": "full", + "method": "ht_mnpo_helpfulness", + "seed": 43, + "attempt": 1, + "gpu": 3, + "gpus": [ + 3 + ], + "returncode": 0, + "finite_metrics": true, + "oom": false, + "traceback": false, + "reason": "passed", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "14a1a6f2db76", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/14a1a6f2db76", + "output_dir": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/full/ht_mnpo_helpfulness/seed43/attempt1", + "log": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/logs/full_ht_mnpo_helpfulness_s43_a1.log", + "completed_at": "2026-07-13T11:37:37Z" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..efbe559 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c250d50124e4949a383b9672d31ce0fb0d7b70783aa07e54e521cf93f4eb9558 +size 16381517208 diff --git a/objective_protocol.json b/objective_protocol.json new file mode 100644 index 0000000..35b3b44 --- /dev/null +++ b/objective_protocol.json @@ -0,0 +1,69 @@ +{ + "schema_version": 1, + "frozen_at_kst": "2026-07-12T21:40:00+09:00", + "base_model": "Qwen/Qwen3-8B", + "primary_triple": { + "helpfulness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "ultrafeedback-helpfulness", + "transform": "identity" + }, + "safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "conciseness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "helpsteer-verbosity", + "transform": "negate" + } + }, + "gate_thresholds": { + "raw_score_std_strictly_greater_than": 0.01, + "aggregate_prompt_pair_spearman_median_strictly_less_than": 0.0, + "aggregate_pairwise_top1_mismatch_at_least": 0.5 + }, + "gate_pool": "source-train Qwen3-8B response pool split into deterministic train/validation; sealed source-test excluded", + "training_seeds": [42, 43, 44], + "optimizer_steps": 900, + "effective_batch_size": 16, + "decode": { + "enable_thinking": false, + "temperature": 0.7, + "top_p": 0.9, + "max_new_tokens": 2048, + "decode_seed": 42 + }, + "methods": [ + "ronpo_full_expect", + "ronpo_k_only", + "dpo", + "ipo", + "simpo", + "kto", + "sppo_avg", + "inpo_avg", + "ht_mnpo_helpfulness", + "ht_mnpo_safety", + "ht_mnpo_conciseness" + ], + "selection_policy": "No sealed-test selection. One preregistered configuration per method; failures use only the declared stability retry recipe, at most three attempts.", + "stability_gate": { + "validation_prompts": 128, + "min_candidate_base_mean_word_ratio": 0.33, + "max_candidate_base_mean_word_ratio": 2.0, + "max_consecutive_identical_word_run": 20, + "think_tag_count": 0, + "empty_response_count": 0, + "retry_recipe": "attempt 2 halves learning rate, enables gradient checkpointing, and adds reference=0.02/SFT=0.002 anchoring to unanchored unified losses; attempt 3 quarters learning rate and uses reference=0.05/SFT=0.005; RONPO uses reference=0.05/SFT=0.005 from attempt 1" + }, + "primary_metric": "mean prompt-level minimum of per-prompt min-max-normalized helpfulness, safety, conciseness", + "bootstrap": { + "unit": "prompt", + "paired": true, + "resamples": 2000, + "interval": "percentile_95", + "seed": 42 + } +} diff --git a/provenance.json b/provenance.json new file mode 100644 index 0000000..e93274a --- /dev/null +++ b/provenance.json @@ -0,0 +1,14 @@ +{ + "method": "ht_mnpo_helpfulness", + "seed": 43, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "14a1a6f2db76", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/14a1a6f2db76", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json", + "scope_amendment": "scope_amendment_20260713.json" +} diff --git a/resource_profile.json b/resource_profile.json new file mode 100644 index 0000000..18eec47 --- /dev/null +++ b/resource_profile.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "measured_at_kst": "2026-07-12T22:38:00+09:00", + "hardware": "NVIDIA B200 183359 MiB reported total; 178.35 GiB PyTorch-visible capacity", + "smoke_evidence": { + "attempted_profile": {"per_device_batch_size": 2, "gradient_accumulation_steps": 8, "gradient_checkpointing": false, "effective_batch_size": 16}, + "outcome": "OOM before completing the 20-step smoke across every launched unified baseline; representative process used 176.87 GiB and failed a 1.62 GiB allocation. KTO also OOMed at the optimizer step.", + "paper_metric": false + }, + "selected_full_training_profile": { + "per_device_batch_size": 2, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": false, + "attention_implementation": "PyTorch SDPA with cuDNN SDPA disabled", + "effective_batch_size": 16, + "applies_symmetrically_to_all_unified_pairwise_methods": true, + "kto_implementation_constraint": "TRL KTO requires actual batch size > 1. KTO uses batch 2 / accumulation 8 and identical AdamW with foreach=False. PyTorch SDPA is used with cuDNN SDPA disabled. The fixed base reference model is placed on a second B200 and its exact logits are returned to the policy device; the policy/optimizer remain on the first B200. This avoids a measured ~6-hour reference-precompute path while leaving the KTO loss, actual/effective batch, optimizer, and 900-step budget unchanged." + }, + "retry_semantics": "Resource selection is separated from method-stability attempts. Full attempt 1 uses the smoke-validated SDPA batch-2 profile and retains the preregistered method LR/anchor; attempts 2 and 3 fall back to batch 1 / accumulation 16 / gradient checkpointing and apply the declared LR/anchor stabilization recipe.", + "provenance_note": "No reward or sealed-evaluation result was available when this hardware profile was selected." +} diff --git a/run_status.json b/run_status.json new file mode 100644 index 0000000..4a96e2a --- /dev/null +++ b/run_status.json @@ -0,0 +1,15 @@ +{ + "method": "ht_mnpo_helpfulness", + "seed": 43, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "14a1a6f2db76", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/14a1a6f2db76", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json", + "scope_amendment": "scope_amendment_20260713.json", + "status": "completed" +} diff --git a/split_manifest.json b/split_manifest.json new file mode 100644 index 0000000..4fb96bd --- /dev/null +++ b/split_manifest.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "split_rule": "sort source-train prompts by sha256('flagship-v1||'+prompt_id); first round(10%) validation", + "validation_fraction": 0.1, + "source_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/avg_oracle", + "base_model": "Qwen/Qwen3-8B", + "counts": { + "train": 16780, + "validation": 1864, + "sealed_test": 604 + }, + "prompt_id_sha256": { + "train": "fcf6f2ea5a0b7bcd41179e6f5288f4d850b2d04bdb729da91907bd18bf733a69", + "validation": "a87ba6f519192164daa487b109a6547dd8cbde46a27a373d6da201ca2d061b8f", + "sealed_test": "b398ced6af6cf73d731e1a5f32a0631d8060323d51eda1e375695566bf5f1867" + }, + "overlaps": { + "train_validation": 0, + "train_sealed_test": 0, + "validation_sealed_test": 0 + }, + "sealed_test_policy": "prompts only until S4a; no objective scores or model generations used in S0-S3" +} diff --git a/stability_gate.json b/stability_gate.json new file mode 100644 index 0000000..002f98e --- /dev/null +++ b/stability_gate.json @@ -0,0 +1,38 @@ +{ + "status": "passed", + "passed": true, + "base": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 388.1484375, + "max_words": 1379, + "max_repeat_run": 4 + }, + "candidate": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 387.546875, + "max_words": 1399, + "max_repeat_run": 3 + }, + "candidate_base_mean_word_ratio": 0.9984501741038182, + "thresholds": { + "min_length_ratio": 0.33, + "max_length_ratio": 2.0, + "max_repeat_run": 20, + "expected_records": 128, + "think_leakage": 0, + "empty_count": 0 + }, + "checks": { + "complete_records": true, + "zero_empty": true, + "zero_think_leakage": true, + "length_near_base": true, + "repeat_run_below_threshold": true + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ea3c9e2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..d528ce7 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8600609209819029, + "total_flos": 0.0, + "train_loss": 0.3149479303053684, + "train_runtime": 7160.3559, + "train_samples": 16743, + "train_samples_per_second": 2.011, + "train_steps_per_second": 0.126 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..0608882 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,3823 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8600609209819029, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.004778116227677238, + "grad_norm": 250.0, + "ht_mnpo/logit": 0.07157169282436371, + "ht_mnpo/residual": 0.06407169252634048, + "ht_mnpo/residual_abs": 0.18986251950263977, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.222222222222222e-08, + "logits/chosen": -2.270185708999634, + "logits/rejected": -1.9820005893707275, + "logps/chosen": -0.26669391989707947, + "logps/rejected": -0.28120771050453186, + "loss": 0.27043819427490234, + "loss/core": 0.27043819427490234, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 2.370647430419922, + "rewards/margins": 0.7157168388366699, + "rewards/rejected": 1.6549304723739624, + "step": 5 + }, + { + "epoch": 0.009556232455354476, + "grad_norm": 156.0, + "ht_mnpo/logit": 0.02802388370037079, + "ht_mnpo/residual": 0.020523879677057266, + "ht_mnpo/residual_abs": 0.1323598325252533, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5e-08, + "logits/chosen": -2.2279629707336426, + "logits/rejected": -1.9304723739624023, + "logps/chosen": -0.3020592927932739, + "logps/rejected": -0.304666131734848, + "loss": 0.18070298433303833, + "loss/core": 0.18070298433303833, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.5490913391113281, + "rewards/margins": 0.2802388072013855, + "rewards/rejected": 1.2688524723052979, + "step": 10 + }, + { + "epoch": 0.014334348683031715, + "grad_norm": 22.375, + "ht_mnpo/logit": 0.052930913865566254, + "ht_mnpo/residual": 0.04543091356754303, + "ht_mnpo/residual_abs": 0.23316678404808044, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 7.777777777777778e-08, + "logits/chosen": -2.215111255645752, + "logits/rejected": -1.8859649896621704, + "logps/chosen": -0.2659865915775299, + "logps/rejected": -0.2871326804161072, + "loss": 0.708215057849884, + "loss/core": 0.708215057849884, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.4834492206573486, + "rewards/margins": 0.529309093952179, + "rewards/rejected": 1.954140067100525, + "step": 15 + }, + { + "epoch": 0.019112464910708952, + "grad_norm": 10.375, + "ht_mnpo/logit": 0.099542997777462, + "ht_mnpo/residual": 0.09204299747943878, + "ht_mnpo/residual_abs": 0.11109743267297745, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0555555555555555e-07, + "logits/chosen": -2.171400785446167, + "logits/rejected": -2.030099630355835, + "logps/chosen": -0.27144134044647217, + "logps/rejected": -0.28609132766723633, + "loss": 0.08086283504962921, + "loss/core": 0.08086283504962921, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 1.783445119857788, + "rewards/margins": 0.9954298734664917, + "rewards/rejected": 0.7880153656005859, + "step": 20 + }, + { + "epoch": 0.023890581138386192, + "grad_norm": 1.96875, + "ht_mnpo/logit": 0.12193785607814789, + "ht_mnpo/residual": 0.11443785578012466, + "ht_mnpo/residual_abs": 0.19381213188171387, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.3333333333333334e-07, + "logits/chosen": -2.0908684730529785, + "logits/rejected": -1.8903487920761108, + "logps/chosen": -0.27843743562698364, + "logps/rejected": -0.2850627899169922, + "loss": 0.640637218952179, + "loss/core": 0.640637218952179, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.377429485321045, + "rewards/margins": 1.2193785905838013, + "rewards/rejected": 1.158050775527954, + "step": 25 + }, + { + "epoch": 0.02866869736606343, + "grad_norm": 110.5, + "ht_mnpo/logit": 0.003302433993667364, + "ht_mnpo/residual": -0.004197552800178528, + "ht_mnpo/residual_abs": 0.19736668467521667, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.611111111111111e-07, + "logits/chosen": -2.2374136447906494, + "logits/rejected": -2.1604771614074707, + "logps/chosen": -0.2896232604980469, + "logps/rejected": -0.2754831314086914, + "loss": 0.5999057292938232, + "loss/core": 0.5999057292938232, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.1399874687194824, + "rewards/margins": 0.03302428871393204, + "rewards/rejected": 2.1069629192352295, + "step": 30 + }, + { + "epoch": 0.03344681359374067, + "grad_norm": 0.51953125, + "ht_mnpo/logit": 0.14462466537952423, + "ht_mnpo/residual": 0.1371246576309204, + "ht_mnpo/residual_abs": 0.15740394592285156, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8888888888888888e-07, + "logits/chosen": -2.2604546546936035, + "logits/rejected": -2.0551838874816895, + "logps/chosen": -0.2980991303920746, + "logps/rejected": -0.2945564389228821, + "loss": 0.8268828392028809, + "loss/core": 0.8268828392028809, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.3615596294403076, + "rewards/margins": 1.44624662399292, + "rewards/rejected": 0.9153128862380981, + "step": 35 + }, + { + "epoch": 0.038224929821417904, + "grad_norm": 1.0078125, + "ht_mnpo/logit": 0.06618395447731018, + "ht_mnpo/residual": 0.05868394300341606, + "ht_mnpo/residual_abs": 0.1978856474161148, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.1666666666666667e-07, + "logits/chosen": -2.577958822250366, + "logits/rejected": -2.3133766651153564, + "logps/chosen": -0.2794923782348633, + "logps/rejected": -0.2849942445755005, + "loss": 0.6078843474388123, + "loss/core": 0.6078843474388123, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.4120702743530273, + "rewards/margins": 0.6618391275405884, + "rewards/rejected": 1.7502310276031494, + "step": 40 + }, + { + "epoch": 0.04300304604909514, + "grad_norm": 47.25, + "ht_mnpo/logit": 0.02121608331799507, + "ht_mnpo/residual": 0.013716082088649273, + "ht_mnpo/residual_abs": 0.14117369055747986, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4444444444444445e-07, + "logits/chosen": -2.0499496459960938, + "logits/rejected": -1.7805042266845703, + "logps/chosen": -0.26364803314208984, + "logps/rejected": -0.2629775404930115, + "loss": 0.11064072698354721, + "loss/core": 0.11064072698354721, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 2.472045421600342, + "rewards/margins": 0.21216082572937012, + "rewards/rejected": 2.259884834289551, + "step": 45 + }, + { + "epoch": 0.047781162276772385, + "grad_norm": 161.0, + "ht_mnpo/logit": 0.08355962485074997, + "ht_mnpo/residual": 0.07605963945388794, + "ht_mnpo/residual_abs": 0.1379310041666031, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.7222222222222216e-07, + "logits/chosen": -2.0790011882781982, + "logits/rejected": -1.9761953353881836, + "logps/chosen": -0.2877221703529358, + "logps/rejected": -0.29205089807510376, + "loss": 0.10855451971292496, + "loss/core": 0.10855451971292496, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.1596500873565674, + "rewards/margins": 0.8355963826179504, + "rewards/rejected": 1.3240535259246826, + "step": 50 + }, + { + "epoch": 0.05255927850444962, + "grad_norm": 0.4609375, + "ht_mnpo/logit": 0.044849641621112823, + "ht_mnpo/residual": 0.037349633872509, + "ht_mnpo/residual_abs": 0.13990451395511627, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3e-07, + "logits/chosen": -2.390953302383423, + "logits/rejected": -2.1549553871154785, + "logps/chosen": -0.2689022123813629, + "logps/rejected": -0.2913646996021271, + "loss": 0.48172539472579956, + "loss/core": 0.48172539472579956, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.500528335571289, + "rewards/margins": 0.44849643111228943, + "rewards/rejected": 1.0520318746566772, + "step": 55 + }, + { + "epoch": 0.05733739473212686, + "grad_norm": 2.609375, + "ht_mnpo/logit": 0.05682535097002983, + "ht_mnpo/residual": 0.04932534694671631, + "ht_mnpo/residual_abs": 0.09572498500347137, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.2777777777777776e-07, + "logits/chosen": -2.295361042022705, + "logits/rejected": -2.1259632110595703, + "logps/chosen": -0.2934691905975342, + "logps/rejected": -0.2943810522556305, + "loss": 0.0800308957695961, + "loss/core": 0.0800308957695961, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.6094799041748047, + "rewards/margins": 0.5682535171508789, + "rewards/rejected": 1.0412262678146362, + "step": 60 + }, + { + "epoch": 0.062115510959804096, + "grad_norm": 63.0, + "ht_mnpo/logit": 0.060111574828624725, + "ht_mnpo/residual": 0.0526115782558918, + "ht_mnpo/residual_abs": 0.09094792604446411, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.5555555555555553e-07, + "logits/chosen": -2.3533196449279785, + "logits/rejected": -2.0280025005340576, + "logps/chosen": -0.3026924431324005, + "logps/rejected": -0.3124319016933441, + "loss": 0.05209514498710632, + "loss/core": 0.05209514498710632, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.5579849481582642, + "rewards/margins": 0.6011158227920532, + "rewards/rejected": 0.9568690061569214, + "step": 65 + }, + { + "epoch": 0.06689362718748133, + "grad_norm": 22.625, + "ht_mnpo/logit": 0.0568360760807991, + "ht_mnpo/residual": 0.04933607205748558, + "ht_mnpo/residual_abs": 0.08929934352636337, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.8333333333333335e-07, + "logits/chosen": -2.2468817234039307, + "logits/rejected": -2.062488079071045, + "logps/chosen": -0.30542343854904175, + "logps/rejected": -0.3070066273212433, + "loss": 0.06507541984319687, + "loss/core": 0.06507541984319687, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.744479775428772, + "rewards/margins": 0.5683607459068298, + "rewards/rejected": 1.176119089126587, + "step": 70 + }, + { + "epoch": 0.07167174341515857, + "grad_norm": 27.5, + "ht_mnpo/logit": 0.09052614867687225, + "ht_mnpo/residual": 0.08302616328001022, + "ht_mnpo/residual_abs": 0.13352297246456146, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.1111111111111107e-07, + "logits/chosen": -2.002654552459717, + "logits/rejected": -1.8294117450714111, + "logps/chosen": -0.3003098964691162, + "logps/rejected": -0.3102906346321106, + "loss": 0.1903291791677475, + "loss/core": 0.1903291791677475, + "rewards/accuracies": 0.75, + "rewards/chosen": 1.9985805749893188, + "rewards/margins": 0.9052616357803345, + "rewards/rejected": 1.0933191776275635, + "step": 75 + }, + { + "epoch": 0.07644985964283581, + "grad_norm": 165.0, + "ht_mnpo/logit": 0.0681958794593811, + "ht_mnpo/residual": 0.060695894062519073, + "ht_mnpo/residual_abs": 0.19254747033119202, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.3888888888888884e-07, + "logits/chosen": -2.140778064727783, + "logits/rejected": -1.9811999797821045, + "logps/chosen": -0.27189767360687256, + "logps/rejected": -0.2811708450317383, + "loss": 0.4638647437095642, + "loss/core": 0.4638647437095642, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.553395986557007, + "rewards/margins": 0.681958794593811, + "rewards/rejected": 1.871437430381775, + "step": 80 + }, + { + "epoch": 0.08122797587051304, + "grad_norm": 5.09375, + "ht_mnpo/logit": 0.041115902364254, + "ht_mnpo/residual": 0.033615902066230774, + "ht_mnpo/residual_abs": 0.096328504383564, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.6666666666666666e-07, + "logits/chosen": -2.344470977783203, + "logits/rejected": -2.041018009185791, + "logps/chosen": -0.2965394854545593, + "logps/rejected": -0.30132466554641724, + "loss": 0.10672229528427124, + "loss/core": 0.10672229528427124, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 1.5701828002929688, + "rewards/margins": 0.4111589789390564, + "rewards/rejected": 1.159023642539978, + "step": 85 + }, + { + "epoch": 0.08600609209819028, + "grad_norm": 95.0, + "ht_mnpo/logit": 0.013142749667167664, + "ht_mnpo/residual": 0.0056427521631121635, + "ht_mnpo/residual_abs": 0.12206294387578964, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.944444444444445e-07, + "logits/chosen": -2.291677951812744, + "logits/rejected": -2.0287234783172607, + "logps/chosen": -0.25615039467811584, + "logps/rejected": -0.27058348059654236, + "loss": 0.11879338324069977, + "loss/core": 0.11879338324069977, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.6907819509506226, + "rewards/margins": 0.1314275860786438, + "rewards/rejected": 1.559354543685913, + "step": 90 + }, + { + "epoch": 0.09078420832586753, + "grad_norm": 20.75, + "ht_mnpo/logit": -0.08061865717172623, + "ht_mnpo/residual": -0.08811865001916885, + "ht_mnpo/residual_abs": 0.2554932236671448, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.999699149323369e-07, + "logits/chosen": -2.2245447635650635, + "logits/rejected": -1.850603699684143, + "logps/chosen": -0.28188449144363403, + "logps/rejected": -0.299966961145401, + "loss": 0.804580807685852, + "loss/core": 0.804580807685852, + "rewards/accuracies": 0.75, + "rewards/chosen": 1.833380103111267, + "rewards/margins": -0.8061864972114563, + "rewards/rejected": 2.63956618309021, + "step": 95 + }, + { + "epoch": 0.09556232455354477, + "grad_norm": 40.25, + "ht_mnpo/logit": 0.028320733457803726, + "ht_mnpo/residual": 0.02082073874771595, + "ht_mnpo/residual_abs": 0.06879810988903046, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.998477067547739e-07, + "logits/chosen": -2.1782548427581787, + "logits/rejected": -1.9620933532714844, + "logps/chosen": -0.27486157417297363, + "logps/rejected": -0.2764720916748047, + "loss": 0.039663899689912796, + "loss/core": 0.039663899689912796, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.7030102014541626, + "rewards/margins": 0.28320735692977905, + "rewards/rejected": 1.4198030233383179, + "step": 100 + }, + { + "epoch": 0.100340440781222, + "grad_norm": 4.1875, + "ht_mnpo/logit": 0.04795417934656143, + "ht_mnpo/residual": 0.040454182773828506, + "ht_mnpo/residual_abs": 0.13497649133205414, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.996315410727229e-07, + "logits/chosen": -2.1107373237609863, + "logits/rejected": -1.921032190322876, + "logps/chosen": -0.29457932710647583, + "logps/rejected": -0.2966824173927307, + "loss": 0.13080136477947235, + "loss/core": 0.13080136477947235, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.2252862453460693, + "rewards/margins": 0.4795418381690979, + "rewards/rejected": 1.7457443475723267, + "step": 105 + }, + { + "epoch": 0.10511855700889924, + "grad_norm": 12.375, + "ht_mnpo/logit": 0.17580091953277588, + "ht_mnpo/residual": 0.16830092668533325, + "ht_mnpo/residual_abs": 0.1905735582113266, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.993214991772562e-07, + "logits/chosen": -2.214897871017456, + "logits/rejected": -1.9663807153701782, + "logps/chosen": -0.2909243702888489, + "logps/rejected": -0.29862135648727417, + "loss": 0.4717007577419281, + "loss/core": 0.4717007577419281, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.3238365650177, + "rewards/margins": 1.7580095529556274, + "rewards/rejected": 1.5658271312713623, + "step": 110 + }, + { + "epoch": 0.10989667323657648, + "grad_norm": 0.234375, + "ht_mnpo/logit": -0.0033284418750554323, + "ht_mnpo/residual": -0.010828444734215736, + "ht_mnpo/residual_abs": 0.11969165503978729, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.989176976624511e-07, + "logits/chosen": -2.2799150943756104, + "logits/rejected": -2.022031307220459, + "logps/chosen": -0.31012484431266785, + "logps/rejected": -0.300957590341568, + "loss": 0.16961291432380676, + "loss/core": 0.16961291432380676, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 1.8041832447052002, + "rewards/margins": -0.03328438848257065, + "rewards/rejected": 1.8374675512313843, + "step": 115 + }, + { + "epoch": 0.11467478946425372, + "grad_norm": 7.875, + "ht_mnpo/logit": 0.037880975753068924, + "ht_mnpo/residual": 0.0303809754550457, + "ht_mnpo/residual_abs": 0.18625445663928986, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.984202883815428e-07, + "logits/chosen": -2.1674695014953613, + "logits/rejected": -1.9356225728988647, + "logps/chosen": -0.2807902693748474, + "logps/rejected": -0.2767426073551178, + "loss": 0.4123569130897522, + "loss/core": 0.4123569130897522, + "rewards/accuracies": 0.75, + "rewards/chosen": 2.6680102348327637, + "rewards/margins": 0.3788098692893982, + "rewards/rejected": 2.2892005443573, + "step": 120 + }, + { + "epoch": 0.11945290569193096, + "grad_norm": 0.458984375, + "ht_mnpo/logit": 0.02273702248930931, + "ht_mnpo/residual": 0.015237025916576385, + "ht_mnpo/residual_abs": 0.06529071182012558, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.978294583898195e-07, + "logits/chosen": -2.3368637561798096, + "logits/rejected": -2.065802574157715, + "logps/chosen": -0.27813562750816345, + "logps/rejected": -0.30318138003349304, + "loss": 0.018811456859111786, + "loss/core": 0.018811456859111786, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.5549370050430298, + "rewards/margins": 0.2273702174425125, + "rewards/rejected": 1.3275667428970337, + "step": 125 + }, + { + "epoch": 0.12423102191960819, + "grad_norm": 42.5, + "ht_mnpo/logit": 0.16898369789123535, + "ht_mnpo/residual": 0.16148369014263153, + "ht_mnpo/residual_abs": 0.18639250099658966, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.971454298742779e-07, + "logits/chosen": -2.2218174934387207, + "logits/rejected": -2.0029990673065186, + "logps/chosen": -0.2937593460083008, + "logps/rejected": -0.2997642159461975, + "loss": 0.4487941861152649, + "loss/core": 0.4487941861152649, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.533940315246582, + "rewards/margins": 1.689836859703064, + "rewards/rejected": 0.8441034555435181, + "step": 130 + }, + { + "epoch": 0.12900913814728543, + "grad_norm": 31.75, + "ht_mnpo/logit": 0.07704837620258331, + "ht_mnpo/residual": 0.06954839080572128, + "ht_mnpo/residual_abs": 0.08229651302099228, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.963684600700678e-07, + "logits/chosen": -2.335357189178467, + "logits/rejected": -2.066044807434082, + "logps/chosen": -0.269487202167511, + "logps/rejected": -0.273403525352478, + "loss": 0.03789084404706955, + "loss/core": 0.03789084404706955, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.7870798110961914, + "rewards/margins": 0.7704838514328003, + "rewards/rejected": 1.016595721244812, + "step": 135 + }, + { + "epoch": 0.13378725437496267, + "grad_norm": 3.734375, + "ht_mnpo/logit": 0.013967243023216724, + "ht_mnpo/residual": 0.006467245519161224, + "ht_mnpo/residual_abs": 0.06466107070446014, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.954988411637571e-07, + "logits/chosen": -2.3525476455688477, + "logits/rejected": -2.1170451641082764, + "logps/chosen": -0.2825363278388977, + "logps/rejected": -0.2803972661495209, + "loss": 0.02001281827688217, + "loss/core": 0.02001281827688217, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 1.4781088829040527, + "rewards/margins": 0.1396724134683609, + "rewards/rejected": 1.338436484336853, + "step": 140 + }, + { + "epoch": 0.1385653706026399, + "grad_norm": 0.216796875, + "ht_mnpo/logit": -0.0018466196488589048, + "ht_mnpo/residual": -0.009346622973680496, + "ht_mnpo/residual_abs": 0.09110254794359207, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.945369001834514e-07, + "logits/chosen": -2.3927693367004395, + "logits/rejected": -2.182438611984253, + "logps/chosen": -0.2859313488006592, + "logps/rejected": -0.2825351059436798, + "loss": 0.17166972160339355, + "loss/core": 0.17166972160339355, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.1670573949813843, + "rewards/margins": -0.01846626028418541, + "rewards/rejected": 1.1855233907699585, + "step": 145 + }, + { + "epoch": 0.14334348683031714, + "grad_norm": 11.375, + "ht_mnpo/logit": 0.03970382362604141, + "ht_mnpo/residual": 0.032203830778598785, + "ht_mnpo/residual_abs": 0.2119670808315277, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.93482998875813e-07, + "logits/chosen": -2.184814929962158, + "logits/rejected": -1.9867866039276123, + "logps/chosen": -0.27966034412384033, + "logps/rejected": -0.3121265172958374, + "loss": 0.901578426361084, + "loss/core": 0.901578426361084, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.2797250747680664, + "rewards/margins": 0.3970384895801544, + "rewards/rejected": 1.8826866149902344, + "step": 150 + }, + { + "epoch": 0.14812160305799438, + "grad_norm": 888.0, + "ht_mnpo/logit": 0.037043869495391846, + "ht_mnpo/residual": 0.029543865472078323, + "ht_mnpo/residual_abs": 0.2591553032398224, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.923375335700223e-07, + "logits/chosen": -2.214869737625122, + "logits/rejected": -1.9647678136825562, + "logps/chosen": -0.29476529359817505, + "logps/rejected": -0.33797982335090637, + "loss": 0.9004504084587097, + "loss/core": 0.9004504084587097, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 2.5209193229675293, + "rewards/margins": 0.37043848633766174, + "rewards/rejected": 2.1504807472229004, + "step": 155 + }, + { + "epoch": 0.15289971928567161, + "grad_norm": 896.0, + "ht_mnpo/logit": 0.10632804781198502, + "ht_mnpo/residual": 0.09882805496454239, + "ht_mnpo/residual_abs": 0.17886283993721008, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.911009350287347e-07, + "logits/chosen": -2.0985219478607178, + "logits/rejected": -1.8167247772216797, + "logps/chosen": -0.29428020119667053, + "logps/rejected": -0.30171844363212585, + "loss": 0.6274815797805786, + "loss/core": 0.6274815797805786, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 2.4871866703033447, + "rewards/margins": 1.0632803440093994, + "rewards/rejected": 1.4239060878753662, + "step": 160 + }, + { + "epoch": 0.15767783551334885, + "grad_norm": 2.296875, + "ht_mnpo/logit": 0.07954417169094086, + "ht_mnpo/residual": 0.07204417884349823, + "ht_mnpo/residual_abs": 0.07974160462617874, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.897736682860885e-07, + "logits/chosen": -2.258405923843384, + "logits/rejected": -1.9701240062713623, + "logps/chosen": -0.2872469127178192, + "logps/rejected": -0.2988366186618805, + "loss": 0.11342660337686539, + "loss/core": 0.11342660337686539, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.7711330652236938, + "rewards/margins": 0.7954418063163757, + "rewards/rejected": 0.9756911396980286, + "step": 165 + }, + { + "epoch": 0.1624559517410261, + "grad_norm": 2.0625, + "ht_mnpo/logit": 0.14767876267433167, + "ht_mnpo/residual": 0.14017875492572784, + "ht_mnpo/residual_abs": 0.20500735938549042, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.883562324728241e-07, + "logits/chosen": -2.192570447921753, + "logits/rejected": -2.0448708534240723, + "logps/chosen": -0.3334094285964966, + "logps/rejected": -0.3062793016433716, + "loss": 0.7441643476486206, + "loss/core": 0.7441643476486206, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.821702241897583, + "rewards/margins": 1.4767874479293823, + "rewards/rejected": 1.3449146747589111, + "step": 170 + }, + { + "epoch": 0.16723406796870333, + "grad_norm": 0.494140625, + "ht_mnpo/logit": 0.017073174938559532, + "ht_mnpo/residual": 0.009573178365826607, + "ht_mnpo/residual_abs": 0.11641023308038712, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.868491606285823e-07, + "logits/chosen": -2.202766180038452, + "logits/rejected": -2.0135915279388428, + "logps/chosen": -0.27780964970588684, + "logps/rejected": -0.2846834063529968, + "loss": 0.12164388597011566, + "loss/core": 0.12164388597011566, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.587665319442749, + "rewards/margins": 0.17073175311088562, + "rewards/rejected": 1.4169334173202515, + "step": 175 + }, + { + "epoch": 0.17201218419638056, + "grad_norm": 288.0, + "ht_mnpo/logit": 0.1441071778535843, + "ht_mnpo/residual": 0.13660718500614166, + "ht_mnpo/residual_abs": 0.15255433320999146, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.852530195014489e-07, + "logits/chosen": -2.3559622764587402, + "logits/rejected": -2.1174533367156982, + "logps/chosen": -0.2628288269042969, + "logps/rejected": -0.2772200405597687, + "loss": 0.320454478263855, + "loss/core": 0.320454478263855, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.7258729934692383, + "rewards/margins": 1.4410715103149414, + "rewards/rejected": 1.2848012447357178, + "step": 180 + }, + { + "epoch": 0.17679030042405783, + "grad_norm": 0.859375, + "ht_mnpo/logit": -0.04746574908494949, + "ht_mnpo/residual": -0.05496574565768242, + "ht_mnpo/residual_abs": 0.1622922122478485, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.835684093348244e-07, + "logits/chosen": -2.2382771968841553, + "logits/rejected": -2.074378728866577, + "logps/chosen": -0.3004586696624756, + "logps/rejected": -0.2946397364139557, + "loss": 0.5814380049705505, + "loss/core": 0.5814380049705505, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.3076417446136475, + "rewards/margins": -0.4746573865413666, + "rewards/rejected": 1.782299280166626, + "step": 185 + }, + { + "epoch": 0.18156841665173507, + "grad_norm": 1008.0, + "ht_mnpo/logit": 0.15239554643630981, + "ht_mnpo/residual": 0.14489556849002838, + "ht_mnpo/residual_abs": 0.153962180018425, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.817959636416969e-07, + "logits/chosen": -2.1816508769989014, + "logits/rejected": -2.0114400386810303, + "logps/chosen": -0.32795587182044983, + "logps/rejected": -0.29940658807754517, + "loss": 0.4533259868621826, + "loss/core": 0.4533259868621826, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.608320713043213, + "rewards/margins": 1.5239555835723877, + "rewards/rejected": 1.0843651294708252, + "step": 190 + }, + { + "epoch": 0.1863465328794123, + "grad_norm": 11.8125, + "ht_mnpo/logit": 0.08571872860193253, + "ht_mnpo/residual": 0.0782187357544899, + "ht_mnpo/residual_abs": 0.17521581053733826, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.799363489664039e-07, + "logits/chosen": -2.221156597137451, + "logits/rejected": -1.9798905849456787, + "logps/chosen": -0.3095605969429016, + "logps/rejected": -0.30117204785346985, + "loss": 0.31682509183883667, + "loss/core": 0.31682509183883667, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.334531545639038, + "rewards/margins": 0.8571873903274536, + "rewards/rejected": 1.477344274520874, + "step": 195 + }, + { + "epoch": 0.19112464910708954, + "grad_norm": 60.0, + "ht_mnpo/logit": 0.028831344097852707, + "ht_mnpo/residual": 0.021331345662474632, + "ht_mnpo/residual_abs": 0.08674871176481247, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.779902646339721e-07, + "logits/chosen": -2.182504892349243, + "logits/rejected": -1.9949296712875366, + "logps/chosen": -0.31937891244888306, + "logps/rejected": -0.3028191924095154, + "loss": 0.05946077033877373, + "loss/core": 0.05946077033877373, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.7149394750595093, + "rewards/margins": 0.2883134186267853, + "rewards/rejected": 1.4266260862350464, + "step": 200 + }, + { + "epoch": 0.19590276533476678, + "grad_norm": 17.0, + "ht_mnpo/logit": 0.08596853911876678, + "ht_mnpo/residual": 0.07846853882074356, + "ht_mnpo/residual_abs": 0.09083747863769531, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.759584424871301e-07, + "logits/chosen": -2.3666977882385254, + "logits/rejected": -2.1427879333496094, + "logps/chosen": -0.2890446186065674, + "logps/rejected": -0.29884833097457886, + "loss": 0.06484078615903854, + "loss/core": 0.06484078615903854, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.7943065166473389, + "rewards/margins": 0.8596854209899902, + "rewards/rejected": 0.9346210360527039, + "step": 205 + }, + { + "epoch": 0.200680881562444, + "grad_norm": 4.96875, + "ht_mnpo/logit": 0.04586448892951012, + "ht_mnpo/residual": 0.03836449235677719, + "ht_mnpo/residual_abs": 0.04392882436513901, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.738416466110917e-07, + "logits/chosen": -2.3530592918395996, + "logits/rejected": -2.1788575649261475, + "logps/chosen": -0.3264649212360382, + "logps/rejected": -0.32113516330718994, + "loss": 0.01702629216015339, + "loss/core": 0.01702629216015339, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 0.848310112953186, + "rewards/margins": 0.45864492654800415, + "rewards/rejected": 0.38966527581214905, + "step": 210 + }, + { + "epoch": 0.20545899779012125, + "grad_norm": 77.0, + "ht_mnpo/logit": 0.10470955073833466, + "ht_mnpo/residual": 0.09720954298973083, + "ht_mnpo/residual_abs": 0.1478584110736847, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.716406730462153e-07, + "logits/chosen": -2.1095123291015625, + "logits/rejected": -1.9788084030151367, + "logps/chosen": -0.2908587157726288, + "logps/rejected": -0.30672210454940796, + "loss": 0.18497677147388458, + "loss/core": 0.18497677147388458, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.263723373413086, + "rewards/margins": 1.047095537185669, + "rewards/rejected": 1.216628074645996, + "step": 215 + }, + { + "epoch": 0.2102371140177985, + "grad_norm": 236.0, + "ht_mnpo/logit": 0.021329589188098907, + "ht_mnpo/residual": 0.013829593546688557, + "ht_mnpo/residual_abs": 0.11300545930862427, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.693563494886454e-07, + "logits/chosen": -2.06388521194458, + "logits/rejected": -1.8469206094741821, + "logps/chosen": -0.28565680980682373, + "logps/rejected": -0.29494622349739075, + "loss": 0.1155991330742836, + "loss/core": 0.1155991330742836, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.018338203430176, + "rewards/margins": 0.21329593658447266, + "rewards/rejected": 1.8050426244735718, + "step": 220 + }, + { + "epoch": 0.21501523024547572, + "grad_norm": 10.0, + "ht_mnpo/logit": 0.1730707436800003, + "ht_mnpo/residual": 0.16557076573371887, + "ht_mnpo/residual_abs": 0.19626513123512268, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.6698953497905016e-07, + "logits/chosen": -2.2850561141967773, + "logits/rejected": -2.097651720046997, + "logps/chosen": -0.30766889452934265, + "logps/rejected": -0.30594539642333984, + "loss": 0.4711076617240906, + "loss/core": 0.4711076617240906, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 3.2686572074890137, + "rewards/margins": 1.7307075262069702, + "rewards/rejected": 1.537950038909912, + "step": 225 + }, + { + "epoch": 0.21979334647315296, + "grad_norm": 9.0, + "ht_mnpo/logit": 0.20456573367118835, + "ht_mnpo/residual": 0.19706572592258453, + "ht_mnpo/residual_abs": 0.23375244438648224, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.645411195795709e-07, + "logits/chosen": -2.202606439590454, + "logits/rejected": -1.9548810720443726, + "logps/chosen": -0.30069398880004883, + "logps/rejected": -0.3219486176967621, + "loss": 0.7216225862503052, + "loss/core": 0.7216225862503052, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.3075389862060547, + "rewards/margins": 2.0456573963165283, + "rewards/rejected": 1.2618815898895264, + "step": 230 + }, + { + "epoch": 0.2245714627008302, + "grad_norm": 206.0, + "ht_mnpo/logit": 0.031354598701000214, + "ht_mnpo/residual": 0.023854583501815796, + "ht_mnpo/residual_abs": 0.18463179469108582, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.6201202403910643e-07, + "logits/chosen": -2.1225569248199463, + "logits/rejected": -1.8874324560165405, + "logps/chosen": -0.2835076153278351, + "logps/rejected": -0.3018529713153839, + "loss": 0.449360191822052, + "loss/core": 0.449360191822052, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.9686769247055054, + "rewards/margins": 0.31354600191116333, + "rewards/rejected": 1.6551311016082764, + "step": 235 + }, + { + "epoch": 0.22934957892850744, + "grad_norm": 20.0, + "ht_mnpo/logit": 0.031544990837574005, + "ht_mnpo/residual": 0.024044986814260483, + "ht_mnpo/residual_abs": 0.1752614676952362, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.594031994470573e-07, + "logits/chosen": -2.209033966064453, + "logits/rejected": -1.8773391246795654, + "logps/chosen": -0.27358725666999817, + "logps/rejected": -0.28609001636505127, + "loss": 0.25187164545059204, + "loss/core": 0.25187164545059204, + "rewards/accuracies": 0.75, + "rewards/chosen": 1.8158626556396484, + "rewards/margins": 0.31544986367225647, + "rewards/rejected": 1.5004127025604248, + "step": 240 + }, + { + "epoch": 0.23412769515618467, + "grad_norm": 35.25, + "ht_mnpo/logit": 0.20634432137012482, + "ht_mnpo/residual": 0.1988442987203598, + "ht_mnpo/residual_abs": 0.24610598385334015, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.567156268756593e-07, + "logits/chosen": -2.1333718299865723, + "logits/rejected": -1.875788688659668, + "logps/chosen": -0.29131993651390076, + "logps/rejected": -0.292595237493515, + "loss": 0.5379303693771362, + "loss/core": 0.5379303693771362, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 4.162245273590088, + "rewards/margins": 2.0634429454803467, + "rewards/rejected": 2.098802089691162, + "step": 245 + }, + { + "epoch": 0.2389058113838619, + "grad_norm": 0.8984375, + "ht_mnpo/logit": 0.12799809873104095, + "ht_mnpo/residual": 0.12049808353185654, + "ht_mnpo/residual_abs": 0.17880940437316895, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.5395031701104303e-07, + "logits/chosen": -2.1053507328033447, + "logits/rejected": -1.8193076848983765, + "logps/chosen": -0.32429033517837524, + "logps/rejected": -0.29927438497543335, + "loss": 0.6025797724723816, + "loss/core": 0.6025797724723816, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.8428993225097656, + "rewards/margins": 1.2799808979034424, + "rewards/rejected": 1.5629183053970337, + "step": 250 + }, + { + "epoch": 0.24368392761153915, + "grad_norm": 4.875, + "ht_mnpo/logit": 0.13739672303199768, + "ht_mnpo/residual": 0.12989673018455505, + "ht_mnpo/residual_abs": 0.14480608701705933, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.511083097731555e-07, + "logits/chosen": -2.117206573486328, + "logits/rejected": -1.9836727380752563, + "logps/chosen": -0.3060274124145508, + "logps/rejected": -0.30183497071266174, + "loss": 0.2743765711784363, + "loss/core": 0.2743765711784363, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.924288749694824, + "rewards/margins": 1.3739672899246216, + "rewards/rejected": 1.550321340560913, + "step": 255 + }, + { + "epoch": 0.24846204383921638, + "grad_norm": 10.75, + "ht_mnpo/logit": 0.06477851420640945, + "ht_mnpo/residual": 0.05727851390838623, + "ht_mnpo/residual_abs": 0.07653465867042542, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.481906739246894e-07, + "logits/chosen": -2.1527230739593506, + "logits/rejected": -1.9991321563720703, + "logps/chosen": -0.28355228900909424, + "logps/rejected": -0.2868475615978241, + "loss": 0.038742970675230026, + "loss/core": 0.038742970675230026, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.7354450225830078, + "rewards/margins": 0.6477851867675781, + "rewards/rejected": 1.0876598358154297, + "step": 260 + }, + { + "epoch": 0.25324016006689365, + "grad_norm": 60.5, + "ht_mnpo/logit": 0.0033052079379558563, + "ht_mnpo/residual": -0.004194788634777069, + "ht_mnpo/residual_abs": 0.09127648174762726, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.451985066691648e-07, + "logits/chosen": -2.1906323432922363, + "logits/rejected": -1.9368772506713867, + "logps/chosen": -0.29127225279808044, + "logps/rejected": -0.28326812386512756, + "loss": 0.08905141055583954, + "loss/core": 0.08905141055583954, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 1.4620263576507568, + "rewards/margins": 0.03305206447839737, + "rewards/rejected": 1.4289741516113281, + "step": 265 + }, + { + "epoch": 0.25801827629457086, + "grad_norm": 2.1875, + "ht_mnpo/logit": 0.043637726455926895, + "ht_mnpo/residual": 0.03613772615790367, + "ht_mnpo/residual_abs": 0.04942305386066437, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.421329332383158e-07, + "logits/chosen": -2.236889123916626, + "logits/rejected": -2.070533514022827, + "logps/chosen": -0.30425819754600525, + "logps/rejected": -0.29968196153640747, + "loss": 0.012058206833899021, + "loss/core": 0.012058206833899021, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.2041269540786743, + "rewards/margins": 0.43637722730636597, + "rewards/rejected": 0.7677494883537292, + "step": 270 + }, + { + "epoch": 0.2627963925222481, + "grad_norm": 1160.0, + "ht_mnpo/logit": 0.1649816781282425, + "ht_mnpo/residual": 0.15748167037963867, + "ht_mnpo/residual_abs": 0.21221141517162323, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.3899510646893696e-07, + "logits/chosen": -2.2023870944976807, + "logits/rejected": -2.0466129779815674, + "logps/chosen": -0.28603219985961914, + "logps/rejected": -0.27672260999679565, + "loss": 0.9627658724784851, + "loss/core": 0.9627658724784851, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.6714327335357666, + "rewards/margins": 1.6498167514801025, + "rewards/rejected": 1.021615982055664, + "step": 275 + }, + { + "epoch": 0.26757450874992533, + "grad_norm": 8.1875, + "ht_mnpo/logit": 0.008860750123858452, + "ht_mnpo/residual": 0.0013607516884803772, + "ht_mnpo/residual_abs": 0.15622496604919434, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.357862063693485e-07, + "logits/chosen": -2.1357626914978027, + "logits/rejected": -1.8891808986663818, + "logps/chosen": -0.2642098069190979, + "logps/rejected": -0.28465694189071655, + "loss": 0.242923766374588, + "loss/core": 0.242923766374588, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.0400595664978027, + "rewards/margins": 0.08860747516155243, + "rewards/rejected": 1.9514522552490234, + "step": 280 + }, + { + "epoch": 0.2723526249776026, + "grad_norm": 5.90625, + "ht_mnpo/logit": 0.11119908094406128, + "ht_mnpo/residual": 0.10369908809661865, + "ht_mnpo/residual_abs": 0.15644970536231995, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.3250743967564364e-07, + "logits/chosen": -1.981865644454956, + "logits/rejected": -1.7845958471298218, + "logps/chosen": -0.28549736738204956, + "logps/rejected": -0.27184969186782837, + "loss": 0.2947588562965393, + "loss/core": 0.2947588562965393, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.6670725345611572, + "rewards/margins": 1.1119908094406128, + "rewards/rejected": 1.555082082748413, + "step": 285 + }, + { + "epoch": 0.2771307412052798, + "grad_norm": 1.5546875, + "ht_mnpo/logit": 0.03820858150720596, + "ht_mnpo/residual": 0.030708584934473038, + "ht_mnpo/residual_abs": 0.058207251131534576, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.29160039397884e-07, + "logits/chosen": -2.2808995246887207, + "logits/rejected": -2.0846009254455566, + "logps/chosen": -0.2757318615913391, + "logps/rejected": -0.2664252817630768, + "loss": 0.011909561231732368, + "loss/core": 0.011909561231732368, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.4301050901412964, + "rewards/margins": 0.3820858299732208, + "rewards/rejected": 1.0480191707611084, + "step": 290 + }, + { + "epoch": 0.28190885743295707, + "grad_norm": 744.0, + "ht_mnpo/logit": 0.15831990540027618, + "ht_mnpo/residual": 0.15081989765167236, + "ht_mnpo/residual_abs": 0.35883960127830505, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.2574526435641546e-07, + "logits/chosen": -2.3191518783569336, + "logits/rejected": -2.0893778800964355, + "logps/chosen": -0.31087130308151245, + "logps/rejected": -0.32478809356689453, + "loss": 1.9273204803466797, + "loss/core": 1.9273204803466797, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 3.648682117462158, + "rewards/margins": 1.5831987857818604, + "rewards/rejected": 2.0654828548431396, + "step": 295 + }, + { + "epoch": 0.2866869736606343, + "grad_norm": 4.28125, + "ht_mnpo/logit": 0.0808088406920433, + "ht_mnpo/residual": 0.07330884784460068, + "ht_mnpo/residual_abs": 0.10061073303222656, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.22264398708477e-07, + "logits/chosen": -1.9314266443252563, + "logits/rejected": -1.6769113540649414, + "logps/chosen": -0.3090340495109558, + "logps/rejected": -0.30936768651008606, + "loss": 0.0936809629201889, + "loss/core": 0.0936809629201889, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.8696715831756592, + "rewards/margins": 0.808088481426239, + "rewards/rejected": 1.0615832805633545, + "step": 300 + }, + { + "epoch": 0.29146508988831155, + "grad_norm": 6.5, + "ht_mnpo/logit": 0.05852525308728218, + "ht_mnpo/residual": 0.05102524906396866, + "ht_mnpo/residual_abs": 0.139701247215271, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.187187514652819e-07, + "logits/chosen": -2.129746675491333, + "logits/rejected": -1.9119545221328735, + "logps/chosen": -0.28492045402526855, + "logps/rejected": -0.29339781403541565, + "loss": 0.16279859840869904, + "loss/core": 0.16279859840869904, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.5068206787109375, + "rewards/margins": 0.5852525234222412, + "rewards/rejected": 1.9215682744979858, + "step": 305 + }, + { + "epoch": 0.29624320611598876, + "grad_norm": 14.5, + "ht_mnpo/logit": 0.06949242204427719, + "ht_mnpo/residual": 0.06199241429567337, + "ht_mnpo/residual_abs": 0.0795050710439682, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.151096559997519e-07, + "logits/chosen": -2.2989072799682617, + "logits/rejected": -2.007084369659424, + "logps/chosen": -0.28377217054367065, + "logps/rejected": -0.2830241024494171, + "loss": 0.0549379363656044, + "loss/core": 0.0549379363656044, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.7683111429214478, + "rewards/margins": 0.6949242353439331, + "rewards/rejected": 1.073386788368225, + "step": 310 + }, + { + "epoch": 0.301021322343666, + "grad_norm": 6.4375, + "ht_mnpo/logit": 0.038666121661663055, + "ht_mnpo/residual": 0.03116612136363983, + "ht_mnpo/residual_abs": 0.04453511908650398, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.114384695450905e-07, + "logits/chosen": -2.3783583641052246, + "logits/rejected": -2.1003165245056152, + "logps/chosen": -0.2959398627281189, + "logps/rejected": -0.3017369508743286, + "loss": 0.012863054871559143, + "loss/core": 0.012863054871559143, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 0.9655852317810059, + "rewards/margins": 0.38666123151779175, + "rewards/rejected": 0.5789240002632141, + "step": 315 + }, + { + "epoch": 0.30579943857134323, + "grad_norm": 12.5625, + "ht_mnpo/logit": 0.009172516874969006, + "ht_mnpo/residual": 0.0016725212335586548, + "ht_mnpo/residual_abs": 0.1754622906446457, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.077065726843828e-07, + "logits/chosen": -2.2958977222442627, + "logits/rejected": -2.032662868499756, + "logps/chosen": -0.2879457473754883, + "logps/rejected": -0.30056875944137573, + "loss": 0.37386396527290344, + "loss/core": 0.37386396527290344, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 1.9279342889785767, + "rewards/margins": 0.09172508865594864, + "rewards/rejected": 1.8362090587615967, + "step": 320 + }, + { + "epoch": 0.3105775547990205, + "grad_norm": 11.5, + "ht_mnpo/logit": 0.21785497665405273, + "ht_mnpo/residual": 0.2103549689054489, + "ht_mnpo/residual_abs": 0.22608724236488342, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.039153688314145e-07, + "logits/chosen": -2.0493011474609375, + "logits/rejected": -1.8347457647323608, + "logps/chosen": -0.2787044942378998, + "logps/rejected": -0.27662545442581177, + "loss": 0.6209802031517029, + "loss/core": 0.6209802031517029, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.5399444103240967, + "rewards/margins": 2.1785497665405273, + "rewards/rejected": 1.3613946437835693, + "step": 325 + }, + { + "epoch": 0.3153556710266977, + "grad_norm": 19.125, + "ht_mnpo/logit": 0.14707127213478088, + "ht_mnpo/residual": 0.13957123458385468, + "ht_mnpo/residual_abs": 0.17394544184207916, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.0006628370290613e-07, + "logits/chosen": -2.2151148319244385, + "logits/rejected": -1.889974594116211, + "logps/chosen": -0.2892436385154724, + "logps/rejected": -0.2921144664287567, + "loss": 0.4615117907524109, + "loss/core": 0.4615117907524109, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.9435486793518066, + "rewards/margins": 1.470712423324585, + "rewards/rejected": 1.4728357791900635, + "step": 330 + }, + { + "epoch": 0.32013378725437497, + "grad_norm": 47.5, + "ht_mnpo/logit": 0.060556523501873016, + "ht_mnpo/residual": 0.05305652692914009, + "ht_mnpo/residual_abs": 0.1689012497663498, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.9616076478235826e-07, + "logits/chosen": -2.222839593887329, + "logits/rejected": -2.075805425643921, + "logps/chosen": -0.2583637237548828, + "logps/rejected": -0.2836921215057373, + "loss": 0.3947281241416931, + "loss/core": 0.3947281241416931, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.1558752059936523, + "rewards/margins": 0.6055654287338257, + "rewards/rejected": 1.550309658050537, + "step": 335 + }, + { + "epoch": 0.3249119034820522, + "grad_norm": 258.0, + "ht_mnpo/logit": 0.15883901715278625, + "ht_mnpo/residual": 0.15133902430534363, + "ht_mnpo/residual_abs": 0.16413579881191254, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.922002807757129e-07, + "logits/chosen": -2.2033660411834717, + "logits/rejected": -1.9985862970352173, + "logps/chosen": -0.26738107204437256, + "logps/rejected": -0.2792484164237976, + "loss": 0.4059799313545227, + "loss/core": 0.4059799313545227, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.9665191173553467, + "rewards/margins": 1.5883904695510864, + "rewards/rejected": 1.3781286478042603, + "step": 340 + }, + { + "epoch": 0.32969001970972944, + "grad_norm": 0.6875, + "ht_mnpo/logit": 0.15273532271385193, + "ht_mnpo/residual": 0.1452353298664093, + "ht_mnpo/residual_abs": 0.17430062592029572, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.8818632105903315e-07, + "logits/chosen": -2.4266574382781982, + "logits/rejected": -2.118107318878174, + "logps/chosen": -0.27640026807785034, + "logps/rejected": -0.28678444027900696, + "loss": 0.63597571849823, + "loss/core": 0.63597571849823, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.5499401092529297, + "rewards/margins": 1.527353286743164, + "rewards/rejected": 1.0225868225097656, + "step": 345 + }, + { + "epoch": 0.33446813593740665, + "grad_norm": 7.84375, + "ht_mnpo/logit": 0.1263376772403717, + "ht_mnpo/residual": 0.11883767694234848, + "ht_mnpo/residual_abs": 0.14816422760486603, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.841203951184094e-07, + "logits/chosen": -2.320239543914795, + "logits/rejected": -2.0900626182556152, + "logps/chosen": -0.2879655361175537, + "logps/rejected": -0.30228275060653687, + "loss": 0.41849765181541443, + "loss/core": 0.41849765181541443, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.399965286254883, + "rewards/margins": 1.2633768320083618, + "rewards/rejected": 1.136588454246521, + "step": 350 + }, + { + "epoch": 0.3392462521650839, + "grad_norm": 48.5, + "ht_mnpo/logit": 0.172264963388443, + "ht_mnpo/residual": 0.16476497054100037, + "ht_mnpo/residual_abs": 0.2448660433292389, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.800040319823038e-07, + "logits/chosen": -2.225900650024414, + "logits/rejected": -1.9668943881988525, + "logps/chosen": -0.2590000033378601, + "logps/rejected": -0.2526404559612274, + "loss": 0.5311131477355957, + "loss/core": 0.5311131477355957, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.443131923675537, + "rewards/margins": 1.7226498126983643, + "rewards/rejected": 1.7204822301864624, + "step": 355 + }, + { + "epoch": 0.3440243683927611, + "grad_norm": 3.640625, + "ht_mnpo/logit": 0.1497168242931366, + "ht_mnpo/residual": 0.14221683144569397, + "ht_mnpo/residual_abs": 0.18900860846042633, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.75838779646545e-07, + "logits/chosen": -2.070448875427246, + "logits/rejected": -1.9167006015777588, + "logps/chosen": -0.2813272476196289, + "logps/rejected": -0.2787591516971588, + "loss": 0.5066571831703186, + "loss/core": 0.5066571831703186, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.164358377456665, + "rewards/margins": 1.4971683025360107, + "rewards/rejected": 1.6671901941299438, + "step": 360 + }, + { + "epoch": 0.3488024846204384, + "grad_norm": 4.40625, + "ht_mnpo/logit": 0.135546013712883, + "ht_mnpo/residual": 0.12804600596427917, + "ht_mnpo/residual_abs": 0.15786965191364288, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.7162620449218993e-07, + "logits/chosen": -2.3442587852478027, + "logits/rejected": -2.12084698677063, + "logps/chosen": -0.2686806619167328, + "logps/rejected": -0.27344799041748047, + "loss": 0.699191689491272, + "loss/core": 0.699191689491272, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.5767464637756348, + "rewards/margins": 1.3554600477218628, + "rewards/rejected": 1.2212862968444824, + "step": 365 + }, + { + "epoch": 0.35358060084811566, + "grad_norm": 1.859375, + "ht_mnpo/logit": 0.09349587559700012, + "ht_mnpo/residual": 0.0859958678483963, + "ht_mnpo/residual_abs": 0.10029371827840805, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.673678906964727e-07, + "logits/chosen": -2.2674036026000977, + "logits/rejected": -1.944549560546875, + "logps/chosen": -0.3023375868797302, + "logps/rejected": -0.3050137162208557, + "loss": 0.2242615520954132, + "loss/core": 0.2242615520954132, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": 1.805890440940857, + "rewards/margins": 0.9349587559700012, + "rewards/rejected": 0.8709318041801453, + "step": 370 + }, + { + "epoch": 0.35835871707579287, + "grad_norm": 8.8125, + "ht_mnpo/logit": 0.09880022704601288, + "ht_mnpo/residual": 0.09130023419857025, + "ht_mnpo/residual_abs": 0.12248305231332779, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.6306543963705936e-07, + "logits/chosen": -1.9330936670303345, + "logits/rejected": -1.8162342309951782, + "logps/chosen": -0.3033140003681183, + "logps/rejected": -0.27995291352272034, + "loss": 0.2245170623064041, + "loss/core": 0.2245170623064041, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 2.7184536457061768, + "rewards/margins": 0.9880023002624512, + "rewards/rejected": 1.730450987815857, + "step": 375 + }, + { + "epoch": 0.36313683330347013, + "grad_norm": 8.0625, + "ht_mnpo/logit": 0.13801223039627075, + "ht_mnpo/residual": 0.13051220774650574, + "ht_mnpo/residual_abs": 0.13972285389900208, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.5872046928983623e-07, + "logits/chosen": -2.269474506378174, + "logits/rejected": -1.9879837036132812, + "logps/chosen": -0.29586416482925415, + "logps/rejected": -0.303752064704895, + "loss": 0.2670932412147522, + "loss/core": 0.2670932412147522, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.13942289352417, + "rewards/margins": 1.3801223039627075, + "rewards/rejected": 0.7593008279800415, + "step": 380 + }, + { + "epoch": 0.36791494953114734, + "grad_norm": 8.3125, + "ht_mnpo/logit": 0.07426253706216812, + "ht_mnpo/residual": 0.0667625367641449, + "ht_mnpo/residual_abs": 0.08736227452754974, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.5433461362045447e-07, + "logits/chosen": -2.2373242378234863, + "logits/rejected": -2.0958285331726074, + "logps/chosen": -0.2665890157222748, + "logps/rejected": -0.2846522033214569, + "loss": 0.03473305329680443, + "loss/core": 0.03473305329680443, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.7353073358535767, + "rewards/margins": 0.74262535572052, + "rewards/rejected": 0.9926820993423462, + "step": 385 + }, + { + "epoch": 0.3726930657588246, + "grad_norm": 0.98828125, + "ht_mnpo/logit": 0.11850671470165253, + "ht_mnpo/residual": 0.1110067218542099, + "ht_mnpo/residual_abs": 0.1288645714521408, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.4990952196986305e-07, + "logits/chosen": -2.2801146507263184, + "logits/rejected": -2.045973300933838, + "logps/chosen": -0.29588863253593445, + "logps/rejected": -0.26954060792922974, + "loss": 0.3808225989341736, + "loss/core": 0.3808225989341736, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.7486746311187744, + "rewards/margins": 1.1850672960281372, + "rewards/rejected": 1.5636073350906372, + "step": 390 + }, + { + "epoch": 0.3774711819865018, + "grad_norm": 25.0, + "ht_mnpo/logit": 0.2065129578113556, + "ht_mnpo/residual": 0.19901293516159058, + "ht_mnpo/residual_abs": 0.24453966319561005, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.4544685843405875e-07, + "logits/chosen": -2.167894124984741, + "logits/rejected": -2.0230815410614014, + "logps/chosen": -0.2743033766746521, + "logps/rejected": -0.3099581003189087, + "loss": 0.7823916673660278, + "loss/core": 0.7823916673660278, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.2027816772460938, + "rewards/margins": 2.065129280090332, + "rewards/rejected": 1.1376521587371826, + "step": 395 + }, + { + "epoch": 0.3822492982141791, + "grad_norm": 1.9765625, + "ht_mnpo/logit": 0.04907558485865593, + "ht_mnpo/residual": 0.041575588285923004, + "ht_mnpo/residual_abs": 0.056028060615062714, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.4094830123828786e-07, + "logits/chosen": -2.0534815788269043, + "logits/rejected": -1.87503182888031, + "logps/chosen": -0.2717309594154358, + "logps/rejected": -0.2837705910205841, + "loss": 0.010935614816844463, + "loss/core": 0.010935614816844463, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.9626553058624268, + "rewards/margins": 0.4907558560371399, + "rewards/rejected": 1.4718996286392212, + "step": 400 + }, + { + "epoch": 0.3870274144418563, + "grad_norm": 12.5625, + "ht_mnpo/logit": 0.13887158036231995, + "ht_mnpo/residual": 0.13137157261371613, + "ht_mnpo/residual_abs": 0.22304904460906982, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.3641554210593414e-07, + "logits/chosen": -2.0426554679870605, + "logits/rejected": -1.777740478515625, + "logps/chosen": -0.2529454827308655, + "logps/rejected": -0.2843603491783142, + "loss": 0.6288084983825684, + "loss/core": 0.6288084983825684, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 3.3556621074676514, + "rewards/margins": 1.3887159824371338, + "rewards/rejected": 1.9669462442398071, + "step": 405 + }, + { + "epoch": 0.39180553066953355, + "grad_norm": 0.4140625, + "ht_mnpo/logit": 0.11563078314065933, + "ht_mnpo/residual": 0.1081307902932167, + "ht_mnpo/residual_abs": 0.13895943760871887, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.3185028562233107e-07, + "logits/chosen": -2.381803035736084, + "logits/rejected": -2.0271756649017334, + "logps/chosen": -0.2997882664203644, + "logps/rejected": -0.3019891381263733, + "loss": 0.2540455758571625, + "loss/core": 0.2540455758571625, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": 2.6567492485046387, + "rewards/margins": 1.1563078165054321, + "rewards/rejected": 1.5004416704177856, + "step": 410 + }, + { + "epoch": 0.39658364689721076, + "grad_norm": 8.3125, + "ht_mnpo/logit": 0.16058483719825745, + "ht_mnpo/residual": 0.15308482944965363, + "ht_mnpo/residual_abs": 0.17841237783432007, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.272542485937368e-07, + "logits/chosen": -2.159789800643921, + "logits/rejected": -1.9371411800384521, + "logps/chosen": -0.2774130403995514, + "logps/rejected": -0.2748417258262634, + "loss": 0.4761256277561188, + "loss/core": 0.4761256277561188, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.7579145431518555, + "rewards/margins": 1.6058483123779297, + "rewards/rejected": 1.1520664691925049, + "step": 415 + }, + { + "epoch": 0.401361763124888, + "grad_norm": 360.0, + "ht_mnpo/logit": 0.17204149067401886, + "ht_mnpo/residual": 0.16454148292541504, + "ht_mnpo/residual_abs": 0.25641435384750366, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.226291594017137e-07, + "logits/chosen": -1.9525874853134155, + "logits/rejected": -1.7593332529067993, + "logps/chosen": -0.2873152196407318, + "logps/rejected": -0.28734180331230164, + "loss": 0.5311289429664612, + "loss/core": 0.5311289429664612, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.404254913330078, + "rewards/margins": 1.7204147577285767, + "rewards/rejected": 1.6838403940200806, + "step": 420 + }, + { + "epoch": 0.40613987935256524, + "grad_norm": 1112.0, + "ht_mnpo/logit": 0.13960780203342438, + "ht_mnpo/residual": 0.13210779428482056, + "ht_mnpo/residual_abs": 0.2718185782432556, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.1797675735315454e-07, + "logits/chosen": -2.2387444972991943, + "logits/rejected": -2.000307083129883, + "logps/chosen": -0.2742324471473694, + "logps/rejected": -0.28232383728027344, + "loss": 0.9235121607780457, + "loss/core": 0.9235121607780457, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 3.1787946224212646, + "rewards/margins": 1.3960778713226318, + "rewards/rejected": 1.7827165126800537, + "step": 425 + }, + { + "epoch": 0.4109179955802425, + "grad_norm": 306.0, + "ht_mnpo/logit": 0.03273645043373108, + "ht_mnpo/residual": 0.025236451998353004, + "ht_mnpo/residual_abs": 0.08493396639823914, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.1329879202620047e-07, + "logits/chosen": -2.2073371410369873, + "logits/rejected": -1.9156125783920288, + "logps/chosen": -0.284179151058197, + "logps/rejected": -0.2822193503379822, + "loss": 0.08687691390514374, + "loss/core": 0.08687691390514374, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.3117029666900635, + "rewards/margins": 0.3273644745349884, + "rewards/rejected": 0.9843384027481079, + "step": 430 + }, + { + "epoch": 0.4156961118079197, + "grad_norm": 1040.0, + "ht_mnpo/logit": 0.147271066904068, + "ht_mnpo/residual": 0.13977108895778656, + "ht_mnpo/residual_abs": 0.2996138334274292, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.0859702261229613e-07, + "logits/chosen": -2.086392641067505, + "logits/rejected": -1.9442352056503296, + "logps/chosen": -0.30270153284072876, + "logps/rejected": -0.33257466554641724, + "loss": 1.1545547246932983, + "loss/core": 1.1545547246932983, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 3.7823562622070312, + "rewards/margins": 1.4727110862731934, + "rewards/rejected": 2.309644937515259, + "step": 435 + }, + { + "epoch": 0.420474228035597, + "grad_norm": 5.09375, + "ht_mnpo/logit": 0.15315040946006775, + "ht_mnpo/residual": 0.14565041661262512, + "ht_mnpo/residual_abs": 0.2222578078508377, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.0387321725463e-07, + "logits/chosen": -2.4342799186706543, + "logits/rejected": -2.189716100692749, + "logps/chosen": -0.27770087122917175, + "logps/rejected": -0.291808158159256, + "loss": 0.6261534690856934, + "loss/core": 0.6261534690856934, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.841123104095459, + "rewards/margins": 1.5315040349960327, + "rewards/rejected": 1.3096191883087158, + "step": 440 + }, + { + "epoch": 0.4252523442632742, + "grad_norm": 5.125, + "ht_mnpo/logit": 0.0066137416288256645, + "ht_mnpo/residual": -0.0008862599497660995, + "ht_mnpo/residual_abs": 0.1915988177061081, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.991291523832075e-07, + "logits/chosen": -2.1221556663513184, + "logits/rejected": -1.8717864751815796, + "logps/chosen": -0.30727943778038025, + "logps/rejected": -0.3041136562824249, + "loss": 0.40154901146888733, + "loss/core": 0.40154901146888733, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.6310601234436035, + "rewards/margins": 0.06613747030496597, + "rewards/rejected": 2.56492280960083, + "step": 445 + }, + { + "epoch": 0.43003046049095145, + "grad_norm": 4.5625, + "ht_mnpo/logit": 0.028625909239053726, + "ht_mnpo/residual": 0.021125907078385353, + "ht_mnpo/residual_abs": 0.06126154586672783, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.943666120468088e-07, + "logits/chosen": -2.1283726692199707, + "logits/rejected": -1.7972466945648193, + "logps/chosen": -0.2941470742225647, + "logps/rejected": -0.31552690267562866, + "loss": 0.015647191554307938, + "loss/core": 0.015647191554307938, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": 1.5162620544433594, + "rewards/margins": 0.28625908493995667, + "rewards/rejected": 1.2300031185150146, + "step": 450 + }, + { + "epoch": 0.43480857671862866, + "grad_norm": 7.875, + "ht_mnpo/logit": -0.03804566711187363, + "ht_mnpo/residual": -0.04554566740989685, + "ht_mnpo/residual_abs": 0.13553011417388916, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.8958738724208073e-07, + "logits/chosen": -2.3924567699432373, + "logits/rejected": -2.0088376998901367, + "logps/chosen": -0.3002735674381256, + "logps/rejected": -0.3013390600681305, + "loss": 0.30527040362358093, + "loss/core": 0.30527040362358093, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.0105655193328857, + "rewards/margins": -0.3804566264152527, + "rewards/rejected": 1.3910223245620728, + "step": 455 + }, + { + "epoch": 0.4395866929463059, + "grad_norm": 3.1875, + "ht_mnpo/logit": 0.020327895879745483, + "ht_mnpo/residual": 0.012827901169657707, + "ht_mnpo/residual_abs": 0.11514680087566376, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.8479327524001633e-07, + "logits/chosen": -2.346726894378662, + "logits/rejected": -1.9906597137451172, + "logps/chosen": -0.2989094853401184, + "logps/rejected": -0.3026769757270813, + "loss": 0.1966794729232788, + "loss/core": 0.1966794729232788, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.6205774545669556, + "rewards/margins": 0.20327889919281006, + "rewards/rejected": 1.417298674583435, + "step": 460 + }, + { + "epoch": 0.44436480917398313, + "grad_norm": 9.9375, + "ht_mnpo/logit": 0.01999581605195999, + "ht_mnpo/residual": 0.012495816685259342, + "ht_mnpo/residual_abs": 0.11453278362751007, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.7998607891007493e-07, + "logits/chosen": -1.9809582233428955, + "logits/rejected": -1.6697006225585938, + "logps/chosen": -0.2852214276790619, + "logps/rejected": -0.29375481605529785, + "loss": 0.09421398490667343, + "loss/core": 0.09421398490667343, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.281210422515869, + "rewards/margins": 0.19995816051959991, + "rewards/rejected": 2.081252336502075, + "step": 465 + }, + { + "epoch": 0.4491429254016604, + "grad_norm": 804.0, + "ht_mnpo/logit": 0.07451383024454117, + "ht_mnpo/residual": 0.06701384484767914, + "ht_mnpo/residual_abs": 0.19596199691295624, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.7516760604219616e-07, + "logits/chosen": -2.164752721786499, + "logits/rejected": -1.8490082025527954, + "logps/chosen": -0.25757962465286255, + "logps/rejected": -0.2674221098423004, + "loss": 0.694079577922821, + "loss/core": 0.694079577922821, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.4484660625457764, + "rewards/margins": 0.74513840675354, + "rewards/rejected": 1.7033277750015259, + "step": 470 + }, + { + "epoch": 0.4539210416293376, + "grad_norm": 21.5, + "ht_mnpo/logit": 0.05053982138633728, + "ht_mnpo/residual": 0.043039821088314056, + "ht_mnpo/residual_abs": 0.17135195434093475, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.703396686669646e-07, + "logits/chosen": -2.3393898010253906, + "logits/rejected": -1.9308264255523682, + "logps/chosen": -0.27896398305892944, + "logps/rejected": -0.2943103611469269, + "loss": 0.2023368626832962, + "loss/core": 0.2023368626832962, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.079939365386963, + "rewards/margins": 0.5053982734680176, + "rewards/rejected": 1.5745410919189453, + "step": 475 + }, + { + "epoch": 0.45869915785701487, + "grad_norm": 145.0, + "ht_mnpo/logit": 0.07996602356433868, + "ht_mnpo/residual": 0.07246602326631546, + "ht_mnpo/residual_abs": 0.15748628973960876, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.6550408237417884e-07, + "logits/chosen": -2.064171314239502, + "logits/rejected": -1.7457411289215088, + "logps/chosen": -0.28379932045936584, + "logps/rejected": -0.30818262696266174, + "loss": 0.26114827394485474, + "loss/core": 0.26114827394485474, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.5865588188171387, + "rewards/margins": 0.799660325050354, + "rewards/rejected": 1.7868983745574951, + "step": 480 + }, + { + "epoch": 0.46347727408469214, + "grad_norm": 2.625, + "ht_mnpo/logit": 0.062350332736968994, + "ht_mnpo/residual": 0.05485032871365547, + "ht_mnpo/residual_abs": 0.08066234737634659, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.6066266563008265e-07, + "logits/chosen": -2.3802075386047363, + "logits/rejected": -2.1280293464660645, + "logps/chosen": -0.2738611102104187, + "logps/rejected": -0.289838969707489, + "loss": 0.0550173744559288, + "loss/core": 0.0550173744559288, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.4340312480926514, + "rewards/margins": 0.6235032677650452, + "rewards/rejected": 0.8105279803276062, + "step": 485 + }, + { + "epoch": 0.46825539031236935, + "grad_norm": 41.0, + "ht_mnpo/logit": 0.07819999009370804, + "ht_mnpo/residual": 0.07069997489452362, + "ht_mnpo/residual_abs": 0.1625690758228302, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.5581723909351404e-07, + "logits/chosen": -2.249577522277832, + "logits/rejected": -2.0636165142059326, + "logps/chosen": -0.2568047046661377, + "logps/rejected": -0.3003430962562561, + "loss": 0.22979728877544403, + "loss/core": 0.22979728877544403, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.4754576683044434, + "rewards/margins": 0.7819998264312744, + "rewards/rejected": 1.693457841873169, + "step": 490 + }, + { + "epoch": 0.4730335065400466, + "grad_norm": 6.25, + "ht_mnpo/logit": 0.06184772402048111, + "ht_mnpo/residual": 0.05434773117303848, + "ht_mnpo/residual_abs": 0.08136261999607086, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.509696249312301e-07, + "logits/chosen": -2.36091947555542, + "logits/rejected": -2.0397167205810547, + "logps/chosen": -0.2872092127799988, + "logps/rejected": -0.284702330827713, + "loss": 0.059263549745082855, + "loss/core": 0.059263549745082855, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 1.4394937753677368, + "rewards/margins": 0.6184772253036499, + "rewards/rejected": 0.8210164904594421, + "step": 495 + }, + { + "epoch": 0.4778116227677238, + "grad_norm": 1.7734375, + "ht_mnpo/logit": 0.05665602162480354, + "ht_mnpo/residual": 0.04915601760149002, + "ht_mnpo/residual_abs": 0.1744033843278885, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.461216461326642e-07, + "logits/chosen": -2.30210280418396, + "logits/rejected": -2.0592434406280518, + "logps/chosen": -0.28479260206222534, + "logps/rejected": -0.295644611120224, + "loss": 0.4020271897315979, + "loss/core": 0.4020271897315979, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.8941959142684937, + "rewards/margins": 0.5665602684020996, + "rewards/rejected": 1.3276355266571045, + "step": 500 + }, + { + "epoch": 0.4825897389954011, + "grad_norm": 14.1875, + "ht_mnpo/logit": 0.043277788907289505, + "ht_mnpo/residual": 0.03577778860926628, + "ht_mnpo/residual_abs": 0.0675283819437027, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.412751258243748e-07, + "logits/chosen": -2.371837854385376, + "logits/rejected": -2.1066370010375977, + "logps/chosen": -0.260089635848999, + "logps/rejected": -0.28670915961265564, + "loss": 0.03348282352089882, + "loss/core": 0.03348282352089882, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.3851932287216187, + "rewards/margins": 0.43277788162231445, + "rewards/rejected": 0.9524153470993042, + "step": 505 + }, + { + "epoch": 0.4873678552230783, + "grad_norm": 572.0, + "ht_mnpo/logit": 0.0772649273276329, + "ht_mnpo/residual": 0.06976493448019028, + "ht_mnpo/residual_abs": 0.23304533958435059, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.3643188658444158e-07, + "logits/chosen": -2.180732011795044, + "logits/rejected": -1.9350513219833374, + "logps/chosen": -0.2900449335575104, + "logps/rejected": -0.3275887370109558, + "loss": 0.701738178730011, + "loss/core": 0.701738178730011, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.6179404258728027, + "rewards/margins": 0.7726494073867798, + "rewards/rejected": 1.8452911376953125, + "step": 510 + }, + { + "epoch": 0.49214597145075556, + "grad_norm": 2080.0, + "ht_mnpo/logit": 0.13127067685127258, + "ht_mnpo/residual": 0.12377066910266876, + "ht_mnpo/residual_abs": 0.1691356599330902, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.315937497570688e-07, + "logits/chosen": -2.2999091148376465, + "logits/rejected": -1.9634205102920532, + "logps/chosen": -0.29252487421035767, + "logps/rejected": -0.29366955161094666, + "loss": 0.72135990858078, + "loss/core": 0.72135990858078, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.4953925609588623, + "rewards/margins": 1.312706708908081, + "rewards/rejected": 1.1826857328414917, + "step": 515 + }, + { + "epoch": 0.49692408767843277, + "grad_norm": 2.390625, + "ht_mnpo/logit": 0.028129365295171738, + "ht_mnpo/residual": 0.02062937058508396, + "ht_mnpo/residual_abs": 0.12287096679210663, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.2676253476765194e-07, + "logits/chosen": -2.214996576309204, + "logits/rejected": -1.9160572290420532, + "logps/chosen": -0.2983536124229431, + "logps/rejected": -0.31177738308906555, + "loss": 0.20848357677459717, + "loss/core": 0.20848357677459717, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": 1.8202794790267944, + "rewards/margins": 0.28129369020462036, + "rewards/rejected": 1.5389859676361084, + "step": 520 + }, + { + "epoch": 0.50170220390611, + "grad_norm": 800.0, + "ht_mnpo/logit": 0.19240263104438782, + "ht_mnpo/residual": 0.184902623295784, + "ht_mnpo/residual_abs": 0.3580849766731262, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.2194005843856633e-07, + "logits/chosen": -2.0721850395202637, + "logits/rejected": -1.8336656093597412, + "logps/chosen": -0.27473676204681396, + "logps/rejected": -0.2788922190666199, + "loss": 1.030470848083496, + "loss/core": 1.030470848083496, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 4.851513862609863, + "rewards/margins": 1.9240261316299438, + "rewards/rejected": 2.927487850189209, + "step": 525 + }, + { + "epoch": 0.5064803201337873, + "grad_norm": 13.3125, + "ht_mnpo/logit": 0.06076349690556526, + "ht_mnpo/residual": 0.05326349288225174, + "ht_mnpo/residual_abs": 0.08253750950098038, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.1712813430593434e-07, + "logits/chosen": -2.1719086170196533, + "logits/rejected": -1.937659502029419, + "logps/chosen": -0.29302453994750977, + "logps/rejected": -0.3102428913116455, + "loss": 0.044716306030750275, + "loss/core": 0.044716306030750275, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.4354779720306396, + "rewards/margins": 0.6076349020004272, + "rewards/rejected": 1.8278430700302124, + "step": 530 + }, + { + "epoch": 0.5112584363614645, + "grad_norm": 167.0, + "ht_mnpo/logit": 0.06611252576112747, + "ht_mnpo/residual": 0.05861252546310425, + "ht_mnpo/residual_abs": 0.11790484189987183, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.123285719376292e-07, + "logits/chosen": -2.1390864849090576, + "logits/rejected": -1.9728997945785522, + "logps/chosen": -0.28939175605773926, + "logps/rejected": -0.29434826970100403, + "loss": 0.09479711949825287, + "loss/core": 0.09479711949825287, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.8622554540634155, + "rewards/margins": 0.6611253023147583, + "rewards/rejected": 1.2011301517486572, + "step": 535 + }, + { + "epoch": 0.5160365525891417, + "grad_norm": 9.875, + "ht_mnpo/logit": 0.10702864080667496, + "ht_mnpo/residual": 0.09952862560749054, + "ht_mnpo/residual_abs": 0.1450154036283493, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.0754317625276982e-07, + "logits/chosen": -2.327497959136963, + "logits/rejected": -2.0987296104431152, + "logps/chosen": -0.29696059226989746, + "logps/rejected": -0.3068595230579376, + "loss": 0.19229401648044586, + "loss/core": 0.19229401648044586, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 2.24061918258667, + "rewards/margins": 1.0702862739562988, + "rewards/rejected": 1.1703331470489502, + "step": 540 + }, + { + "epoch": 0.5208146688168189, + "grad_norm": 12.5, + "ht_mnpo/logit": 0.04363741725683212, + "ht_mnpo/residual": 0.0361374206840992, + "ht_mnpo/residual_abs": 0.07851780205965042, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.0277374684296498e-07, + "logits/chosen": -2.2913150787353516, + "logits/rejected": -2.051034450531006, + "logps/chosen": -0.3131157159805298, + "logps/rejected": -0.3089737296104431, + "loss": 0.04530810937285423, + "loss/core": 0.04530810937285423, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 1.512770414352417, + "rewards/margins": 0.4363742470741272, + "rewards/rejected": 1.0763962268829346, + "step": 545 + }, + { + "epoch": 0.5255927850444962, + "grad_norm": 9.0, + "ht_mnpo/logit": 0.06660556048154831, + "ht_mnpo/residual": 0.059105563908815384, + "ht_mnpo/residual_abs": 0.07340370863676071, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.980220772955602e-07, + "logits/chosen": -2.189758062362671, + "logits/rejected": -2.015587568283081, + "logps/chosen": -0.29240942001342773, + "logps/rejected": -0.31375378370285034, + "loss": 0.03384558483958244, + "loss/core": 0.03384558483958244, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.594940185546875, + "rewards/margins": 0.6660556197166443, + "rewards/rejected": 0.9288846850395203, + "step": 550 + }, + { + "epoch": 0.5303709012721735, + "grad_norm": 3.078125, + "ht_mnpo/logit": 0.06365452706813812, + "ht_mnpo/residual": 0.0561545304954052, + "ht_mnpo/residual_abs": 0.08707477897405624, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.932899545191433e-07, + "logits/chosen": -2.128129005432129, + "logits/rejected": -2.004274368286133, + "logps/chosen": -0.3144676983356476, + "logps/rejected": -0.32734498381614685, + "loss": 0.04949260503053665, + "loss/core": 0.04949260503053665, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.6513652801513672, + "rewards/margins": 0.6365452408790588, + "rewards/rejected": 1.014819860458374, + "step": 555 + }, + { + "epoch": 0.5351490174998507, + "grad_norm": 0.72265625, + "ht_mnpo/logit": -0.0056523485109210014, + "ht_mnpo/residual": -0.013152359053492546, + "ht_mnpo/residual_abs": 0.2248121052980423, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.885791580715609e-07, + "logits/chosen": -2.174121141433716, + "logits/rejected": -1.9470113515853882, + "logps/chosen": -0.28464481234550476, + "logps/rejected": -0.3088465631008148, + "loss": 0.7891527414321899, + "loss/core": 0.7891527414321899, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.3156628608703613, + "rewards/margins": -0.05652338266372681, + "rewards/rejected": 2.3721861839294434, + "step": 560 + }, + { + "epoch": 0.5399271337275279, + "grad_norm": 7.9375, + "ht_mnpo/logit": 0.002409947570413351, + "ht_mnpo/residual": -0.005090050399303436, + "ht_mnpo/residual_abs": 0.06951995939016342, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8389145949069951e-07, + "logits/chosen": -2.351963520050049, + "logits/rejected": -2.055279016494751, + "logps/chosen": -0.3088880181312561, + "logps/rejected": -0.32034802436828613, + "loss": 0.04919041693210602, + "loss/core": 0.04919041693210602, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": 1.2688082456588745, + "rewards/margins": 0.024099458009004593, + "rewards/rejected": 1.244708776473999, + "step": 565 + }, + { + "epoch": 0.5447052499552052, + "grad_norm": 3.375, + "ht_mnpo/logit": 0.051477622240781784, + "ht_mnpo/residual": 0.04397762194275856, + "ht_mnpo/residual_abs": 0.13583263754844666, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.792286216282824e-07, + "logits/chosen": -2.2267661094665527, + "logits/rejected": -2.0639772415161133, + "logps/chosen": -0.29423585534095764, + "logps/rejected": -0.31717735528945923, + "loss": 0.18170244991779327, + "loss/core": 0.18170244991779327, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.9906375408172607, + "rewards/margins": 0.5147761106491089, + "rewards/rejected": 1.4758613109588623, + "step": 570 + }, + { + "epoch": 0.5494833661828824, + "grad_norm": 14.75, + "ht_mnpo/logit": 0.05904921144247055, + "ht_mnpo/residual": 0.05154920369386673, + "ht_mnpo/residual_abs": 0.09378327429294586, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.745923979869336e-07, + "logits/chosen": -2.3841331005096436, + "logits/rejected": -2.1283442974090576, + "logps/chosen": -0.2951911687850952, + "logps/rejected": -0.29570120573043823, + "loss": 0.07935883849859238, + "loss/core": 0.07935883849859238, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 1.4893664121627808, + "rewards/margins": 0.5904920101165771, + "rewards/rejected": 0.8988744616508484, + "step": 575 + }, + { + "epoch": 0.5542614824105596, + "grad_norm": 20.875, + "ht_mnpo/logit": 0.08906254172325134, + "ht_mnpo/residual": 0.08156253397464752, + "ht_mnpo/residual_abs": 0.1606951355934143, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.6998453206075708e-07, + "logits/chosen": -2.1410086154937744, + "logits/rejected": -1.9277909994125366, + "logps/chosen": -0.296114057302475, + "logps/rejected": -0.3074384927749634, + "loss": 0.1655869483947754, + "loss/core": 0.1655869483947754, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.5124917030334473, + "rewards/margins": 0.8906253576278687, + "rewards/rejected": 1.6218664646148682, + "step": 580 + }, + { + "epoch": 0.5590395986382368, + "grad_norm": 21.375, + "ht_mnpo/logit": 0.11307352781295776, + "ht_mnpo/residual": 0.10557352006435394, + "ht_mnpo/residual_abs": 0.11429446935653687, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.6540675667967973e-07, + "logits/chosen": -2.2346272468566895, + "logits/rejected": -1.9701683521270752, + "logps/chosen": -0.2883383631706238, + "logps/rejected": -0.3066983222961426, + "loss": 0.09720277786254883, + "loss/core": 0.09720277786254883, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.7855782508850098, + "rewards/margins": 1.130735158920288, + "rewards/rejected": 1.6548430919647217, + "step": 585 + }, + { + "epoch": 0.5638177148659141, + "grad_norm": 165.0, + "ht_mnpo/logit": 0.07173161953687668, + "ht_mnpo/residual": 0.06423161923885345, + "ht_mnpo/residual_abs": 0.13386103510856628, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.60860793357805e-07, + "logits/chosen": -2.5045523643493652, + "logits/rejected": -2.1490824222564697, + "logps/chosen": -0.2741159200668335, + "logps/rejected": -0.2885042428970337, + "loss": 0.2545531392097473, + "loss/core": 0.2545531392097473, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 1.7796194553375244, + "rewards/margins": 0.7173162698745728, + "rewards/rejected": 1.0623031854629517, + "step": 590 + }, + { + "epoch": 0.5685958310935914, + "grad_norm": 11.9375, + "ht_mnpo/logit": 0.051647841930389404, + "ht_mnpo/residual": 0.04414784163236618, + "ht_mnpo/residual_abs": 0.05486813187599182, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.5634835164602196e-07, + "logits/chosen": -2.4213550090789795, + "logits/rejected": -2.244135618209839, + "logps/chosen": -0.2851046323776245, + "logps/rejected": -0.28078338503837585, + "loss": 0.023371253162622452, + "loss/core": 0.023371253162622452, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.1057530641555786, + "rewards/margins": 0.516478419303894, + "rewards/rejected": 0.5892745852470398, + "step": 595 + }, + { + "epoch": 0.5733739473212686, + "grad_norm": 42.0, + "ht_mnpo/logit": 0.042090535163879395, + "ht_mnpo/residual": 0.03459053486585617, + "ht_mnpo/residual_abs": 0.13397017121315002, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.518711284891132e-07, + "logits/chosen": -2.1188714504241943, + "logits/rejected": -2.0343403816223145, + "logps/chosen": -0.27570241689682007, + "logps/rejected": -0.2801577150821686, + "loss": 0.20747938752174377, + "loss/core": 0.20747938752174377, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.7335128784179688, + "rewards/margins": 0.42090529203414917, + "rewards/rejected": 1.3126074075698853, + "step": 600 + }, + { + "epoch": 0.5781520635489459, + "grad_norm": 50.0, + "ht_mnpo/logit": 0.13987644016742706, + "ht_mnpo/residual": 0.13237643241882324, + "ht_mnpo/residual_abs": 0.1692580282688141, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.47430807587603e-07, + "logits/chosen": -2.182706832885742, + "logits/rejected": -2.002788543701172, + "logps/chosen": -0.2711128294467926, + "logps/rejected": -0.28066202998161316, + "loss": 0.31835612654685974, + "loss/core": 0.31835612654685974, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.817673444747925, + "rewards/margins": 1.3987643718719482, + "rewards/rejected": 1.4189090728759766, + "step": 605 + }, + { + "epoch": 0.5829301797766231, + "grad_norm": 2.625, + "ht_mnpo/logit": 0.05944681912660599, + "ht_mnpo/residual": 0.051946818828582764, + "ht_mnpo/residual_abs": 0.11564245074987411, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.430290587645865e-07, + "logits/chosen": -2.3501338958740234, + "logits/rejected": -2.1967575550079346, + "logps/chosen": -0.25377607345581055, + "logps/rejected": -0.26492077112197876, + "loss": 0.06198473647236824, + "loss/core": 0.06198473647236824, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.9941737651824951, + "rewards/margins": 0.5944682359695435, + "rewards/rejected": 1.3997056484222412, + "step": 610 + }, + { + "epoch": 0.5877082960043003, + "grad_norm": 101.5, + "ht_mnpo/logit": 0.023357708007097244, + "ht_mnpo/residual": 0.015857713297009468, + "ht_mnpo/residual_abs": 0.14009633660316467, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.3866753733777765e-07, + "logits/chosen": -2.2582335472106934, + "logits/rejected": -2.0130774974823, + "logps/chosen": -0.26372048258781433, + "logps/rejected": -0.26262927055358887, + "loss": 0.11254549026489258, + "loss/core": 0.11254549026489258, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 2.158015012741089, + "rewards/margins": 0.23357710242271423, + "rewards/rejected": 1.9244378805160522, + "step": 615 + }, + { + "epoch": 0.5924864122319775, + "grad_norm": 4.09375, + "ht_mnpo/logit": 0.03670644760131836, + "ht_mnpo/residual": 0.029206443578004837, + "ht_mnpo/residual_abs": 0.10636608302593231, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.343478834970121e-07, + "logits/chosen": -2.3275980949401855, + "logits/rejected": -2.0934624671936035, + "logps/chosen": -0.2519906163215637, + "logps/rejected": -0.26454100012779236, + "loss": 0.08830438554286957, + "loss/core": 0.08830438554286957, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.702371597290039, + "rewards/margins": 0.3670644164085388, + "rewards/rejected": 1.3353068828582764, + "step": 620 + }, + { + "epoch": 0.5972645284596548, + "grad_norm": 5.9375, + "ht_mnpo/logit": 0.10891832411289215, + "ht_mnpo/residual": 0.10141833126544952, + "ht_mnpo/residual_abs": 0.18419775366783142, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.3007172168743852e-07, + "logits/chosen": -2.145535469055176, + "logits/rejected": -1.952048659324646, + "logps/chosen": -0.2804136872291565, + "logps/rejected": -0.2959136664867401, + "loss": 0.693868100643158, + "loss/core": 0.693868100643158, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.70146107673645, + "rewards/margins": 1.0891832113265991, + "rewards/rejected": 1.6122777462005615, + "step": 625 + }, + { + "epoch": 0.602042644687332, + "grad_norm": 6.875, + "ht_mnpo/logit": 0.1092601791024208, + "ht_mnpo/residual": 0.10176018625497818, + "ht_mnpo/residual_abs": 0.25469109416007996, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.25840659998631e-07, + "logits/chosen": -2.1239399909973145, + "logits/rejected": -1.8291689157485962, + "logps/chosen": -0.2866722047328949, + "logps/rejected": -0.30721598863601685, + "loss": 0.8774070739746094, + "loss/core": 0.8774070739746094, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 3.1832964420318604, + "rewards/margins": 1.0926018953323364, + "rewards/rejected": 2.0906946659088135, + "step": 630 + }, + { + "epoch": 0.6068207609150092, + "grad_norm": 4.1875, + "ht_mnpo/logit": 0.1126597672700882, + "ht_mnpo/residual": 0.10515977442264557, + "ht_mnpo/residual_abs": 0.1412106454372406, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.2165628955985313e-07, + "logits/chosen": -2.3301758766174316, + "logits/rejected": -2.0502209663391113, + "logps/chosen": -0.2599521577358246, + "logps/rejected": -0.27332812547683716, + "loss": 0.23787467181682587, + "loss/core": 0.23787467181682587, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 3.085704803466797, + "rewards/margins": 1.1265976428985596, + "rewards/rejected": 1.9591071605682373, + "step": 635 + }, + { + "epoch": 0.6115988771426865, + "grad_norm": 76.0, + "ht_mnpo/logit": 0.12395603954792023, + "ht_mnpo/residual": 0.1164560317993164, + "ht_mnpo/residual_abs": 0.13639070093631744, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.175201839416988e-07, + "logits/chosen": -2.1668243408203125, + "logits/rejected": -2.0291244983673096, + "logps/chosen": -0.29634636640548706, + "logps/rejected": -0.3084748387336731, + "loss": 0.22144794464111328, + "loss/core": 0.22144794464111328, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.488347053527832, + "rewards/margins": 1.2395600080490112, + "rewards/rejected": 1.2487866878509521, + "step": 640 + }, + { + "epoch": 0.6163769933703638, + "grad_norm": 276.0, + "ht_mnpo/logit": 0.01642979122698307, + "ht_mnpo/residual": 0.008929798379540443, + "ht_mnpo/residual_abs": 0.1276627480983734, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.1343389856433658e-07, + "logits/chosen": -2.2169954776763916, + "logits/rejected": -1.9446134567260742, + "logps/chosen": -0.2655065059661865, + "logps/rejected": -0.2783452868461609, + "loss": 0.21419398486614227, + "loss/core": 0.21419398486614227, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.4721295833587646, + "rewards/margins": 0.16429801285266876, + "rewards/rejected": 1.3078315258026123, + "step": 645 + }, + { + "epoch": 0.621155109598041, + "grad_norm": 23.5, + "ht_mnpo/logit": 0.1376076340675354, + "ht_mnpo/residual": 0.13010765612125397, + "ht_mnpo/residual_abs": 0.15285083651542664, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0939897011258e-07, + "logits/chosen": -2.1072707176208496, + "logits/rejected": -1.9887821674346924, + "logps/chosen": -0.2807113230228424, + "logps/rejected": -0.2912576496601105, + "loss": 0.3052452504634857, + "loss/core": 0.3052452504634857, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.705549955368042, + "rewards/margins": 1.3760764598846436, + "rewards/rejected": 1.3294737339019775, + "step": 650 + }, + { + "epoch": 0.6259332258257182, + "grad_norm": 66.0, + "ht_mnpo/logit": 0.03930670768022537, + "ht_mnpo/residual": 0.031806714832782745, + "ht_mnpo/residual_abs": 0.07188865542411804, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0541691595800336e-07, + "logits/chosen": -2.338975429534912, + "logits/rejected": -2.051806688308716, + "logps/chosen": -0.2882004678249359, + "logps/rejected": -0.31319910287857056, + "loss": 0.0360516794025898, + "loss/core": 0.0360516794025898, + "rewards/accuracies": 0.75, + "rewards/chosen": 1.4952980279922485, + "rewards/margins": 0.3930671513080597, + "rewards/rejected": 1.1022307872772217, + "step": 655 + }, + { + "epoch": 0.6307113420533954, + "grad_norm": 6.78125, + "ht_mnpo/logit": 0.056034285575151443, + "ht_mnpo/residual": 0.04853427782654762, + "ht_mnpo/residual_abs": 0.07523567974567413, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0148923358832021e-07, + "logits/chosen": -2.217348098754883, + "logits/rejected": -2.0050175189971924, + "logps/chosen": -0.2928197979927063, + "logps/rejected": -0.33136898279190063, + "loss": 0.04290390759706497, + "loss/core": 0.04290390759706497, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.5474987030029297, + "rewards/margins": 0.5603427886962891, + "rewards/rejected": 0.9871557950973511, + "step": 660 + }, + { + "epoch": 0.6354894582810727, + "grad_norm": 142.0, + "ht_mnpo/logit": 0.14371220767498016, + "ht_mnpo/residual": 0.13621219992637634, + "ht_mnpo/residual_abs": 0.15110185742378235, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.761740004423926e-08, + "logits/chosen": -2.243295669555664, + "logits/rejected": -1.9963970184326172, + "logps/chosen": -0.2896881401538849, + "logps/rejected": -0.3032901883125305, + "loss": 0.4213927388191223, + "loss/core": 0.4213927388191223, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.0614871978759766, + "rewards/margins": 1.437122106552124, + "rewards/rejected": 0.6243652701377869, + "step": 665 + }, + { + "epoch": 0.6402675745087499, + "grad_norm": 110.0, + "ht_mnpo/logit": 0.10394921153783798, + "ht_mnpo/residual": 0.09644920378923416, + "ht_mnpo/residual_abs": 0.1331409364938736, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.380287136400999e-08, + "logits/chosen": -2.4142959117889404, + "logits/rejected": -2.069192409515381, + "logps/chosen": -0.29473966360092163, + "logps/rejected": -0.31998682022094727, + "loss": 0.4319857954978943, + "loss/core": 0.4319857954978943, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.7670764923095703, + "rewards/margins": 1.039492130279541, + "rewards/rejected": 0.7275844216346741, + "step": 670 + }, + { + "epoch": 0.6450456907364271, + "grad_norm": 174.0, + "ht_mnpo/logit": 0.06010546162724495, + "ht_mnpo/residual": 0.05260546877980232, + "ht_mnpo/residual_abs": 0.1043517217040062, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.004708203586628e-08, + "logits/chosen": -2.420896291732788, + "logits/rejected": -2.177229166030884, + "logps/chosen": -0.26569315791130066, + "logps/rejected": -0.2760010063648224, + "loss": 0.17001619935035706, + "loss/core": 0.17001619935035706, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.536733865737915, + "rewards/margins": 0.6010546684265137, + "rewards/rejected": 0.9356793165206909, + "step": 675 + }, + { + "epoch": 0.6498238069641044, + "grad_norm": 5.78125, + "ht_mnpo/logit": 0.006507441401481628, + "ht_mnpo/residual": -0.0009925604099407792, + "ht_mnpo/residual_abs": 0.12537150084972382, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.635144445857407e-08, + "logits/chosen": -2.0410115718841553, + "logits/rejected": -1.7555296421051025, + "logps/chosen": -0.29283419251441956, + "logps/rejected": -0.302853524684906, + "loss": 0.1671266406774521, + "loss/core": 0.1671266406774521, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 1.7759294509887695, + "rewards/margins": 0.06507435441017151, + "rewards/rejected": 1.7108551263809204, + "step": 680 + }, + { + "epoch": 0.6546019231917817, + "grad_norm": 14.1875, + "ht_mnpo/logit": -0.01924746111035347, + "ht_mnpo/residual": -0.026747453957796097, + "ht_mnpo/residual_abs": 0.21825659275054932, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.271734841028552e-08, + "logits/chosen": -2.1334753036499023, + "logits/rejected": -1.8235372304916382, + "logps/chosen": -0.2628119885921478, + "logps/rejected": -0.31102806329727173, + "loss": 0.42427802085876465, + "loss/core": 0.42427802085876465, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.3661179542541504, + "rewards/margins": -0.19247445464134216, + "rewards/rejected": 2.5585925579071045, + "step": 685 + }, + { + "epoch": 0.6593800394194589, + "grad_norm": 21.625, + "ht_mnpo/logit": 0.061263781040906906, + "ht_mnpo/residual": 0.053763777017593384, + "ht_mnpo/residual_abs": 0.07502800226211548, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 7.91461605259007e-08, + "logits/chosen": -2.293973684310913, + "logits/rejected": -2.080751419067383, + "logps/chosen": -0.2748379111289978, + "logps/rejected": -0.2834241986274719, + "loss": 0.03468334674835205, + "loss/core": 0.03468334674835205, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.741302251815796, + "rewards/margins": 0.6126378178596497, + "rewards/rejected": 1.128664493560791, + "step": 690 + }, + { + "epoch": 0.6641581556471361, + "grad_norm": 3.921875, + "ht_mnpo/logit": 0.23695942759513855, + "ht_mnpo/residual": 0.22945943474769592, + "ht_mnpo/residual_abs": 0.24223342537879944, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 7.563922378313217e-08, + "logits/chosen": -2.222285509109497, + "logits/rejected": -1.9881519079208374, + "logps/chosen": -0.3527771532535553, + "logps/rejected": -0.3299418091773987, + "loss": 1.1627099514007568, + "loss/core": 1.1627099514007568, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 3.928004503250122, + "rewards/margins": 2.369594097137451, + "rewards/rejected": 1.558410406112671, + "step": 695 + }, + { + "epoch": 0.6689362718748133, + "grad_norm": 5.03125, + "ht_mnpo/logit": 0.14287082850933075, + "ht_mnpo/residual": 0.13537083566188812, + "ht_mnpo/residual_abs": 0.21414688229560852, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 7.219785699746572e-08, + "logits/chosen": -2.14042329788208, + "logits/rejected": -1.9120944738388062, + "logps/chosen": -0.3552348017692566, + "logps/rejected": -0.263727605342865, + "loss": 0.5581281781196594, + "loss/core": 0.5581281781196594, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 3.3907947540283203, + "rewards/margins": 1.4287083148956299, + "rewards/rejected": 1.9620863199234009, + "step": 700 + }, + { + "epoch": 0.6737143881024906, + "grad_norm": 0.734375, + "ht_mnpo/logit": 0.000579959130845964, + "ht_mnpo/residual": -0.006920039653778076, + "ht_mnpo/residual_abs": 0.12014114856719971, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.882335432620779e-08, + "logits/chosen": -2.2957048416137695, + "logits/rejected": -1.9869102239608765, + "logps/chosen": -0.2796603739261627, + "logps/rejected": -0.29228758811950684, + "loss": 0.1334332525730133, + "loss/core": 0.1334332525730133, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 1.6085796356201172, + "rewards/margins": 0.005799674894660711, + "rewards/rejected": 1.602779746055603, + "step": 705 + }, + { + "epoch": 0.6784925043301678, + "grad_norm": 5.625, + "ht_mnpo/logit": 0.023095760494470596, + "ht_mnpo/residual": 0.015595759265124798, + "ht_mnpo/residual_abs": 0.10114194452762604, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.551698478180589e-08, + "logits/chosen": -2.0242340564727783, + "logits/rejected": -1.878504753112793, + "logps/chosen": -0.3086937963962555, + "logps/rejected": -0.32835036516189575, + "loss": 0.0820605680346489, + "loss/core": 0.0820605680346489, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.449677586555481, + "rewards/margins": 0.23095755279064178, + "rewards/rejected": 1.2187201976776123, + "step": 710 + }, + { + "epoch": 0.683270620557845, + "grad_norm": 81.5, + "ht_mnpo/logit": 0.0778246745467186, + "ht_mnpo/residual": 0.07032467424869537, + "ht_mnpo/residual_abs": 0.15270543098449707, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.22799917546252e-08, + "logits/chosen": -2.0666940212249756, + "logits/rejected": -1.8228778839111328, + "logps/chosen": -0.3078117072582245, + "logps/rejected": -0.3023551106452942, + "loss": 0.18929661810398102, + "loss/core": 0.18929661810398102, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 2.2512452602386475, + "rewards/margins": 0.7782467007637024, + "rewards/rejected": 1.4729986190795898, + "step": 715 + }, + { + "epoch": 0.6880487367855223, + "grad_norm": 3.859375, + "ht_mnpo/logit": 0.05400656536221504, + "ht_mnpo/residual": 0.04650656133890152, + "ht_mnpo/residual_abs": 0.07696235924959183, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.9113592545359944e-08, + "logits/chosen": -2.173678159713745, + "logits/rejected": -1.9528112411499023, + "logps/chosen": -0.2821721136569977, + "logps/rejected": -0.3137813210487366, + "loss": 0.03128375485539436, + "loss/core": 0.03128375485539436, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 2.233147144317627, + "rewards/margins": 0.5400656461715698, + "rewards/rejected": 1.6930816173553467, + "step": 720 + }, + { + "epoch": 0.6928268530131996, + "grad_norm": 147.0, + "ht_mnpo/logit": 0.03776507079601288, + "ht_mnpo/residual": 0.030265068635344505, + "ht_mnpo/residual_abs": 0.1769125610589981, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.601897790725643e-08, + "logits/chosen": -2.2530579566955566, + "logits/rejected": -1.901747465133667, + "logps/chosen": -0.29231947660446167, + "logps/rejected": -0.30032289028167725, + "loss": 0.28970783948898315, + "loss/core": 0.28970783948898315, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.0136709213256836, + "rewards/margins": 0.37765076756477356, + "rewards/rejected": 1.6360204219818115, + "step": 725 + }, + { + "epoch": 0.6976049692408768, + "grad_norm": 16.375, + "ht_mnpo/logit": 0.1292092353105545, + "ht_mnpo/residual": 0.12170922756195068, + "ht_mnpo/residual_abs": 0.19856879115104675, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.299731159831952e-08, + "logits/chosen": -2.076169967651367, + "logits/rejected": -1.8429524898529053, + "logps/chosen": -0.2840667963027954, + "logps/rejected": -0.30480578541755676, + "loss": 0.28391191363334656, + "loss/core": 0.28391191363334656, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.3225340843200684, + "rewards/margins": 1.2920923233032227, + "rewards/rejected": 2.0304417610168457, + "step": 730 + }, + { + "epoch": 0.702383085468554, + "grad_norm": 158.0, + "ht_mnpo/logit": 0.004397551529109478, + "ht_mnpo/residual": -0.0031024485360831022, + "ht_mnpo/residual_abs": 0.16906771063804626, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.0049729943671013e-08, + "logits/chosen": -2.103620767593384, + "logits/rejected": -1.879486083984375, + "logps/chosen": -0.314971923828125, + "logps/rejected": -0.2881259024143219, + "loss": 0.4299296736717224, + "loss/core": 0.4299296736717224, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.3823599815368652, + "rewards/margins": 0.04397547245025635, + "rewards/rejected": 2.3383851051330566, + "step": 735 + }, + { + "epoch": 0.7071612016962313, + "grad_norm": 2.171875, + "ht_mnpo/logit": 0.10530763864517212, + "ht_mnpo/residual": 0.09780765324831009, + "ht_mnpo/residual_abs": 0.16430619359016418, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.7177341408223994e-08, + "logits/chosen": -2.098759889602661, + "logits/rejected": -1.9267299175262451, + "logps/chosen": -0.27479445934295654, + "logps/rejected": -0.27386826276779175, + "loss": 0.2672664225101471, + "loss/core": 0.2672664225101471, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.1735119819641113, + "rewards/margins": 1.0530763864517212, + "rewards/rejected": 1.1204354763031006, + "step": 740 + }, + { + "epoch": 0.7119393179239085, + "grad_norm": 15.4375, + "ht_mnpo/logit": 0.09088204801082611, + "ht_mnpo/residual": 0.08338205516338348, + "ht_mnpo/residual_abs": 0.10531797260046005, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.438122617983442e-08, + "logits/chosen": -2.1337571144104004, + "logits/rejected": -1.9622604846954346, + "logps/chosen": -0.28831762075424194, + "logps/rejected": -0.2993650436401367, + "loss": 0.07868538051843643, + "loss/core": 0.07868538051843643, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.8238359689712524, + "rewards/margins": 0.9088205099105835, + "rewards/rejected": 0.9150153398513794, + "step": 745 + }, + { + "epoch": 0.7167174341515857, + "grad_norm": 51.75, + "ht_mnpo/logit": 0.05060400813817978, + "ht_mnpo/residual": 0.043104007840156555, + "ht_mnpo/residual_abs": 0.06728905439376831, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.1662435763087114e-08, + "logits/chosen": -2.316929578781128, + "logits/rejected": -2.122931957244873, + "logps/chosen": -0.29240328073501587, + "logps/rejected": -0.29196321964263916, + "loss": 0.03344697132706642, + "loss/core": 0.03344697132706642, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.3464288711547852, + "rewards/margins": 0.506040096282959, + "rewards/rejected": 0.8403886556625366, + "step": 750 + }, + { + "epoch": 0.7214955503792629, + "grad_norm": 7.78125, + "ht_mnpo/logit": 0.11179667711257935, + "ht_mnpo/residual": 0.10429668426513672, + "ht_mnpo/residual_abs": 0.17980945110321045, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.902199258386732e-08, + "logits/chosen": -2.128986358642578, + "logits/rejected": -1.8619842529296875, + "logps/chosen": -0.29354777932167053, + "logps/rejected": -0.31483781337738037, + "loss": 0.3539460003376007, + "loss/core": 0.3539460003376007, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.8777692317962646, + "rewards/margins": 1.117966651916504, + "rewards/rejected": 1.7598024606704712, + "step": 755 + }, + { + "epoch": 0.7262736666069403, + "grad_norm": 18.625, + "ht_mnpo/logit": 0.0071696205995976925, + "ht_mnpo/residual": -0.0003303714038338512, + "ht_mnpo/residual_abs": 0.1921873241662979, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.646088960486862e-08, + "logits/chosen": -2.150174379348755, + "logits/rejected": -1.9791088104248047, + "logps/chosen": -0.29564157128334045, + "logps/rejected": -0.3015114367008209, + "loss": 0.293133944272995, + "loss/core": 0.293133944272995, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.4021666049957275, + "rewards/margins": 0.0716962069272995, + "rewards/rejected": 2.330470561981201, + "step": 760 + }, + { + "epoch": 0.7310517828346175, + "grad_norm": 13.0, + "ht_mnpo/logit": 0.04746823385357857, + "ht_mnpo/residual": 0.03996824100613594, + "ht_mnpo/residual_abs": 0.0960867777466774, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.398008995217988e-08, + "logits/chosen": -2.1956920623779297, + "logits/rejected": -1.8496367931365967, + "logps/chosen": -0.2813601791858673, + "logps/rejected": -0.2744866907596588, + "loss": 0.05206139758229256, + "loss/core": 0.05206139758229256, + "rewards/accuracies": 0.75, + "rewards/chosen": 2.0784502029418945, + "rewards/margins": 0.47468239068984985, + "rewards/rejected": 1.6037676334381104, + "step": 765 + }, + { + "epoch": 0.7358298990622947, + "grad_norm": 87.0, + "ht_mnpo/logit": 0.019193410873413086, + "ht_mnpo/residual": 0.011693410575389862, + "ht_mnpo/residual_abs": 0.11662435531616211, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.1580526553093315e-08, + "logits/chosen": -2.220792293548584, + "logits/rejected": -2.0077321529388428, + "logps/chosen": -0.27943068742752075, + "logps/rejected": -0.2912485897541046, + "loss": 0.0868755504488945, + "loss/core": 0.0868755504488945, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.315573215484619, + "rewards/margins": 0.19193415343761444, + "rewards/rejected": 2.1236391067504883, + "step": 770 + }, + { + "epoch": 0.7406080152899719, + "grad_norm": 3.46875, + "ht_mnpo/logit": 0.03249342739582062, + "ht_mnpo/residual": 0.024993427097797394, + "ht_mnpo/residual_abs": 0.058049798011779785, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.9263101785268252e-08, + "logits/chosen": -2.2310421466827393, + "logits/rejected": -2.000556230545044, + "logps/chosen": -0.3053019642829895, + "logps/rejected": -0.3055349886417389, + "loss": 0.022902490571141243, + "loss/core": 0.022902490571141243, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 1.5678751468658447, + "rewards/margins": 0.3249342441558838, + "rewards/rejected": 1.242940902709961, + "step": 775 + }, + { + "epoch": 0.7453861315176492, + "grad_norm": 1.9609375, + "ht_mnpo/logit": -0.004409261513501406, + "ht_mnpo/residual": -0.011909263208508492, + "ht_mnpo/residual_abs": 0.09732024371623993, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.7028687137384264e-08, + "logits/chosen": -2.4477899074554443, + "logits/rejected": -2.261199951171875, + "logps/chosen": -0.316506028175354, + "logps/rejected": -0.31703802943229675, + "loss": 0.20356547832489014, + "loss/core": 0.20356547832489014, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 0.9836400747299194, + "rewards/margins": -0.04409266263246536, + "rewards/rejected": 1.0277327299118042, + "step": 780 + }, + { + "epoch": 0.7501642477453264, + "grad_norm": 0.17578125, + "ht_mnpo/logit": 0.08832958340644836, + "ht_mnpo/residual": 0.08082958310842514, + "ht_mnpo/residual_abs": 0.12986600399017334, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4878122881409447e-08, + "logits/chosen": -2.177534341812134, + "logits/rejected": -1.9233522415161133, + "logps/chosen": -0.29545319080352783, + "logps/rejected": -0.313882976770401, + "loss": 0.19263024628162384, + "loss/core": 0.19263024628162384, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.1445488929748535, + "rewards/margins": 0.8832958340644836, + "rewards/rejected": 1.2612531185150146, + "step": 785 + }, + { + "epoch": 0.7549423639730036, + "grad_norm": 9.0, + "ht_mnpo/logit": 0.0722842663526535, + "ht_mnpo/residual": 0.06478426605463028, + "ht_mnpo/residual_abs": 0.0955527275800705, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.2812217756608937e-08, + "logits/chosen": -2.3180410861968994, + "logits/rejected": -2.018643379211426, + "logps/chosen": -0.24670171737670898, + "logps/rejected": -0.25532251596450806, + "loss": 0.05465935915708542, + "loss/core": 0.05465935915708542, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.9452526569366455, + "rewards/margins": 0.7228427529335022, + "rewards/rejected": 1.2224102020263672, + "step": 790 + }, + { + "epoch": 0.7597204802006808, + "grad_norm": 159.0, + "ht_mnpo/logit": -0.06759858131408691, + "ht_mnpo/residual": -0.07509857416152954, + "ht_mnpo/residual_abs": 0.16478586196899414, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.0831748665410763e-08, + "logits/chosen": -2.2069625854492188, + "logits/rejected": -1.985700249671936, + "logps/chosen": -0.2841342091560364, + "logps/rejected": -0.28607645630836487, + "loss": 0.44809237122535706, + "loss/core": 0.44809237122535706, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 1.9460901021957397, + "rewards/margins": -0.6759856939315796, + "rewards/rejected": 2.6220760345458984, + "step": 795 + }, + { + "epoch": 0.7644985964283582, + "grad_norm": 0.19921875, + "ht_mnpo/logit": 0.15688112378120422, + "ht_mnpo/residual": 0.1493811309337616, + "ht_mnpo/residual_abs": 0.15697996318340302, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8937460381244967e-08, + "logits/chosen": -2.2979936599731445, + "logits/rejected": -2.1380813121795654, + "logps/chosen": -0.243856281042099, + "logps/rejected": -0.25572383403778076, + "loss": 0.35162821412086487, + "loss/core": 0.35162821412086487, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.6728522777557373, + "rewards/margins": 1.5688111782073975, + "rewards/rejected": 1.1040410995483398, + "step": 800 + }, + { + "epoch": 0.7692767126560354, + "grad_norm": 2.578125, + "ht_mnpo/logit": 0.023322004824876785, + "ht_mnpo/residual": 0.01582200825214386, + "ht_mnpo/residual_abs": 0.059140510857105255, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.713006526846439e-08, + "logits/chosen": -2.3517727851867676, + "logits/rejected": -2.1378703117370605, + "logps/chosen": -0.31542864441871643, + "logps/rejected": -0.3220391571521759, + "loss": 0.021347444504499435, + "loss/core": 0.021347444504499435, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 0.9367267489433289, + "rewards/margins": 0.23322002589702606, + "rewards/rejected": 0.7035066485404968, + "step": 805 + }, + { + "epoch": 0.7740548288837126, + "grad_norm": 251.0, + "ht_mnpo/logit": 0.18958637118339539, + "ht_mnpo/residual": 0.18208637833595276, + "ht_mnpo/residual_abs": 0.21843548119068146, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.541024301445404e-08, + "logits/chosen": -2.0479319095611572, + "logits/rejected": -1.8690176010131836, + "logps/chosen": -0.3233865797519684, + "logps/rejected": -0.330311119556427, + "loss": 0.7746216654777527, + "loss/core": 0.7746216654777527, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.8327841758728027, + "rewards/margins": 1.8958637714385986, + "rewards/rejected": 0.9369203448295593, + "step": 810 + }, + { + "epoch": 0.7788329451113898, + "grad_norm": 62.5, + "ht_mnpo/logit": 0.08295156061649323, + "ht_mnpo/residual": 0.0754515528678894, + "ht_mnpo/residual_abs": 0.1541987508535385, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.3778640374027983e-08, + "logits/chosen": -2.2893285751342773, + "logits/rejected": -2.0092718601226807, + "logps/chosen": -0.2440989464521408, + "logps/rejected": -0.2622814476490021, + "loss": 0.1557014435529709, + "loss/core": 0.1557014435529709, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.43802547454834, + "rewards/margins": 0.8295154571533203, + "rewards/rejected": 1.6085097789764404, + "step": 815 + }, + { + "epoch": 0.7836110613390671, + "grad_norm": 6.78125, + "ht_mnpo/logit": 0.14719262719154358, + "ht_mnpo/residual": 0.13969263434410095, + "ht_mnpo/residual_abs": 0.17651373147964478, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.2235870926211616e-08, + "logits/chosen": -2.12347412109375, + "logits/rejected": -1.8785676956176758, + "logps/chosen": -0.27428001165390015, + "logps/rejected": -0.3116721212863922, + "loss": 0.4936157763004303, + "loss/core": 0.4936157763004303, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.819645643234253, + "rewards/margins": 1.4719264507293701, + "rewards/rejected": 1.3477191925048828, + "step": 820 + }, + { + "epoch": 0.7883891775667443, + "grad_norm": 736.0, + "ht_mnpo/logit": 0.2026674449443817, + "ht_mnpo/residual": 0.1951674520969391, + "ht_mnpo/residual_abs": 0.22531075775623322, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0782514843499652e-08, + "logits/chosen": -2.2073588371276855, + "logits/rejected": -2.0113062858581543, + "logps/chosen": -0.3016754984855652, + "logps/rejected": -0.29153111577033997, + "loss": 0.9155440330505371, + "loss/core": 0.9155440330505371, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 2.9878766536712646, + "rewards/margins": 2.026674270629883, + "rewards/rejected": 0.9612022638320923, + "step": 825 + }, + { + "epoch": 0.7931672937944215, + "grad_norm": 0.83203125, + "ht_mnpo/logit": 0.06144893914461136, + "ht_mnpo/residual": 0.053948938846588135, + "ht_mnpo/residual_abs": 0.08334354311227798, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.419118673676924e-09, + "logits/chosen": -2.2769930362701416, + "logits/rejected": -2.078378200531006, + "logps/chosen": -0.28695565462112427, + "logps/rejected": -0.2857491075992584, + "loss": 0.08329080790281296, + "loss/core": 0.08329080790281296, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": 1.6761438846588135, + "rewards/margins": 0.6144893169403076, + "rewards/rejected": 1.0616545677185059, + "step": 830 + }, + { + "epoch": 0.7979454100220987, + "grad_norm": 428.0, + "ht_mnpo/logit": -0.0352485366165638, + "ht_mnpo/residual": -0.04274853318929672, + "ht_mnpo/residual_abs": 0.21524009108543396, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.14619513428405e-09, + "logits/chosen": -2.257803440093994, + "logits/rejected": -1.8916332721710205, + "logps/chosen": -0.280212938785553, + "logps/rejected": -0.34277522563934326, + "loss": 0.42016157507896423, + "loss/core": 0.42016157507896423, + "rewards/accuracies": 0.75, + "rewards/chosen": 2.795287609100342, + "rewards/margins": -0.3524852693080902, + "rewards/rejected": 3.147773027420044, + "step": 835 + }, + { + "epoch": 0.802723526249776, + "grad_norm": 10.375, + "ht_mnpo/logit": 0.022869501262903214, + "ht_mnpo/residual": 0.015369502827525139, + "ht_mnpo/residual_abs": 0.06976980715990067, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.96422291980539e-09, + "logits/chosen": -2.278902530670166, + "logits/rejected": -2.0701346397399902, + "logps/chosen": -0.2895943224430084, + "logps/rejected": -0.31108802556991577, + "loss": 0.021526243537664413, + "loss/core": 0.021526243537664413, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 1.143057107925415, + "rewards/margins": 0.22869500517845154, + "rewards/rejected": 0.9143620729446411, + "step": 840 + }, + { + "epoch": 0.8075016424774533, + "grad_norm": 4.71875, + "ht_mnpo/logit": 0.03050408326089382, + "ht_mnpo/residual": 0.0230040792375803, + "ht_mnpo/residual_abs": 0.16995054483413696, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.8736465216517594e-09, + "logits/chosen": -2.1468443870544434, + "logits/rejected": -1.917742133140564, + "logps/chosen": -0.2518450617790222, + "logps/rejected": -0.27282077074050903, + "loss": 0.4268716275691986, + "loss/core": 0.4268716275691986, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.2956597805023193, + "rewards/margins": 0.3050408363342285, + "rewards/rejected": 1.9906187057495117, + "step": 845 + }, + { + "epoch": 0.8122797587051305, + "grad_norm": 2.1875, + "ht_mnpo/logit": 0.01607021503150463, + "ht_mnpo/residual": 0.008570205420255661, + "ht_mnpo/residual_abs": 0.14475668966770172, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.874876061005173e-09, + "logits/chosen": -2.292693614959717, + "logits/rejected": -2.110647678375244, + "logps/chosen": -0.3145769536495209, + "logps/rejected": -0.30854150652885437, + "loss": 0.40554890036582947, + "loss/core": 0.40554890036582947, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.2903105020523071, + "rewards/margins": 0.16070207953453064, + "rewards/rejected": 1.129608392715454, + "step": 850 + }, + { + "epoch": 0.8170578749328078, + "grad_norm": 169.0, + "ht_mnpo/logit": 0.0835026353597641, + "ht_mnpo/residual": 0.07600263506174088, + "ht_mnpo/residual_abs": 0.19562701880931854, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.968287134589188e-09, + "logits/chosen": -2.2144742012023926, + "logits/rejected": -1.9130665063858032, + "logps/chosen": -0.2866402268409729, + "logps/rejected": -0.29126277565956116, + "loss": 0.24363788962364197, + "loss/core": 0.24363788962364197, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.9844703674316406, + "rewards/margins": 0.8350263833999634, + "rewards/rejected": 2.149444103240967, + "step": 855 + }, + { + "epoch": 0.821835991160485, + "grad_norm": 89.5, + "ht_mnpo/logit": 0.12511678040027618, + "ht_mnpo/residual": 0.11761675775051117, + "ht_mnpo/residual_abs": 0.19286657869815826, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.154220673422192e-09, + "logits/chosen": -2.188568592071533, + "logits/rejected": -1.984760046005249, + "logps/chosen": -0.3057268261909485, + "logps/rejected": -0.3176608979701996, + "loss": 0.2807512879371643, + "loss/core": 0.2807512879371643, + "rewards/accuracies": 0.75, + "rewards/chosen": 2.978262424468994, + "rewards/margins": 1.2511677742004395, + "rewards/rejected": 1.7270948886871338, + "step": 860 + }, + { + "epoch": 0.8266141073881622, + "grad_norm": 158.0, + "ht_mnpo/logit": 0.01128818467259407, + "ht_mnpo/residual": 0.00378818204626441, + "ht_mnpo/residual_abs": 0.15559642016887665, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4329828146074096e-09, + "logits/chosen": -2.1104423999786377, + "logits/rejected": -1.8544162511825562, + "logps/chosen": -0.27419915795326233, + "logps/rejected": -0.3105354309082031, + "loss": 0.20748119056224823, + "loss/core": 0.20748119056224823, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.2308783531188965, + "rewards/margins": 0.11288192123174667, + "rewards/rejected": 2.1179964542388916, + "step": 865 + }, + { + "epoch": 0.8313922236158394, + "grad_norm": 2.625, + "ht_mnpo/logit": 0.07102836668491364, + "ht_mnpo/residual": 0.06352836638689041, + "ht_mnpo/residual_abs": 0.13127169013023376, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8048447862070714e-09, + "logits/chosen": -2.0416035652160645, + "logits/rejected": -1.8697378635406494, + "logps/chosen": -0.27052196860313416, + "logps/rejected": -0.290427029132843, + "loss": 0.16899879276752472, + "loss/core": 0.16899879276752472, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.2697510719299316, + "rewards/margins": 0.7102835774421692, + "rewards/rejected": 1.5594675540924072, + "step": 870 + }, + { + "epoch": 0.8361703398435167, + "grad_norm": 76.5, + "ht_mnpo/logit": -0.0015212118159979582, + "ht_mnpo/residual": -0.009021207690238953, + "ht_mnpo/residual_abs": 0.08979154378175735, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.2700428052447033e-09, + "logits/chosen": -2.4572672843933105, + "logits/rejected": -2.1243271827697754, + "logps/chosen": -0.2736932635307312, + "logps/rejected": -0.296703040599823, + "loss": 0.06740878522396088, + "loss/core": 0.06740878522396088, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.2251904010772705, + "rewards/margins": -0.0152121065184474, + "rewards/rejected": 1.2404022216796875, + "step": 875 + }, + { + "epoch": 0.840948456071194, + "grad_norm": 2.078125, + "ht_mnpo/logit": 0.00960737094283104, + "ht_mnpo/residual": 0.0021073739044368267, + "ht_mnpo/residual_abs": 0.08030150830745697, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.287779888734858e-10, + "logits/chosen": -2.3116345405578613, + "logits/rejected": -2.1608469486236572, + "logps/chosen": -0.28452223539352417, + "logps/rejected": -0.30294761061668396, + "loss": 0.09350978583097458, + "loss/core": 0.09350978583097458, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.2112499475479126, + "rewards/margins": 0.0960737094283104, + "rewards/rejected": 1.1151762008666992, + "step": 880 + }, + { + "epoch": 0.8457265722988712, + "grad_norm": 7.125, + "ht_mnpo/logit": 0.09174073487520218, + "ht_mnpo/residual": 0.08424073457717896, + "ht_mnpo/residual_abs": 0.13746334612369537, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.812162787445062e-10, + "logits/chosen": -2.1234521865844727, + "logits/rejected": -1.9998477697372437, + "logps/chosen": -0.2799425721168518, + "logps/rejected": -0.2617390751838684, + "loss": 0.14151671528816223, + "loss/core": 0.14151671528816223, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.404963970184326, + "rewards/margins": 0.9174073934555054, + "rewards/rejected": 1.4875565767288208, + "step": 885 + }, + { + "epoch": 0.8505046885265484, + "grad_norm": 19.625, + "ht_mnpo/logit": 0.09642306715250015, + "ht_mnpo/residual": 0.08892306685447693, + "ht_mnpo/residual_abs": 0.11294756829738617, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.2748837860270265e-10, + "logits/chosen": -2.3085834980010986, + "logits/rejected": -2.0683159828186035, + "logps/chosen": -0.2952830195426941, + "logps/rejected": -0.30370476841926575, + "loss": 0.17521965503692627, + "loss/core": 0.17521965503692627, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.697967290878296, + "rewards/margins": 0.9642306566238403, + "rewards/rejected": 1.7337367534637451, + "step": 890 + }, + { + "epoch": 0.8552828047542257, + "grad_norm": 16.625, + "ht_mnpo/logit": 0.05871545523405075, + "ht_mnpo/residual": 0.05121545121073723, + "ht_mnpo/residual_abs": 0.1308048516511917, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.768970513457151e-11, + "logits/chosen": -2.4693121910095215, + "logits/rejected": -2.181762218475342, + "logps/chosen": -0.2554033398628235, + "logps/rejected": -0.2718905806541443, + "loss": 0.20783178508281708, + "loss/core": 0.20783178508281708, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.2181339263916016, + "rewards/margins": 0.5871545672416687, + "rewards/rejected": 1.6309795379638672, + "step": 895 + }, + { + "epoch": 0.8600609209819029, + "grad_norm": 97.0, + "ht_mnpo/logit": 0.06254192441701889, + "ht_mnpo/residual": 0.05504193156957626, + "ht_mnpo/residual_abs": 0.15723498165607452, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8803520859811406e-12, + "logits/chosen": -2.3032777309417725, + "logits/rejected": -2.058039665222168, + "logps/chosen": -0.27136459946632385, + "logps/rejected": -0.27709969878196716, + "loss": 0.19733476638793945, + "loss/core": 0.19733476638793945, + "rewards/accuracies": 0.75, + "rewards/chosen": 1.982447862625122, + "rewards/margins": 0.6254192590713501, + "rewards/rejected": 1.3570287227630615, + "step": 900 + }, + { + "epoch": 0.8600609209819029, + "step": 900, + "total_flos": 0.0, + "train_loss": 0.3149479303053684, + "train_runtime": 7160.3559, + "train_samples_per_second": 2.011, + "train_steps_per_second": 0.126 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..06519b2 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:814ac036a3308126d3cf22e2371ee110a0e0c7626dd67b29de7d551749715e65 +size 7057