From 303afd033942d27150a2ab4f79648a10bfda8771 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Thu, 23 Jul 2026 10:34:10 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: promotion/qwen3-8b-aaai27-flagship-ht-mnpo-conciseness-s42 Source: Original Platform --- .gitattributes | 36 + README.md | 24 + all_results.json | 9 + chat_template.jinja | 89 + config.json | 71 + config.yaml | 58 + generation_config.json | 12 + job_status.json | 23 + model.safetensors | 3 + objective_protocol.json | 69 + provenance.json | 14 + resource_profile.json | 21 + run_status.json | 15 + split_manifest.json | 23 + stability_gate.json | 38 + tokenizer.json | 3 + tokenizer_config.json | 30 + train_results.json | 9 + trainer_state.json | 3823 +++++++++++++++++++++++++++++++++++++++ training_args.bin | 3 + 20 files changed, 4373 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 config.yaml create mode 100644 generation_config.json create mode 100644 job_status.json create mode 100644 model.safetensors create mode 100644 objective_protocol.json create mode 100644 provenance.json create mode 100644 resource_profile.json create mode 100644 run_status.json create mode 100644 split_manifest.json create mode 100644 stability_gate.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..b411a44 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +--- +library_name: transformers +base_model: Qwen/Qwen3-8B +tags: +- ronpo +- mnpo +- preference-optimization +--- + +# qwen3-8b-aaai27-flagship-ht-mnpo-conciseness-s42 + +Research checkpoint for the RONPO AAAI revision experiments. + +- Method: ht_mnpo_conciseness +- Base model: `Qwen/Qwen3-8B` +- Seed: 42 +- Local source at upload time: `/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/full/ht_mnpo_conciseness/seed42/attempt1` +- Uploaded at UTC: 2026-07-13T15:57:15Z +- Run status metadata: `{"attempt": 1, "effective_batch_size": 16, "method": "ht_mnpo_conciseness", "objective_protocol": "objective_protocol.json", "optimizer_steps": 900, "resource_profile": "resource_profile.json", "scope_amendment": "scope_amendment_20260713.json", "seed": 42, "split_manifest": "split_manifest.json", "stability_gate": "stability_gate.json", "status": "completed", "wandb_run_id": "617b54d60423", "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/617b54d60423"}` + +AAAI-27 P1 matched budget: 900 optimizer steps, effective batch 16; passed non-thinking and collapse stability gates; sealed-test metrics are not used for checkpoint selection. + +Intended use: reproducibility and evaluation for the RONPO paper. This +checkpoint is not intended as a production assistant. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..707b6cf --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8594449418084154, + "total_flos": 0.0, + "train_loss": 0.7468786084651947, + "train_runtime": 6934.2752, + "train_samples": 16755, + "train_samples_per_second": 2.077, + "train_steps_per_second": 0.13 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..1a1f2f6 --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +model_name_or_path: Qwen/Qwen3-8B +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/ht_mnpo_conciseness: 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: ht_mnpo +eta: 0.0075 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.0 +preference_sft_weight: 0.0 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 5.0e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 42 +gradient_accumulation_steps: 16 +gradient_checkpointing: true +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 1 +per_device_eval_batch_size: 1 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/full/ht_mnpo_conciseness/seed42/attempt1 +run_name: aaai27-flagship-full-ht_mnpo_conciseness-s42-a1 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/job_status.json b/job_status.json new file mode 100644 index 0000000..dac6a44 --- /dev/null +++ b/job_status.json @@ -0,0 +1,23 @@ +{ + "status": "completed", + "stage": "full", + "method": "ht_mnpo_conciseness", + "seed": 42, + "attempt": 1, + "gpu": 0, + "gpus": [ + 0 + ], + "returncode": 0, + "finite_metrics": true, + "oom": false, + "traceback": false, + "reason": "passed", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "617b54d60423", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/617b54d60423", + "output_dir": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/full/ht_mnpo_conciseness/seed42/attempt1", + "log": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/logs/full_ht_mnpo_conciseness_s42_a1.log", + "completed_at": "2026-07-13T15:54:29Z" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..1ee9f4c --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0581cc303f219c4421275f9a68662f41c979a3fa688f62451c3d07f390004d7e +size 16381517208 diff --git a/objective_protocol.json b/objective_protocol.json new file mode 100644 index 0000000..35b3b44 --- /dev/null +++ b/objective_protocol.json @@ -0,0 +1,69 @@ +{ + "schema_version": 1, + "frozen_at_kst": "2026-07-12T21:40:00+09:00", + "base_model": "Qwen/Qwen3-8B", + "primary_triple": { + "helpfulness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "ultrafeedback-helpfulness", + "transform": "identity" + }, + "safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "conciseness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "helpsteer-verbosity", + "transform": "negate" + } + }, + "gate_thresholds": { + "raw_score_std_strictly_greater_than": 0.01, + "aggregate_prompt_pair_spearman_median_strictly_less_than": 0.0, + "aggregate_pairwise_top1_mismatch_at_least": 0.5 + }, + "gate_pool": "source-train Qwen3-8B response pool split into deterministic train/validation; sealed source-test excluded", + "training_seeds": [42, 43, 44], + "optimizer_steps": 900, + "effective_batch_size": 16, + "decode": { + "enable_thinking": false, + "temperature": 0.7, + "top_p": 0.9, + "max_new_tokens": 2048, + "decode_seed": 42 + }, + "methods": [ + "ronpo_full_expect", + "ronpo_k_only", + "dpo", + "ipo", + "simpo", + "kto", + "sppo_avg", + "inpo_avg", + "ht_mnpo_helpfulness", + "ht_mnpo_safety", + "ht_mnpo_conciseness" + ], + "selection_policy": "No sealed-test selection. One preregistered configuration per method; failures use only the declared stability retry recipe, at most three attempts.", + "stability_gate": { + "validation_prompts": 128, + "min_candidate_base_mean_word_ratio": 0.33, + "max_candidate_base_mean_word_ratio": 2.0, + "max_consecutive_identical_word_run": 20, + "think_tag_count": 0, + "empty_response_count": 0, + "retry_recipe": "attempt 2 halves learning rate, enables gradient checkpointing, and adds reference=0.02/SFT=0.002 anchoring to unanchored unified losses; attempt 3 quarters learning rate and uses reference=0.05/SFT=0.005; RONPO uses reference=0.05/SFT=0.005 from attempt 1" + }, + "primary_metric": "mean prompt-level minimum of per-prompt min-max-normalized helpfulness, safety, conciseness", + "bootstrap": { + "unit": "prompt", + "paired": true, + "resamples": 2000, + "interval": "percentile_95", + "seed": 42 + } +} diff --git a/provenance.json b/provenance.json new file mode 100644 index 0000000..5c68198 --- /dev/null +++ b/provenance.json @@ -0,0 +1,14 @@ +{ + "method": "ht_mnpo_conciseness", + "seed": 42, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "617b54d60423", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/617b54d60423", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json", + "scope_amendment": "scope_amendment_20260713.json" +} diff --git a/resource_profile.json b/resource_profile.json new file mode 100644 index 0000000..18eec47 --- /dev/null +++ b/resource_profile.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "measured_at_kst": "2026-07-12T22:38:00+09:00", + "hardware": "NVIDIA B200 183359 MiB reported total; 178.35 GiB PyTorch-visible capacity", + "smoke_evidence": { + "attempted_profile": {"per_device_batch_size": 2, "gradient_accumulation_steps": 8, "gradient_checkpointing": false, "effective_batch_size": 16}, + "outcome": "OOM before completing the 20-step smoke across every launched unified baseline; representative process used 176.87 GiB and failed a 1.62 GiB allocation. KTO also OOMed at the optimizer step.", + "paper_metric": false + }, + "selected_full_training_profile": { + "per_device_batch_size": 2, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": false, + "attention_implementation": "PyTorch SDPA with cuDNN SDPA disabled", + "effective_batch_size": 16, + "applies_symmetrically_to_all_unified_pairwise_methods": true, + "kto_implementation_constraint": "TRL KTO requires actual batch size > 1. KTO uses batch 2 / accumulation 8 and identical AdamW with foreach=False. PyTorch SDPA is used with cuDNN SDPA disabled. The fixed base reference model is placed on a second B200 and its exact logits are returned to the policy device; the policy/optimizer remain on the first B200. This avoids a measured ~6-hour reference-precompute path while leaving the KTO loss, actual/effective batch, optimizer, and 900-step budget unchanged." + }, + "retry_semantics": "Resource selection is separated from method-stability attempts. Full attempt 1 uses the smoke-validated SDPA batch-2 profile and retains the preregistered method LR/anchor; attempts 2 and 3 fall back to batch 1 / accumulation 16 / gradient checkpointing and apply the declared LR/anchor stabilization recipe.", + "provenance_note": "No reward or sealed-evaluation result was available when this hardware profile was selected." +} diff --git a/run_status.json b/run_status.json new file mode 100644 index 0000000..66d451c --- /dev/null +++ b/run_status.json @@ -0,0 +1,15 @@ +{ + "method": "ht_mnpo_conciseness", + "seed": 42, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "617b54d60423", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/617b54d60423", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json", + "scope_amendment": "scope_amendment_20260713.json", + "status": "completed" +} diff --git a/split_manifest.json b/split_manifest.json new file mode 100644 index 0000000..4fb96bd --- /dev/null +++ b/split_manifest.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "split_rule": "sort source-train prompts by sha256('flagship-v1||'+prompt_id); first round(10%) validation", + "validation_fraction": 0.1, + "source_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/avg_oracle", + "base_model": "Qwen/Qwen3-8B", + "counts": { + "train": 16780, + "validation": 1864, + "sealed_test": 604 + }, + "prompt_id_sha256": { + "train": "fcf6f2ea5a0b7bcd41179e6f5288f4d850b2d04bdb729da91907bd18bf733a69", + "validation": "a87ba6f519192164daa487b109a6547dd8cbde46a27a373d6da201ca2d061b8f", + "sealed_test": "b398ced6af6cf73d731e1a5f32a0631d8060323d51eda1e375695566bf5f1867" + }, + "overlaps": { + "train_validation": 0, + "train_sealed_test": 0, + "validation_sealed_test": 0 + }, + "sealed_test_policy": "prompts only until S4a; no objective scores or model generations used in S0-S3" +} diff --git a/stability_gate.json b/stability_gate.json new file mode 100644 index 0000000..7573970 --- /dev/null +++ b/stability_gate.json @@ -0,0 +1,38 @@ +{ + "status": "passed", + "passed": true, + "base": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 388.1484375, + "max_words": 1379, + "max_repeat_run": 4 + }, + "candidate": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 408.3515625, + "max_words": 1388, + "max_repeat_run": 5 + }, + "candidate_base_mean_word_ratio": 1.0520499969808585, + "thresholds": { + "min_length_ratio": 0.33, + "max_length_ratio": 2.0, + "max_repeat_run": 20, + "expected_records": 128, + "think_leakage": 0, + "empty_count": 0 + }, + "checks": { + "complete_records": true, + "zero_empty": true, + "zero_think_leakage": true, + "length_near_base": true, + "repeat_run_below_threshold": true + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ea3c9e2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..707b6cf --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8594449418084154, + "total_flos": 0.0, + "train_loss": 0.7468786084651947, + "train_runtime": 6934.2752, + "train_samples": 16755, + "train_samples_per_second": 2.077, + "train_steps_per_second": 0.13 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..c86344f --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,3823 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8594449418084154, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.004774694121157864, + "grad_norm": 60.5, + "ht_mnpo/logit": 0.2415936291217804, + "ht_mnpo/residual": 0.2340935915708542, + "ht_mnpo/residual_abs": 0.23548129200935364, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.222222222222222e-08, + "logits/chosen": -2.011902332305908, + "logits/rejected": -2.257323741912842, + "logps/chosen": -0.25824588537216187, + "logps/rejected": -0.2757476568222046, + "loss": 0.5226427316665649, + "loss/core": 0.5226427316665649, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 3.5988030433654785, + "rewards/margins": 2.41593599319458, + "rewards/rejected": 1.1828668117523193, + "step": 5 + }, + { + "epoch": 0.009549388242315727, + "grad_norm": 2736.0, + "ht_mnpo/logit": 0.26693910360336304, + "ht_mnpo/residual": 0.2594391107559204, + "ht_mnpo/residual_abs": 0.2632564902305603, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5e-08, + "logits/chosen": -2.0542407035827637, + "logits/rejected": -2.270714282989502, + "logps/chosen": -0.2949259281158447, + "logps/rejected": -0.29275721311569214, + "loss": 1.0850975513458252, + "loss/core": 1.0850975513458252, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.9222195148468018, + "rewards/margins": 2.669390916824341, + "rewards/rejected": 1.2528284788131714, + "step": 10 + }, + { + "epoch": 0.01432408236347359, + "grad_norm": 10.8125, + "ht_mnpo/logit": 0.19589567184448242, + "ht_mnpo/residual": 0.188395693898201, + "ht_mnpo/residual_abs": 0.19078858196735382, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 7.777777777777778e-08, + "logits/chosen": -2.067415475845337, + "logits/rejected": -2.327845811843872, + "logps/chosen": -0.314541757106781, + "logps/rejected": -0.29293060302734375, + "loss": 0.5424398183822632, + "loss/core": 0.5424398183822632, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.4876604080200195, + "rewards/margins": 1.9589569568634033, + "rewards/rejected": 0.5287034511566162, + "step": 15 + }, + { + "epoch": 0.019098776484631454, + "grad_norm": 36.0, + "ht_mnpo/logit": 0.3147750794887543, + "ht_mnpo/residual": 0.30727505683898926, + "ht_mnpo/residual_abs": 0.30989599227905273, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0555555555555555e-07, + "logits/chosen": -1.927666425704956, + "logits/rejected": -2.2836575508117676, + "logps/chosen": -0.3032557964324951, + "logps/rejected": -0.2924278676509857, + "loss": 1.1851537227630615, + "loss/core": 1.1851537227630615, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.1310954093933105, + "rewards/margins": 3.1477506160736084, + "rewards/rejected": 0.9833448529243469, + "step": 20 + }, + { + "epoch": 0.023873470605789315, + "grad_norm": 73.5, + "ht_mnpo/logit": 0.12462173402309418, + "ht_mnpo/residual": 0.11712173372507095, + "ht_mnpo/residual_abs": 0.12095322459936142, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.3333333333333334e-07, + "logits/chosen": -2.0298550128936768, + "logits/rejected": -2.2757391929626465, + "logps/chosen": -0.31602349877357483, + "logps/rejected": -0.3301704525947571, + "loss": 0.18879806995391846, + "loss/core": 0.18879806995391846, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.6169111728668213, + "rewards/margins": 1.2462173700332642, + "rewards/rejected": 0.3706938624382019, + "step": 25 + }, + { + "epoch": 0.02864816472694718, + "grad_norm": 368.0, + "ht_mnpo/logit": 0.12316179275512695, + "ht_mnpo/residual": 0.11566178500652313, + "ht_mnpo/residual_abs": 0.11892789602279663, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.611111111111111e-07, + "logits/chosen": -2.042910575866699, + "logits/rejected": -2.2183661460876465, + "logps/chosen": -0.28492701053619385, + "logps/rejected": -0.3071616291999817, + "loss": 0.1271527111530304, + "loss/core": 0.1271527111530304, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.4336202144622803, + "rewards/margins": 1.23161780834198, + "rewards/rejected": 1.2020022869110107, + "step": 30 + }, + { + "epoch": 0.033422858848105044, + "grad_norm": 9.625, + "ht_mnpo/logit": 0.3371656835079193, + "ht_mnpo/residual": 0.3296656310558319, + "ht_mnpo/residual_abs": 0.3311299681663513, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8888888888888888e-07, + "logits/chosen": -1.9528906345367432, + "logits/rejected": -2.2516660690307617, + "logps/chosen": -0.27027541399002075, + "logps/rejected": -0.27153652906417847, + "loss": 1.1185133457183838, + "loss/core": 1.1185133457183838, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 4.334385395050049, + "rewards/margins": 3.3716564178466797, + "rewards/rejected": 0.9627290964126587, + "step": 35 + }, + { + "epoch": 0.03819755296926291, + "grad_norm": 14.1875, + "ht_mnpo/logit": 0.21347995102405548, + "ht_mnpo/residual": 0.20597994327545166, + "ht_mnpo/residual_abs": 0.20741498470306396, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.1666666666666667e-07, + "logits/chosen": -2.00959849357605, + "logits/rejected": -2.2939441204071045, + "logps/chosen": -0.26474690437316895, + "logps/rejected": -0.276932954788208, + "loss": 0.41715115308761597, + "loss/core": 0.41715115308761597, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 3.2156689167022705, + "rewards/margins": 2.1347994804382324, + "rewards/rejected": 1.080869436264038, + "step": 40 + }, + { + "epoch": 0.04297224709042077, + "grad_norm": 2.203125, + "ht_mnpo/logit": 0.2520568370819092, + "ht_mnpo/residual": 0.24455685913562775, + "ht_mnpo/residual_abs": 0.24652080237865448, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4444444444444445e-07, + "logits/chosen": -2.1033449172973633, + "logits/rejected": -2.3409695625305176, + "logps/chosen": -0.2933202385902405, + "logps/rejected": -0.3012697398662567, + "loss": 0.9052979350090027, + "loss/core": 0.9052979350090027, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 3.102382183074951, + "rewards/margins": 2.520568609237671, + "rewards/rejected": 0.5818136930465698, + "step": 45 + }, + { + "epoch": 0.04774694121157863, + "grad_norm": 103.5, + "ht_mnpo/logit": 0.2532837688922882, + "ht_mnpo/residual": 0.24578377604484558, + "ht_mnpo/residual_abs": 0.2527261972427368, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.7222222222222216e-07, + "logits/chosen": -1.9192997217178345, + "logits/rejected": -2.134883403778076, + "logps/chosen": -0.26225295662879944, + "logps/rejected": -0.29307669401168823, + "loss": 0.6835693717002869, + "loss/core": 0.6835693717002869, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.6984927654266357, + "rewards/margins": 2.5328376293182373, + "rewards/rejected": 1.1656550168991089, + "step": 50 + }, + { + "epoch": 0.052521635332736495, + "grad_norm": 2.328125, + "ht_mnpo/logit": 0.203359916806221, + "ht_mnpo/residual": 0.1958599090576172, + "ht_mnpo/residual_abs": 0.1985529363155365, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3e-07, + "logits/chosen": -1.9192931652069092, + "logits/rejected": -2.2161107063293457, + "logps/chosen": -0.2721022963523865, + "logps/rejected": -0.2915636897087097, + "loss": 0.437203973531723, + "loss/core": 0.437203973531723, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.0200881958007812, + "rewards/margins": 2.0335988998413086, + "rewards/rejected": 0.9864892959594727, + "step": 55 + }, + { + "epoch": 0.05729632945389436, + "grad_norm": 19.5, + "ht_mnpo/logit": 0.2052539587020874, + "ht_mnpo/residual": 0.19775396585464478, + "ht_mnpo/residual_abs": 0.20126383006572723, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.2777777777777776e-07, + "logits/chosen": -1.9262555837631226, + "logits/rejected": -2.166170120239258, + "logps/chosen": -0.29561707377433777, + "logps/rejected": -0.31034865975379944, + "loss": 1.1692918539047241, + "loss/core": 1.1692918539047241, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.606314182281494, + "rewards/margins": 2.052539348602295, + "rewards/rejected": 0.5537749528884888, + "step": 60 + }, + { + "epoch": 0.062071023575052224, + "grad_norm": 924.0, + "ht_mnpo/logit": 0.2645673453807831, + "ht_mnpo/residual": 0.25706732273101807, + "ht_mnpo/residual_abs": 0.2587380111217499, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.5555555555555553e-07, + "logits/chosen": -2.020003080368042, + "logits/rejected": -2.3714873790740967, + "logps/chosen": -0.2805558741092682, + "logps/rejected": -0.27380621433258057, + "loss": 1.0117337703704834, + "loss/core": 1.0117337703704834, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 3.615847110748291, + "rewards/margins": 2.6456735134124756, + "rewards/rejected": 0.9701736569404602, + "step": 65 + }, + { + "epoch": 0.06684571769621009, + "grad_norm": 1.40625, + "ht_mnpo/logit": 0.14507603645324707, + "ht_mnpo/residual": 0.13757602870464325, + "ht_mnpo/residual_abs": 0.13970807194709778, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.8333333333333335e-07, + "logits/chosen": -1.9647560119628906, + "logits/rejected": -2.2312729358673096, + "logps/chosen": -0.28448420763015747, + "logps/rejected": -0.29094865918159485, + "loss": 0.08910230547189713, + "loss/core": 0.08910230547189713, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.4837958812713623, + "rewards/margins": 1.4507603645324707, + "rewards/rejected": 1.0330355167388916, + "step": 70 + }, + { + "epoch": 0.07162041181736795, + "grad_norm": 99.5, + "ht_mnpo/logit": 0.34166499972343445, + "ht_mnpo/residual": 0.3341650068759918, + "ht_mnpo/residual_abs": 0.3365333080291748, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.1111111111111107e-07, + "logits/chosen": -1.8882007598876953, + "logits/rejected": -2.146094560623169, + "logps/chosen": -0.28841090202331543, + "logps/rejected": -0.28859004378318787, + "loss": 0.9650213122367859, + "loss/core": 0.9650213122367859, + "rewards/accuracies": 0.9375, + "rewards/chosen": 4.434047698974609, + "rewards/margins": 3.4166500568389893, + "rewards/rejected": 1.0173976421356201, + "step": 75 + }, + { + "epoch": 0.07639510593852582, + "grad_norm": 98.0, + "ht_mnpo/logit": 0.4781981110572815, + "ht_mnpo/residual": 0.47069811820983887, + "ht_mnpo/residual_abs": 0.47250255942344666, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.3888888888888884e-07, + "logits/chosen": -2.0093605518341064, + "logits/rejected": -2.3259353637695312, + "logps/chosen": -0.2699472904205322, + "logps/rejected": -0.2692564129829407, + "loss": 2.545933485031128, + "loss/core": 2.545933485031128, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 5.524721622467041, + "rewards/margins": 4.781980991363525, + "rewards/rejected": 0.7427404522895813, + "step": 80 + }, + { + "epoch": 0.08116980005968367, + "grad_norm": 28.375, + "ht_mnpo/logit": 0.3949056565761566, + "ht_mnpo/residual": 0.3874056935310364, + "ht_mnpo/residual_abs": 0.38943880796432495, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.6666666666666666e-07, + "logits/chosen": -1.8436777591705322, + "logits/rejected": -2.107656478881836, + "logps/chosen": -0.27149876952171326, + "logps/rejected": -0.26327431201934814, + "loss": 1.4980366230010986, + "loss/core": 1.4980366230010986, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 5.046863555908203, + "rewards/margins": 3.949056625366211, + "rewards/rejected": 1.097806692123413, + "step": 85 + }, + { + "epoch": 0.08594449418084155, + "grad_norm": 9.6875, + "ht_mnpo/logit": 0.2532029151916504, + "ht_mnpo/residual": 0.24570293724536896, + "ht_mnpo/residual_abs": 0.24952733516693115, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.944444444444445e-07, + "logits/chosen": -1.9614741802215576, + "logits/rejected": -2.207104444503784, + "logps/chosen": -0.308493047952652, + "logps/rejected": -0.2764720320701599, + "loss": 0.856387734413147, + "loss/core": 0.856387734413147, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.3289856910705566, + "rewards/margins": 2.532029151916504, + "rewards/rejected": 0.7969565391540527, + "step": 90 + }, + { + "epoch": 0.0907191883019994, + "grad_norm": 504.0, + "ht_mnpo/logit": 0.2930489778518677, + "ht_mnpo/residual": 0.28554895520210266, + "ht_mnpo/residual_abs": 0.2872154712677002, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.999699149323369e-07, + "logits/chosen": -2.090327262878418, + "logits/rejected": -2.431143283843994, + "logps/chosen": -0.2864113748073578, + "logps/rejected": -0.28018051385879517, + "loss": 1.093916654586792, + "loss/core": 1.093916654586792, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 3.887937545776367, + "rewards/margins": 2.9304897785186768, + "rewards/rejected": 0.9574478268623352, + "step": 95 + }, + { + "epoch": 0.09549388242315726, + "grad_norm": 43.75, + "ht_mnpo/logit": 0.13229675590991974, + "ht_mnpo/residual": 0.12479674816131592, + "ht_mnpo/residual_abs": 0.15026070177555084, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.998477067547739e-07, + "logits/chosen": -1.9240186214447021, + "logits/rejected": -2.271806001663208, + "logps/chosen": -0.2857854664325714, + "logps/rejected": -0.3030196726322174, + "loss": 0.26169103384017944, + "loss/core": 0.26169103384017944, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.2315335273742676, + "rewards/margins": 1.322967529296875, + "rewards/rejected": 0.908565878868103, + "step": 100 + }, + { + "epoch": 0.10026857654431513, + "grad_norm": 788.0, + "ht_mnpo/logit": 0.2821234464645386, + "ht_mnpo/residual": 0.27462345361709595, + "ht_mnpo/residual_abs": 0.275934636592865, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.996315410727229e-07, + "logits/chosen": -2.103553533554077, + "logits/rejected": -2.360391855239868, + "logps/chosen": -0.32235783338546753, + "logps/rejected": -0.29607582092285156, + "loss": 0.564530611038208, + "loss/core": 0.564530611038208, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 4.4333977699279785, + "rewards/margins": 2.8212342262268066, + "rewards/rejected": 1.6121633052825928, + "step": 105 + }, + { + "epoch": 0.10504327066547299, + "grad_norm": 122.0, + "ht_mnpo/logit": 0.29163163900375366, + "ht_mnpo/residual": 0.28413161635398865, + "ht_mnpo/residual_abs": 0.28595611453056335, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.993214991772562e-07, + "logits/chosen": -2.0258805751800537, + "logits/rejected": -2.2438912391662598, + "logps/chosen": -0.289029598236084, + "logps/rejected": -0.2787890136241913, + "loss": 1.244662880897522, + "loss/core": 1.244662880897522, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 4.081713676452637, + "rewards/margins": 2.916316032409668, + "rewards/rejected": 1.1653969287872314, + "step": 110 + }, + { + "epoch": 0.10981796478663086, + "grad_norm": 58.5, + "ht_mnpo/logit": 0.2460227906703949, + "ht_mnpo/residual": 0.23852276802062988, + "ht_mnpo/residual_abs": 0.24091574549674988, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.989176976624511e-07, + "logits/chosen": -1.842900276184082, + "logits/rejected": -2.1219844818115234, + "logps/chosen": -0.27585095167160034, + "logps/rejected": -0.26493367552757263, + "loss": 1.031520128250122, + "loss/core": 1.031520128250122, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.6426234245300293, + "rewards/margins": 2.4602277278900146, + "rewards/rejected": 1.1823960542678833, + "step": 115 + }, + { + "epoch": 0.11459265890778872, + "grad_norm": 516.0, + "ht_mnpo/logit": 0.33325010538101196, + "ht_mnpo/residual": 0.32575008273124695, + "ht_mnpo/residual_abs": 0.3277893662452698, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.984202883815428e-07, + "logits/chosen": -2.084425687789917, + "logits/rejected": -2.2928617000579834, + "logps/chosen": -0.2503313422203064, + "logps/rejected": -0.27189603447914124, + "loss": 0.949650764465332, + "loss/core": 0.949650764465332, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 4.397518157958984, + "rewards/margins": 3.332500457763672, + "rewards/rejected": 1.0650172233581543, + "step": 120 + }, + { + "epoch": 0.11936735302894658, + "grad_norm": 756.0, + "ht_mnpo/logit": 0.4765322208404541, + "ht_mnpo/residual": 0.4690321981906891, + "ht_mnpo/residual_abs": 0.47166210412979126, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.978294583898195e-07, + "logits/chosen": -1.8222243785858154, + "logits/rejected": -2.102375030517578, + "logps/chosen": -0.2784605026245117, + "logps/rejected": -0.2793668210506439, + "loss": 2.5388500690460205, + "loss/core": 2.5388500690460205, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 5.699808120727539, + "rewards/margins": 4.765322208404541, + "rewards/rejected": 0.9344862103462219, + "step": 125 + }, + { + "epoch": 0.12414204715010445, + "grad_norm": 18.875, + "ht_mnpo/logit": 0.28754621744155884, + "ht_mnpo/residual": 0.2800462245941162, + "ht_mnpo/residual_abs": 0.28318971395492554, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.971454298742779e-07, + "logits/chosen": -1.8026081323623657, + "logits/rejected": -2.1300437450408936, + "logps/chosen": -0.3127506673336029, + "logps/rejected": -0.2983434796333313, + "loss": 0.685983419418335, + "loss/core": 0.685983419418335, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.466167449951172, + "rewards/margins": 2.875462293624878, + "rewards/rejected": 0.590705156326294, + "step": 130 + }, + { + "epoch": 0.12891674127126232, + "grad_norm": 248.0, + "ht_mnpo/logit": 0.18734104931354523, + "ht_mnpo/residual": 0.1798410415649414, + "ht_mnpo/residual_abs": 0.18434356153011322, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.963684600700678e-07, + "logits/chosen": -2.0444672107696533, + "logits/rejected": -2.295093059539795, + "logps/chosen": -0.28641682863235474, + "logps/rejected": -0.29985663294792175, + "loss": 0.27129822969436646, + "loss/core": 0.27129822969436646, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.71679949760437, + "rewards/margins": 1.8734104633331299, + "rewards/rejected": 0.8433891534805298, + "step": 135 + }, + { + "epoch": 0.13369143539242018, + "grad_norm": 2.171875, + "ht_mnpo/logit": 0.1653221845626831, + "ht_mnpo/residual": 0.15782217681407928, + "ht_mnpo/residual_abs": 0.16011430323123932, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.954988411637571e-07, + "logits/chosen": -1.931020736694336, + "logits/rejected": -2.2295100688934326, + "logps/chosen": -0.28395944833755493, + "logps/rejected": -0.27869370579719543, + "loss": 0.12267724424600601, + "loss/core": 0.12267724424600601, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.6366994380950928, + "rewards/margins": 1.6532217264175415, + "rewards/rejected": 0.9834780693054199, + "step": 140 + }, + { + "epoch": 0.13846612951357803, + "grad_norm": 0.75, + "ht_mnpo/logit": 0.2931879162788391, + "ht_mnpo/residual": 0.28568798303604126, + "ht_mnpo/residual_abs": 0.2914429306983948, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.945369001834514e-07, + "logits/chosen": -1.8905494213104248, + "logits/rejected": -2.122390031814575, + "logps/chosen": -0.3317320942878723, + "logps/rejected": -0.30477237701416016, + "loss": 1.277919054031372, + "loss/core": 1.277919054031372, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.9621856212615967, + "rewards/margins": 2.9318792819976807, + "rewards/rejected": 1.0303064584732056, + "step": 145 + }, + { + "epoch": 0.1432408236347359, + "grad_norm": 1.546875, + "ht_mnpo/logit": 0.2134993076324463, + "ht_mnpo/residual": 0.20599932968616486, + "ht_mnpo/residual_abs": 0.20773494243621826, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.93482998875813e-07, + "logits/chosen": -2.0588443279266357, + "logits/rejected": -2.308776378631592, + "logps/chosen": -0.2723400592803955, + "logps/rejected": -0.2848970293998718, + "loss": 0.3919183015823364, + "loss/core": 0.3919183015823364, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.260389804840088, + "rewards/margins": 2.134993076324463, + "rewards/rejected": 1.1253963708877563, + "step": 150 + }, + { + "epoch": 0.14801551775589375, + "grad_norm": 165.0, + "ht_mnpo/logit": 0.32777029275894165, + "ht_mnpo/residual": 0.32027024030685425, + "ht_mnpo/residual_abs": 0.3234081268310547, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.923375335700223e-07, + "logits/chosen": -2.0180845260620117, + "logits/rejected": -2.2344937324523926, + "logps/chosen": -0.3052632808685303, + "logps/rejected": -0.2794122099876404, + "loss": 1.312514066696167, + "loss/core": 1.312514066696167, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 4.1154704093933105, + "rewards/margins": 3.277703046798706, + "rewards/rejected": 0.8377677798271179, + "step": 155 + }, + { + "epoch": 0.15279021187705163, + "grad_norm": 89.5, + "ht_mnpo/logit": 0.15068449079990387, + "ht_mnpo/residual": 0.14318448305130005, + "ht_mnpo/residual_abs": 0.14576324820518494, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.911009350287347e-07, + "logits/chosen": -2.039623975753784, + "logits/rejected": -2.4383645057678223, + "logps/chosen": -0.26783841848373413, + "logps/rejected": -0.27481716871261597, + "loss": 0.16402561962604523, + "loss/core": 0.16402561962604523, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.1534738540649414, + "rewards/margins": 1.5068447589874268, + "rewards/rejected": 0.6466289758682251, + "step": 160 + }, + { + "epoch": 0.1575649059982095, + "grad_norm": 3.75, + "ht_mnpo/logit": 0.37046578526496887, + "ht_mnpo/residual": 0.36296579241752625, + "ht_mnpo/residual_abs": 0.36714157462120056, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.897736682860885e-07, + "logits/chosen": -2.004047393798828, + "logits/rejected": -2.2582778930664062, + "logps/chosen": -0.2819061875343323, + "logps/rejected": -0.2967601418495178, + "loss": 1.949097990989685, + "loss/core": 1.949097990989685, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.285637855529785, + "rewards/margins": 3.704658031463623, + "rewards/rejected": 0.5809799432754517, + "step": 165 + }, + { + "epoch": 0.16233960011936735, + "grad_norm": 12.625, + "ht_mnpo/logit": 0.2548084855079651, + "ht_mnpo/residual": 0.24730846285820007, + "ht_mnpo/residual_abs": 0.24948911368846893, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.883562324728241e-07, + "logits/chosen": -2.0560150146484375, + "logits/rejected": -2.2419166564941406, + "logps/chosen": -0.3087208867073059, + "logps/rejected": -0.32812321186065674, + "loss": 0.8445270657539368, + "loss/core": 0.8445270657539368, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.7604167461395264, + "rewards/margins": 2.5480847358703613, + "rewards/rejected": 1.212332010269165, + "step": 170 + }, + { + "epoch": 0.1671142942405252, + "grad_norm": 25.25, + "ht_mnpo/logit": 0.20657019317150116, + "ht_mnpo/residual": 0.19907021522521973, + "ht_mnpo/residual_abs": 0.20315143465995789, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.868491606285823e-07, + "logits/chosen": -2.158867597579956, + "logits/rejected": -2.4179515838623047, + "logps/chosen": -0.29173943400382996, + "logps/rejected": -0.3005591630935669, + "loss": 1.0301963090896606, + "loss/core": 1.0301963090896606, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.7829740047454834, + "rewards/margins": 2.065701961517334, + "rewards/rejected": 0.7172719240188599, + "step": 175 + }, + { + "epoch": 0.1718889883616831, + "grad_norm": 2.28125, + "ht_mnpo/logit": 0.25792625546455383, + "ht_mnpo/residual": 0.2504262328147888, + "ht_mnpo/residual_abs": 0.2523646354675293, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.852530195014489e-07, + "logits/chosen": -1.9154199361801147, + "logits/rejected": -2.108656644821167, + "logps/chosen": -0.29020223021507263, + "logps/rejected": -0.28535544872283936, + "loss": 0.9428588151931763, + "loss/core": 0.9428588151931763, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.405637741088867, + "rewards/margins": 2.5792624950408936, + "rewards/rejected": 0.8263753056526184, + "step": 180 + }, + { + "epoch": 0.17666368248284095, + "grad_norm": 9.1875, + "ht_mnpo/logit": 0.11725939810276031, + "ht_mnpo/residual": 0.1097593903541565, + "ht_mnpo/residual_abs": 0.1118379607796669, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.835684093348244e-07, + "logits/chosen": -2.0307536125183105, + "logits/rejected": -2.279958486557007, + "logps/chosen": -0.2843839228153229, + "logps/rejected": -0.2895074784755707, + "loss": 0.05697331950068474, + "loss/core": 0.05697331950068474, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 1.8723161220550537, + "rewards/margins": 1.1725938320159912, + "rewards/rejected": 0.6997222900390625, + "step": 185 + }, + { + "epoch": 0.1814383766039988, + "grad_norm": 314.0, + "ht_mnpo/logit": 0.2137770652770996, + "ht_mnpo/residual": 0.2062770575284958, + "ht_mnpo/residual_abs": 0.21894094347953796, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.817959636416969e-07, + "logits/chosen": -1.7537132501602173, + "logits/rejected": -1.9378242492675781, + "logps/chosen": -0.29650992155075073, + "logps/rejected": -0.2991756796836853, + "loss": 0.4277607798576355, + "loss/core": 0.4277607798576355, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.179126262664795, + "rewards/margins": 2.137770652770996, + "rewards/rejected": 2.041355609893799, + "step": 190 + }, + { + "epoch": 0.18621307072515667, + "grad_norm": 616.0, + "ht_mnpo/logit": 0.28763315081596375, + "ht_mnpo/residual": 0.2801331579685211, + "ht_mnpo/residual_abs": 0.2813577353954315, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.799363489664039e-07, + "logits/chosen": -1.9013550281524658, + "logits/rejected": -2.1618947982788086, + "logps/chosen": -0.2702489197254181, + "logps/rejected": -0.2578398883342743, + "loss": 0.7751284837722778, + "loss/core": 0.7751284837722778, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 4.433821678161621, + "rewards/margins": 2.876331329345703, + "rewards/rejected": 1.557490587234497, + "step": 195 + }, + { + "epoch": 0.19098776484631452, + "grad_norm": 764.0, + "ht_mnpo/logit": 0.36720559000968933, + "ht_mnpo/residual": 0.3597055971622467, + "ht_mnpo/residual_abs": 0.3611852824687958, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.779902646339721e-07, + "logits/chosen": -1.9602934122085571, + "logits/rejected": -2.284992218017578, + "logps/chosen": -0.30019837617874146, + "logps/rejected": -0.29457828402519226, + "loss": 1.2417619228363037, + "loss/core": 1.2417619228363037, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 4.696887969970703, + "rewards/margins": 3.672055721282959, + "rewards/rejected": 1.024832010269165, + "step": 200 + }, + { + "epoch": 0.1957624589674724, + "grad_norm": 1.15625, + "ht_mnpo/logit": 0.13184615969657898, + "ht_mnpo/residual": 0.12434617429971695, + "ht_mnpo/residual_abs": 0.12654730677604675, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.759584424871301e-07, + "logits/chosen": -2.0989491939544678, + "logits/rejected": -2.2950825691223145, + "logps/chosen": -0.27947577834129333, + "logps/rejected": -0.2976261079311371, + "loss": 0.12396552413702011, + "loss/core": 0.12396552413702011, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.6087183952331543, + "rewards/margins": 1.3184616565704346, + "rewards/rejected": 1.2902568578720093, + "step": 205 + }, + { + "epoch": 0.20053715308863027, + "grad_norm": 1144.0, + "ht_mnpo/logit": 0.2897939682006836, + "ht_mnpo/residual": 0.2822939157485962, + "ht_mnpo/residual_abs": 0.28489789366722107, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.738416466110917e-07, + "logits/chosen": -1.7683238983154297, + "logits/rejected": -2.0142149925231934, + "logps/chosen": -0.30294543504714966, + "logps/rejected": -0.29282546043395996, + "loss": 0.8169538378715515, + "loss/core": 0.8169538378715515, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.211258888244629, + "rewards/margins": 2.8979392051696777, + "rewards/rejected": 1.3133190870285034, + "step": 210 + }, + { + "epoch": 0.20531184720978812, + "grad_norm": 2.859375, + "ht_mnpo/logit": 0.19056037068367004, + "ht_mnpo/residual": 0.18306037783622742, + "ht_mnpo/residual_abs": 0.1847141683101654, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.716406730462153e-07, + "logits/chosen": -2.104823589324951, + "logits/rejected": -2.3929922580718994, + "logps/chosen": -0.27915796637535095, + "logps/rejected": -0.28238147497177124, + "loss": 0.5005974769592285, + "loss/core": 0.5005974769592285, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 2.5159075260162354, + "rewards/margins": 1.9056037664413452, + "rewards/rejected": 0.610303521156311, + "step": 215 + }, + { + "epoch": 0.21008654133094598, + "grad_norm": 71.0, + "ht_mnpo/logit": 0.2221338003873825, + "ht_mnpo/residual": 0.21463382244110107, + "ht_mnpo/residual_abs": 0.21771471202373505, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.693563494886454e-07, + "logits/chosen": -1.793410301208496, + "logits/rejected": -2.138627767562866, + "logps/chosen": -0.2802343964576721, + "logps/rejected": -0.29542267322540283, + "loss": 0.4375545084476471, + "loss/core": 0.4375545084476471, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.305910587310791, + "rewards/margins": 2.2213380336761475, + "rewards/rejected": 1.0845725536346436, + "step": 220 + }, + { + "epoch": 0.21486123545210384, + "grad_norm": 23.5, + "ht_mnpo/logit": 0.20983533561229706, + "ht_mnpo/residual": 0.20233532786369324, + "ht_mnpo/residual_abs": 0.20403678715229034, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.6698953497905016e-07, + "logits/chosen": -2.118230104446411, + "logits/rejected": -2.3553214073181152, + "logps/chosen": -0.2552695572376251, + "logps/rejected": -0.27933183312416077, + "loss": 0.59180748462677, + "loss/core": 0.59180748462677, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.013653039932251, + "rewards/margins": 2.098353147506714, + "rewards/rejected": 0.9152997732162476, + "step": 225 + }, + { + "epoch": 0.21963592957326172, + "grad_norm": 4.125, + "ht_mnpo/logit": 0.13968196511268616, + "ht_mnpo/residual": 0.13218197226524353, + "ht_mnpo/residual_abs": 0.13623417913913727, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.645411195795709e-07, + "logits/chosen": -1.9094196557998657, + "logits/rejected": -2.1887524127960205, + "logps/chosen": -0.29535284638404846, + "logps/rejected": -0.30087095499038696, + "loss": 0.09830106049776077, + "loss/core": 0.09830106049776077, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.2886228561401367, + "rewards/margins": 1.3968197107315063, + "rewards/rejected": 0.8918031454086304, + "step": 230 + }, + { + "epoch": 0.22441062369441958, + "grad_norm": 10.5625, + "ht_mnpo/logit": 0.2398790419101715, + "ht_mnpo/residual": 0.2323790341615677, + "ht_mnpo/residual_abs": 0.23418621718883514, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.6201202403910643e-07, + "logits/chosen": -1.7939506769180298, + "logits/rejected": -2.1177468299865723, + "logps/chosen": -0.2927335500717163, + "logps/rejected": -0.2929953336715698, + "loss": 0.39657431840896606, + "loss/core": 0.39657431840896606, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 3.5993685722351074, + "rewards/margins": 2.3987903594970703, + "rewards/rejected": 1.2005784511566162, + "step": 235 + }, + { + "epoch": 0.22918531781557744, + "grad_norm": 0.52734375, + "ht_mnpo/logit": 0.23736481368541718, + "ht_mnpo/residual": 0.22986480593681335, + "ht_mnpo/residual_abs": 0.23360922932624817, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.594031994470573e-07, + "logits/chosen": -2.2269482612609863, + "logits/rejected": -2.406406879425049, + "logps/chosen": -0.30673980712890625, + "logps/rejected": -0.2728860080242157, + "loss": 0.7180436253547668, + "loss/core": 0.7180436253547668, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.5076498985290527, + "rewards/margins": 2.373648166656494, + "rewards/rejected": 1.1340019702911377, + "step": 240 + }, + { + "epoch": 0.2339600119367353, + "grad_norm": 14.9375, + "ht_mnpo/logit": 0.11371958255767822, + "ht_mnpo/residual": 0.1062195748090744, + "ht_mnpo/residual_abs": 0.1083589419722557, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.567156268756593e-07, + "logits/chosen": -2.077096700668335, + "logits/rejected": -2.2862114906311035, + "logps/chosen": -0.3035627007484436, + "logps/rejected": -0.3022594153881073, + "loss": 0.10577340424060822, + "loss/core": 0.10577340424060822, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.866857886314392, + "rewards/margins": 1.1371958255767822, + "rewards/rejected": 0.7296620607376099, + "step": 245 + }, + { + "epoch": 0.23873470605789315, + "grad_norm": 44.5, + "ht_mnpo/logit": 0.12031557410955429, + "ht_mnpo/residual": 0.11281557381153107, + "ht_mnpo/residual_abs": 0.1145239844918251, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.5395031701104303e-07, + "logits/chosen": -2.05730938911438, + "logits/rejected": -2.210094928741455, + "logps/chosen": -0.2798473536968231, + "logps/rejected": -0.292803019285202, + "loss": 0.09960635006427765, + "loss/core": 0.09960635006427765, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 1.7082903385162354, + "rewards/margins": 1.203155755996704, + "rewards/rejected": 0.505134642124176, + "step": 250 + }, + { + "epoch": 0.24350940017905104, + "grad_norm": 4.5625, + "ht_mnpo/logit": 0.2883337438106537, + "ht_mnpo/residual": 0.28083378076553345, + "ht_mnpo/residual_abs": 0.28313925862312317, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.511083097731555e-07, + "logits/chosen": -1.9648189544677734, + "logits/rejected": -2.2407779693603516, + "logps/chosen": -0.3144630789756775, + "logps/rejected": -0.3002500832080841, + "loss": 1.097538709640503, + "loss/core": 1.097538709640503, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.546959400177002, + "rewards/margins": 2.8833374977111816, + "rewards/rejected": 0.6636216640472412, + "step": 255 + }, + { + "epoch": 0.2482840943002089, + "grad_norm": 108.5, + "ht_mnpo/logit": 0.2983548939228058, + "ht_mnpo/residual": 0.29085490107536316, + "ht_mnpo/residual_abs": 0.2929629981517792, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.481906739246894e-07, + "logits/chosen": -1.9292614459991455, + "logits/rejected": -2.2683722972869873, + "logps/chosen": -0.2792973816394806, + "logps/rejected": -0.310756653547287, + "loss": 1.0083863735198975, + "loss/core": 1.0083863735198975, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.546018600463867, + "rewards/margins": 2.983549118041992, + "rewards/rejected": 0.5624698400497437, + "step": 260 + }, + { + "epoch": 0.25305878842136675, + "grad_norm": 29.375, + "ht_mnpo/logit": 0.22387555241584778, + "ht_mnpo/residual": 0.21637554466724396, + "ht_mnpo/residual_abs": 0.2206393927335739, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.451985066691648e-07, + "logits/chosen": -2.173464059829712, + "logits/rejected": -2.459009885787964, + "logps/chosen": -0.2751771807670593, + "logps/rejected": -0.2682764530181885, + "loss": 0.4876701831817627, + "loss/core": 0.4876701831817627, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.012019634246826, + "rewards/margins": 2.238755464553833, + "rewards/rejected": 0.7732640504837036, + "step": 265 + }, + { + "epoch": 0.25783348254252464, + "grad_norm": 888.0, + "ht_mnpo/logit": 0.4514991343021393, + "ht_mnpo/residual": 0.44399914145469666, + "ht_mnpo/residual_abs": 0.44612884521484375, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.421329332383158e-07, + "logits/chosen": -1.8616456985473633, + "logits/rejected": -2.184025764465332, + "logps/chosen": -0.2934468686580658, + "logps/rejected": -0.2970571517944336, + "loss": 2.1650872230529785, + "loss/core": 2.1650872230529785, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 5.505021572113037, + "rewards/margins": 4.514991283416748, + "rewards/rejected": 0.9900302886962891, + "step": 270 + }, + { + "epoch": 0.26260817666368247, + "grad_norm": 2.640625, + "ht_mnpo/logit": 0.14674557745456696, + "ht_mnpo/residual": 0.13924556970596313, + "ht_mnpo/residual_abs": 0.14186374843120575, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.3899510646893696e-07, + "logits/chosen": -2.0834450721740723, + "logits/rejected": -2.3231890201568604, + "logps/chosen": -0.27442002296447754, + "logps/rejected": -0.27742353081703186, + "loss": 0.23988012969493866, + "loss/core": 0.23988012969493866, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.230260133743286, + "rewards/margins": 1.4674557447433472, + "rewards/rejected": 0.7628042101860046, + "step": 275 + }, + { + "epoch": 0.26738287078484035, + "grad_norm": 8.1875, + "ht_mnpo/logit": 0.2650841772556305, + "ht_mnpo/residual": 0.2575841546058655, + "ht_mnpo/residual_abs": 0.2594790458679199, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.357862063693485e-07, + "logits/chosen": -2.0836455821990967, + "logits/rejected": -2.3042664527893066, + "logps/chosen": -0.32237258553504944, + "logps/rejected": -0.3145085573196411, + "loss": 1.0589872598648071, + "loss/core": 1.0589872598648071, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 3.507110118865967, + "rewards/margins": 2.650841474533081, + "rewards/rejected": 0.8562685251235962, + "step": 280 + }, + { + "epoch": 0.2721575649059982, + "grad_norm": 1.4765625, + "ht_mnpo/logit": 0.10140925645828247, + "ht_mnpo/residual": 0.09390926361083984, + "ht_mnpo/residual_abs": 0.09555725753307343, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.3250743967564364e-07, + "logits/chosen": -1.9129667282104492, + "logits/rejected": -2.1948471069335938, + "logps/chosen": -0.2781878709793091, + "logps/rejected": -0.2911220192909241, + "loss": 0.03205167502164841, + "loss/core": 0.03205167502164841, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 1.7170699834823608, + "rewards/margins": 1.0140926837921143, + "rewards/rejected": 0.7029772996902466, + "step": 285 + }, + { + "epoch": 0.27693225902715607, + "grad_norm": 3.875, + "ht_mnpo/logit": 0.30378761887550354, + "ht_mnpo/residual": 0.2962876260280609, + "ht_mnpo/residual_abs": 0.29867178201675415, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.29160039397884e-07, + "logits/chosen": -1.9081875085830688, + "logits/rejected": -2.3261563777923584, + "logps/chosen": -0.2825949788093567, + "logps/rejected": -0.2706306278705597, + "loss": 1.074646234512329, + "loss/core": 1.074646234512329, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.7699265480041504, + "rewards/margins": 3.0378763675689697, + "rewards/rejected": 0.7320500016212463, + "step": 290 + }, + { + "epoch": 0.28170695314831395, + "grad_norm": 1760.0, + "ht_mnpo/logit": 0.3823835849761963, + "ht_mnpo/residual": 0.37488359212875366, + "ht_mnpo/residual_abs": 0.3766990602016449, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.2574526435641546e-07, + "logits/chosen": -1.8789278268814087, + "logits/rejected": -2.110046625137329, + "logps/chosen": -0.28207749128341675, + "logps/rejected": -0.2834545373916626, + "loss": 1.690529465675354, + "loss/core": 1.690529465675354, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 4.7853851318359375, + "rewards/margins": 3.823835849761963, + "rewards/rejected": 0.961548924446106, + "step": 295 + }, + { + "epoch": 0.2864816472694718, + "grad_norm": 1.203125, + "ht_mnpo/logit": 0.1465030014514923, + "ht_mnpo/residual": 0.13900300860404968, + "ht_mnpo/residual_abs": 0.14254584908485413, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.22264398708477e-07, + "logits/chosen": -1.9352973699569702, + "logits/rejected": -2.1947338581085205, + "logps/chosen": -0.32196083664894104, + "logps/rejected": -0.3307940363883972, + "loss": 0.3272203505039215, + "loss/core": 0.3272203505039215, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.2242655754089355, + "rewards/margins": 1.4650301933288574, + "rewards/rejected": 0.7592355608940125, + "step": 300 + }, + { + "epoch": 0.29125634139062967, + "grad_norm": 99.5, + "ht_mnpo/logit": 0.13784225285053253, + "ht_mnpo/residual": 0.1303422749042511, + "ht_mnpo/residual_abs": 0.1337766945362091, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.187187514652819e-07, + "logits/chosen": -2.080651044845581, + "logits/rejected": -2.253840684890747, + "logps/chosen": -0.30505985021591187, + "logps/rejected": -0.32858914136886597, + "loss": 0.3612667918205261, + "loss/core": 0.3612667918205261, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.7895835638046265, + "rewards/margins": 1.378422498703003, + "rewards/rejected": 0.4111608862876892, + "step": 305 + }, + { + "epoch": 0.2960310355117875, + "grad_norm": 640.0, + "ht_mnpo/logit": 0.2148059904575348, + "ht_mnpo/residual": 0.20730595290660858, + "ht_mnpo/residual_abs": 0.21532678604125977, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.151096559997519e-07, + "logits/chosen": -2.0484871864318848, + "logits/rejected": -2.299739360809326, + "logps/chosen": -0.2571372389793396, + "logps/rejected": -0.2654029428958893, + "loss": 0.6419556736946106, + "loss/core": 0.6419556736946106, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.3131752014160156, + "rewards/margins": 2.148059368133545, + "rewards/rejected": 1.1651160717010498, + "step": 310 + }, + { + "epoch": 0.3008057296329454, + "grad_norm": 6.25, + "ht_mnpo/logit": 0.26450738310813904, + "ht_mnpo/residual": 0.2570074200630188, + "ht_mnpo/residual_abs": 0.2596099078655243, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.114384695450905e-07, + "logits/chosen": -1.8389809131622314, + "logits/rejected": -2.1140105724334717, + "logps/chosen": -0.289066880941391, + "logps/rejected": -0.2824665904045105, + "loss": 0.9795834422111511, + "loss/core": 0.9795834422111511, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.305142641067505, + "rewards/margins": 2.6450741291046143, + "rewards/rejected": 0.6600683331489563, + "step": 315 + }, + { + "epoch": 0.30558042375410327, + "grad_norm": 2.671875, + "ht_mnpo/logit": 0.22446127235889435, + "ht_mnpo/residual": 0.21696126461029053, + "ht_mnpo/residual_abs": 0.2195734977722168, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.077065726843828e-07, + "logits/chosen": -2.0431623458862305, + "logits/rejected": -2.303877830505371, + "logps/chosen": -0.2965102195739746, + "logps/rejected": -0.2953088879585266, + "loss": 0.659471333026886, + "loss/core": 0.659471333026886, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.1237876415252686, + "rewards/margins": 2.244612693786621, + "rewards/rejected": 0.8791753053665161, + "step": 320 + }, + { + "epoch": 0.3103551178752611, + "grad_norm": 528.0, + "ht_mnpo/logit": 0.4989927411079407, + "ht_mnpo/residual": 0.49149274826049805, + "ht_mnpo/residual_abs": 0.49372735619544983, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.039153688314145e-07, + "logits/chosen": -1.8436063528060913, + "logits/rejected": -2.1103482246398926, + "logps/chosen": -0.2776651084423065, + "logps/rejected": -0.2627367377281189, + "loss": 2.5948891639709473, + "loss/core": 2.5948891639709473, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 6.395821571350098, + "rewards/margins": 4.989927768707275, + "rewards/rejected": 1.4058938026428223, + "step": 325 + }, + { + "epoch": 0.315129811996419, + "grad_norm": 84.5, + "ht_mnpo/logit": 0.11850211769342422, + "ht_mnpo/residual": 0.1110021248459816, + "ht_mnpo/residual_abs": 0.11589847505092621, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.0006628370290613e-07, + "logits/chosen": -2.046609401702881, + "logits/rejected": -2.265373945236206, + "logps/chosen": -0.27779677510261536, + "logps/rejected": -0.27654415369033813, + "loss": 0.10753259807825089, + "loss/core": 0.10753259807825089, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.0452699661254883, + "rewards/margins": 1.185021162033081, + "rewards/rejected": 0.8602487444877625, + "step": 330 + }, + { + "epoch": 0.31990450611757687, + "grad_norm": 26.375, + "ht_mnpo/logit": 0.22499430179595947, + "ht_mnpo/residual": 0.21749427914619446, + "ht_mnpo/residual_abs": 0.21924762427806854, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.9616076478235826e-07, + "logits/chosen": -2.0356459617614746, + "logits/rejected": -2.332686424255371, + "logps/chosen": -0.26844361424446106, + "logps/rejected": -0.2597638666629791, + "loss": 0.8870676755905151, + "loss/core": 0.8870676755905151, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.095395088195801, + "rewards/margins": 2.2499425411224365, + "rewards/rejected": 0.8454524874687195, + "step": 335 + }, + { + "epoch": 0.3246792002387347, + "grad_norm": 5.34375, + "ht_mnpo/logit": 0.1895650327205658, + "ht_mnpo/residual": 0.18206503987312317, + "ht_mnpo/residual_abs": 0.18503603339195251, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.922002807757129e-07, + "logits/chosen": -1.8680092096328735, + "logits/rejected": -2.092318296432495, + "logps/chosen": -0.3079831302165985, + "logps/rejected": -0.28415459394454956, + "loss": 0.19596347212791443, + "loss/core": 0.19596347212791443, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.789630651473999, + "rewards/margins": 1.8956505060195923, + "rewards/rejected": 0.8939803242683411, + "step": 340 + }, + { + "epoch": 0.3294538943598926, + "grad_norm": 70.0, + "ht_mnpo/logit": 0.19111791253089905, + "ht_mnpo/residual": 0.18361787497997284, + "ht_mnpo/residual_abs": 0.1863330900669098, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.8818632105903315e-07, + "logits/chosen": -2.109628915786743, + "logits/rejected": -2.291822910308838, + "logps/chosen": -0.3013684153556824, + "logps/rejected": -0.2833828628063202, + "loss": 0.2935481667518616, + "loss/core": 0.2935481667518616, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.4318795204162598, + "rewards/margins": 1.9111789464950562, + "rewards/rejected": 1.5207008123397827, + "step": 345 + }, + { + "epoch": 0.3342285884810504, + "grad_norm": 21.375, + "ht_mnpo/logit": 0.17235636711120605, + "ht_mnpo/residual": 0.16485637426376343, + "ht_mnpo/residual_abs": 0.16569381952285767, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.841203951184094e-07, + "logits/chosen": -2.0808799266815186, + "logits/rejected": -2.3465583324432373, + "logps/chosen": -0.2933531403541565, + "logps/rejected": -0.29250794649124146, + "loss": 0.18648609519004822, + "loss/core": 0.18648609519004822, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.39286470413208, + "rewards/margins": 1.723563551902771, + "rewards/rejected": 0.6693009734153748, + "step": 350 + }, + { + "epoch": 0.3390032826022083, + "grad_norm": 8.8125, + "ht_mnpo/logit": 0.083734892308712, + "ht_mnpo/residual": 0.07623489201068878, + "ht_mnpo/residual_abs": 0.08302940428256989, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.800040319823038e-07, + "logits/chosen": -1.957427978515625, + "logits/rejected": -2.2546541690826416, + "logps/chosen": -0.2971518635749817, + "logps/rejected": -0.29797545075416565, + "loss": 0.02712792530655861, + "loss/core": 0.02712792530655861, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.441888451576233, + "rewards/margins": 0.8373489379882812, + "rewards/rejected": 0.6045395135879517, + "step": 355 + }, + { + "epoch": 0.3437779767233662, + "grad_norm": 12.0, + "ht_mnpo/logit": 0.13458336889743805, + "ht_mnpo/residual": 0.12708337604999542, + "ht_mnpo/residual_abs": 0.12896767258644104, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.75838779646545e-07, + "logits/chosen": -1.9576762914657593, + "logits/rejected": -2.2406909465789795, + "logps/chosen": -0.3130984306335449, + "logps/rejected": -0.3136448860168457, + "loss": 0.10641677677631378, + "loss/core": 0.10641677677631378, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.3418941497802734, + "rewards/margins": 1.3458335399627686, + "rewards/rejected": 0.9960603713989258, + "step": 360 + }, + { + "epoch": 0.348552670844524, + "grad_norm": 37.25, + "ht_mnpo/logit": 0.25376901030540466, + "ht_mnpo/residual": 0.24626903235912323, + "ht_mnpo/residual_abs": 0.24840883910655975, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.7162620449218993e-07, + "logits/chosen": -1.9573538303375244, + "logits/rejected": -2.191314458847046, + "logps/chosen": -0.28709596395492554, + "logps/rejected": -0.28582268953323364, + "loss": 0.5417178273200989, + "loss/core": 0.5417178273200989, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.668320894241333, + "rewards/margins": 2.5376901626586914, + "rewards/rejected": 1.1306307315826416, + "step": 365 + }, + { + "epoch": 0.3533273649656819, + "grad_norm": 23.5, + "ht_mnpo/logit": 0.1474151760339737, + "ht_mnpo/residual": 0.13991518318653107, + "ht_mnpo/residual_abs": 0.14576002955436707, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.673678906964727e-07, + "logits/chosen": -2.0737175941467285, + "logits/rejected": -2.3785977363586426, + "logps/chosen": -0.303182452917099, + "logps/rejected": -0.3058184087276459, + "loss": 0.16569259762763977, + "loss/core": 0.16569259762763977, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.3934152126312256, + "rewards/margins": 1.474151611328125, + "rewards/rejected": 0.9192636609077454, + "step": 370 + }, + { + "epoch": 0.35810205908683973, + "grad_norm": 19.0, + "ht_mnpo/logit": 0.2867982089519501, + "ht_mnpo/residual": 0.27929818630218506, + "ht_mnpo/residual_abs": 0.2815331816673279, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.6306543963705936e-07, + "logits/chosen": -1.7822672128677368, + "logits/rejected": -2.033956289291382, + "logps/chosen": -0.2669016718864441, + "logps/rejected": -0.2721244990825653, + "loss": 0.7982349991798401, + "loss/core": 0.7982349991798401, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 3.7743923664093018, + "rewards/margins": 2.8679816722869873, + "rewards/rejected": 0.9064106941223145, + "step": 375 + }, + { + "epoch": 0.3628767532079976, + "grad_norm": 12.4375, + "ht_mnpo/logit": 0.22812096774578094, + "ht_mnpo/residual": 0.22062095999717712, + "ht_mnpo/residual_abs": 0.22334913909435272, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.5872046928983623e-07, + "logits/chosen": -1.9232280254364014, + "logits/rejected": -2.178192615509033, + "logps/chosen": -0.29933086037635803, + "logps/rejected": -0.28768283128738403, + "loss": 0.6772836446762085, + "loss/core": 0.6772836446762085, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.9951553344726562, + "rewards/margins": 2.281209945678711, + "rewards/rejected": 0.713945746421814, + "step": 380 + }, + { + "epoch": 0.3676514473291555, + "grad_norm": 3.25, + "ht_mnpo/logit": 0.21591122448444366, + "ht_mnpo/residual": 0.20841124653816223, + "ht_mnpo/residual_abs": 0.21050401031970978, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.5433461362045447e-07, + "logits/chosen": -1.847633719444275, + "logits/rejected": -2.065316677093506, + "logps/chosen": -0.28148138523101807, + "logps/rejected": -0.2986345589160919, + "loss": 0.47226905822753906, + "loss/core": 0.47226905822753906, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.5782718658447266, + "rewards/margins": 2.1591124534606934, + "rewards/rejected": 0.4191594123840332, + "step": 385 + }, + { + "epoch": 0.37242614145031333, + "grad_norm": 404.0, + "ht_mnpo/logit": 0.4722256064414978, + "ht_mnpo/residual": 0.46472564339637756, + "ht_mnpo/residual_abs": 0.46695786714553833, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.4990952196986305e-07, + "logits/chosen": -1.9401696920394897, + "logits/rejected": -2.1430230140686035, + "logps/chosen": -0.3209291100502014, + "logps/rejected": -0.29362010955810547, + "loss": 2.3541719913482666, + "loss/core": 2.3541719913482666, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 5.9721999168396, + "rewards/margins": 4.722256183624268, + "rewards/rejected": 1.2499440908432007, + "step": 390 + }, + { + "epoch": 0.3772008355714712, + "grad_norm": 28.75, + "ht_mnpo/logit": 0.2606656849384308, + "ht_mnpo/residual": 0.25316569209098816, + "ht_mnpo/residual_abs": 0.2560683488845825, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.4544685843405875e-07, + "logits/chosen": -1.8940608501434326, + "logits/rejected": -2.079465389251709, + "logps/chosen": -0.2855537533760071, + "logps/rejected": -0.2773713767528534, + "loss": 0.5645236372947693, + "loss/core": 0.5645236372947693, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.6368789672851562, + "rewards/margins": 2.606656789779663, + "rewards/rejected": 1.030221700668335, + "step": 395 + }, + { + "epoch": 0.38197552969262905, + "grad_norm": 187.0, + "ht_mnpo/logit": 0.26339191198349, + "ht_mnpo/residual": 0.25589194893836975, + "ht_mnpo/residual_abs": 0.2572072148323059, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.4094830123828786e-07, + "logits/chosen": -2.0253453254699707, + "logits/rejected": -2.415069818496704, + "logps/chosen": -0.2948891520500183, + "logps/rejected": -0.28987759351730347, + "loss": 0.494838148355484, + "loss/core": 0.494838148355484, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.4587039947509766, + "rewards/margins": 2.6339192390441895, + "rewards/rejected": 0.824785053730011, + "step": 400 + }, + { + "epoch": 0.38675022381378693, + "grad_norm": 796.0, + "ht_mnpo/logit": 0.519372820854187, + "ht_mnpo/residual": 0.5118728876113892, + "ht_mnpo/residual_abs": 0.5148923993110657, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.3641554210593414e-07, + "logits/chosen": -1.5729597806930542, + "logits/rejected": -1.9059674739837646, + "logps/chosen": -0.324828565120697, + "logps/rejected": -0.3063935339450836, + "loss": 2.9073703289031982, + "loss/core": 2.9073703289031982, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 6.127835750579834, + "rewards/margins": 5.193729400634766, + "rewards/rejected": 0.9341069459915161, + "step": 405 + }, + { + "epoch": 0.3915249179349448, + "grad_norm": 428.0, + "ht_mnpo/logit": 0.33313632011413574, + "ht_mnpo/residual": 0.3256363272666931, + "ht_mnpo/residual_abs": 0.3284788727760315, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.3185028562233107e-07, + "logits/chosen": -1.9473098516464233, + "logits/rejected": -2.265756130218506, + "logps/chosen": -0.28903454542160034, + "logps/rejected": -0.29171279072761536, + "loss": 1.68734610080719, + "loss/core": 1.68734610080719, + "rewards/accuracies": 0.9375, + "rewards/chosen": 4.016785621643066, + "rewards/margins": 3.3313636779785156, + "rewards/rejected": 0.6854226589202881, + "step": 410 + }, + { + "epoch": 0.39629961205610265, + "grad_norm": 4.96875, + "ht_mnpo/logit": 0.14160475134849548, + "ht_mnpo/residual": 0.13410475850105286, + "ht_mnpo/residual_abs": 0.1367226243019104, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.272542485937368e-07, + "logits/chosen": -2.0103304386138916, + "logits/rejected": -2.196492910385132, + "logps/chosen": -0.2817983031272888, + "logps/rejected": -0.30435535311698914, + "loss": 0.11378359794616699, + "loss/core": 0.11378359794616699, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.1162495613098145, + "rewards/margins": 1.41604745388031, + "rewards/rejected": 0.7002020478248596, + "step": 415 + }, + { + "epoch": 0.40107430617726053, + "grad_norm": 792.0, + "ht_mnpo/logit": 0.2766280174255371, + "ht_mnpo/residual": 0.2691280245780945, + "ht_mnpo/residual_abs": 0.27129045128822327, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.226291594017137e-07, + "logits/chosen": -1.7459360361099243, + "logits/rejected": -1.9585685729980469, + "logps/chosen": -0.2827063500881195, + "logps/rejected": -0.2677227854728699, + "loss": 0.8373321294784546, + "loss/core": 0.8373321294784546, + "rewards/accuracies": 0.9375, + "rewards/chosen": 4.167675495147705, + "rewards/margins": 2.766279935836792, + "rewards/rejected": 1.401395320892334, + "step": 420 + }, + { + "epoch": 0.40584900029841836, + "grad_norm": 676.0, + "ht_mnpo/logit": 0.22410964965820312, + "ht_mnpo/residual": 0.2166096419095993, + "ht_mnpo/residual_abs": 0.22375044226646423, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.1797675735315454e-07, + "logits/chosen": -1.9364029169082642, + "logits/rejected": -2.2243618965148926, + "logps/chosen": -0.29613256454467773, + "logps/rejected": -0.2813289761543274, + "loss": 0.44919532537460327, + "loss/core": 0.44919532537460327, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.634718418121338, + "rewards/margins": 2.2410967350006104, + "rewards/rejected": 1.3936221599578857, + "step": 425 + }, + { + "epoch": 0.41062369441957625, + "grad_norm": 2.953125, + "ht_mnpo/logit": 0.11446923017501831, + "ht_mnpo/residual": 0.10696922242641449, + "ht_mnpo/residual_abs": 0.10941126197576523, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.1329879202620047e-07, + "logits/chosen": -2.0317301750183105, + "logits/rejected": -2.2824201583862305, + "logps/chosen": -0.28097397089004517, + "logps/rejected": -0.2934049665927887, + "loss": 0.04260837286710739, + "loss/core": 0.04260837286710739, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.770204782485962, + "rewards/margins": 1.1446921825408936, + "rewards/rejected": 0.6255124807357788, + "step": 430 + }, + { + "epoch": 0.41539838854073413, + "grad_norm": 6.75, + "ht_mnpo/logit": 0.17668896913528442, + "ht_mnpo/residual": 0.1691889464855194, + "ht_mnpo/residual_abs": 0.17058250308036804, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.0859702261229613e-07, + "logits/chosen": -2.0280332565307617, + "logits/rejected": -2.2525267601013184, + "logps/chosen": -0.2677651345729828, + "logps/rejected": -0.2582930326461792, + "loss": 0.2649860978126526, + "loss/core": 0.2649860978126526, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 2.587427854537964, + "rewards/margins": 1.7668898105621338, + "rewards/rejected": 0.8205384016036987, + "step": 435 + }, + { + "epoch": 0.42017308266189196, + "grad_norm": 18.25, + "ht_mnpo/logit": 0.146823450922966, + "ht_mnpo/residual": 0.13932344317436218, + "ht_mnpo/residual_abs": 0.1414867490530014, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.0387321725463e-07, + "logits/chosen": -1.963629961013794, + "logits/rejected": -2.183755397796631, + "logps/chosen": -0.2657316327095032, + "logps/rejected": -0.2711542248725891, + "loss": 0.07426687330007553, + "loss/core": 0.07426687330007553, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.3465564250946045, + "rewards/margins": 1.4682343006134033, + "rewards/rejected": 0.878322422504425, + "step": 440 + }, + { + "epoch": 0.42494777678304985, + "grad_norm": 3.625, + "ht_mnpo/logit": 0.2173123061656952, + "ht_mnpo/residual": 0.20981231331825256, + "ht_mnpo/residual_abs": 0.21300652623176575, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.991291523832075e-07, + "logits/chosen": -1.9978408813476562, + "logits/rejected": -2.254274606704712, + "logps/chosen": -0.3106047511100769, + "logps/rejected": -0.28541427850723267, + "loss": 1.295969843864441, + "loss/core": 1.295969843864441, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.9272522926330566, + "rewards/margins": 2.1731228828430176, + "rewards/rejected": 0.7541292905807495, + "step": 445 + }, + { + "epoch": 0.4297224709042077, + "grad_norm": 1128.0, + "ht_mnpo/logit": 0.2916508913040161, + "ht_mnpo/residual": 0.2841508984565735, + "ht_mnpo/residual_abs": 0.28610268235206604, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.943666120468088e-07, + "logits/chosen": -1.9271923303604126, + "logits/rejected": -2.2027246952056885, + "logps/chosen": -0.3097872734069824, + "logps/rejected": -0.28542861342430115, + "loss": 0.8190312385559082, + "loss/core": 0.8190312385559082, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.744999408721924, + "rewards/margins": 2.9165091514587402, + "rewards/rejected": 0.8284900784492493, + "step": 450 + }, + { + "epoch": 0.43449716502536556, + "grad_norm": 6.40625, + "ht_mnpo/logit": 0.21481645107269287, + "ht_mnpo/residual": 0.20731644332408905, + "ht_mnpo/residual_abs": 0.2103075236082077, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.8958738724208073e-07, + "logits/chosen": -1.8237874507904053, + "logits/rejected": -2.129227638244629, + "logps/chosen": -0.30449140071868896, + "logps/rejected": -0.27840521931648254, + "loss": 0.42009902000427246, + "loss/core": 0.42009902000427246, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 3.241947889328003, + "rewards/margins": 2.1481645107269287, + "rewards/rejected": 1.0937836170196533, + "step": 455 + }, + { + "epoch": 0.43927185914652345, + "grad_norm": 167.0, + "ht_mnpo/logit": 0.29557928442955017, + "ht_mnpo/residual": 0.28807929158210754, + "ht_mnpo/residual_abs": 0.2902158796787262, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.8479327524001633e-07, + "logits/chosen": -1.9467674493789673, + "logits/rejected": -2.1150288581848145, + "logps/chosen": -0.25731417536735535, + "logps/rejected": -0.2749153673648834, + "loss": 1.0729789733886719, + "loss/core": 1.0729789733886719, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.7630081176757812, + "rewards/margins": 2.9557926654815674, + "rewards/rejected": 0.8072150945663452, + "step": 460 + }, + { + "epoch": 0.4440465532676813, + "grad_norm": 5.28125, + "ht_mnpo/logit": 0.1232837587594986, + "ht_mnpo/residual": 0.11578376591205597, + "ht_mnpo/residual_abs": 0.11795320361852646, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.7998607891007493e-07, + "logits/chosen": -1.9201545715332031, + "logits/rejected": -2.2683331966400146, + "logps/chosen": -0.3000238239765167, + "logps/rejected": -0.3087528347969055, + "loss": 0.0906161218881607, + "loss/core": 0.0906161218881607, + "rewards/accuracies": 0.9375, + "rewards/chosen": 1.7928030490875244, + "rewards/margins": 1.2328376770019531, + "rewards/rejected": 0.5599652528762817, + "step": 465 + }, + { + "epoch": 0.44882124738883916, + "grad_norm": 55.5, + "ht_mnpo/logit": 0.3084077537059784, + "ht_mnpo/residual": 0.3009077310562134, + "ht_mnpo/residual_abs": 0.3186038136482239, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.7516760604219616e-07, + "logits/chosen": -1.9410159587860107, + "logits/rejected": -2.1513659954071045, + "logps/chosen": -0.3083992600440979, + "logps/rejected": -0.3033507764339447, + "loss": 1.498625636100769, + "loss/core": 1.498625636100769, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 4.285151481628418, + "rewards/margins": 3.084077835083008, + "rewards/rejected": 1.2010738849639893, + "step": 470 + }, + { + "epoch": 0.453595941509997, + "grad_norm": 352.0, + "ht_mnpo/logit": 0.36416906118392944, + "ht_mnpo/residual": 0.35666903853416443, + "ht_mnpo/residual_abs": 0.3577561676502228, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.703396686669646e-07, + "logits/chosen": -2.192099094390869, + "logits/rejected": -2.485234498977661, + "logps/chosen": -0.29125604033470154, + "logps/rejected": -0.3030247092247009, + "loss": 1.63297438621521, + "loss/core": 1.63297438621521, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 4.850213050842285, + "rewards/margins": 3.641690492630005, + "rewards/rejected": 1.2085225582122803, + "step": 475 + }, + { + "epoch": 0.4583706356311549, + "grad_norm": 15.0, + "ht_mnpo/logit": 0.3097284436225891, + "ht_mnpo/residual": 0.30222851037979126, + "ht_mnpo/residual_abs": 0.3041606545448303, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.6550408237417884e-07, + "logits/chosen": -1.9110229015350342, + "logits/rejected": -2.150383472442627, + "logps/chosen": -0.29265767335891724, + "logps/rejected": -0.28647303581237793, + "loss": 0.8241177797317505, + "loss/core": 0.8241177797317505, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 4.338518142700195, + "rewards/margins": 3.0972847938537598, + "rewards/rejected": 1.241233229637146, + "step": 480 + }, + { + "epoch": 0.46314532975231276, + "grad_norm": 238.0, + "ht_mnpo/logit": 0.3325381875038147, + "ht_mnpo/residual": 0.32503825426101685, + "ht_mnpo/residual_abs": 0.3343810439109802, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.6066266563008265e-07, + "logits/chosen": -1.8738826513290405, + "logits/rejected": -2.16226863861084, + "logps/chosen": -0.3215123116970062, + "logps/rejected": -0.3182523846626282, + "loss": 1.0384169816970825, + "loss/core": 1.0384169816970825, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 4.18947696685791, + "rewards/margins": 3.3253822326660156, + "rewards/rejected": 0.8640945553779602, + "step": 485 + }, + { + "epoch": 0.4679200238734706, + "grad_norm": 12.5, + "ht_mnpo/logit": 0.14976148307323456, + "ht_mnpo/residual": 0.14226147532463074, + "ht_mnpo/residual_abs": 0.14404740929603577, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.5581723909351404e-07, + "logits/chosen": -2.215057849884033, + "logits/rejected": -2.4560017585754395, + "logps/chosen": -0.2996280789375305, + "logps/rejected": -0.30207791924476624, + "loss": 0.15293726325035095, + "loss/core": 0.15293726325035095, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.62994384765625, + "rewards/margins": 1.497614860534668, + "rewards/rejected": 1.1323288679122925, + "step": 490 + }, + { + "epoch": 0.4726947179946285, + "grad_norm": 1.109375, + "ht_mnpo/logit": 0.1131284236907959, + "ht_mnpo/residual": 0.10562840849161148, + "ht_mnpo/residual_abs": 0.10809159278869629, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.509696249312301e-07, + "logits/chosen": -2.2300727367401123, + "logits/rejected": -2.570033550262451, + "logps/chosen": -0.2869029641151428, + "logps/rejected": -0.2894020676612854, + "loss": 0.0645766630768776, + "loss/core": 0.0645766630768776, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.0836386680603027, + "rewards/margins": 1.1312841176986694, + "rewards/rejected": 0.9523545503616333, + "step": 495 + }, + { + "epoch": 0.4774694121157863, + "grad_norm": 384.0, + "ht_mnpo/logit": 0.1857949197292328, + "ht_mnpo/residual": 0.17829494178295135, + "ht_mnpo/residual_abs": 0.1815871000289917, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.461216461326642e-07, + "logits/chosen": -1.7350391149520874, + "logits/rejected": -2.03250789642334, + "logps/chosen": -0.30827873945236206, + "logps/rejected": -0.30130141973495483, + "loss": 0.2996276617050171, + "loss/core": 0.2996276617050171, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.6126999855041504, + "rewards/margins": 1.8579492568969727, + "rewards/rejected": 0.7547506093978882, + "step": 500 + }, + { + "epoch": 0.4822441062369442, + "grad_norm": 11.75, + "ht_mnpo/logit": 0.196578711271286, + "ht_mnpo/residual": 0.18907873332500458, + "ht_mnpo/residual_abs": 0.19180910289287567, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.412751258243748e-07, + "logits/chosen": -1.9780470132827759, + "logits/rejected": -2.1910805702209473, + "logps/chosen": -0.31306812167167664, + "logps/rejected": -0.31634530425071716, + "loss": 0.27675363421440125, + "loss/core": 0.27675363421440125, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.940455913543701, + "rewards/margins": 1.9657869338989258, + "rewards/rejected": 0.9746688008308411, + "step": 505 + }, + { + "epoch": 0.4870188003581021, + "grad_norm": 11.6875, + "ht_mnpo/logit": 0.29334843158721924, + "ht_mnpo/residual": 0.2858484089374542, + "ht_mnpo/residual_abs": 0.28805553913116455, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.3643188658444158e-07, + "logits/chosen": -1.839543104171753, + "logits/rejected": -2.139441728591919, + "logps/chosen": -0.2953009009361267, + "logps/rejected": -0.27880555391311646, + "loss": 0.7938116788864136, + "loss/core": 0.7938116788864136, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.985734462738037, + "rewards/margins": 2.9334845542907715, + "rewards/rejected": 1.0522500276565552, + "step": 510 + }, + { + "epoch": 0.4917934944792599, + "grad_norm": 2.59375, + "ht_mnpo/logit": 0.11522988229990005, + "ht_mnpo/residual": 0.10772988945245743, + "ht_mnpo/residual_abs": 0.11010618507862091, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.315937497570688e-07, + "logits/chosen": -1.898514747619629, + "logits/rejected": -2.2502241134643555, + "logps/chosen": -0.2737577557563782, + "logps/rejected": -0.2691265940666199, + "loss": 0.06796002388000488, + "loss/core": 0.06796002388000488, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.908595323562622, + "rewards/margins": 1.1522986888885498, + "rewards/rejected": 0.7562967538833618, + "step": 515 + }, + { + "epoch": 0.4965681886004178, + "grad_norm": 2.53125, + "ht_mnpo/logit": 0.13243892788887024, + "ht_mnpo/residual": 0.12493894249200821, + "ht_mnpo/residual_abs": 0.13076035678386688, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.2676253476765194e-07, + "logits/chosen": -1.9941246509552002, + "logits/rejected": -2.2544379234313965, + "logps/chosen": -0.2522723078727722, + "logps/rejected": -0.2937101721763611, + "loss": 0.08826544135808945, + "loss/core": 0.08826544135808945, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.3944621086120605, + "rewards/margins": 1.3243893384933472, + "rewards/rejected": 1.0700726509094238, + "step": 520 + }, + { + "epoch": 0.5013428827215757, + "grad_norm": 5.6875, + "ht_mnpo/logit": 0.12501785159111023, + "ht_mnpo/residual": 0.117517851293087, + "ht_mnpo/residual_abs": 0.11929130554199219, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.2194005843856633e-07, + "logits/chosen": -2.2246017456054688, + "logits/rejected": -2.410853624343872, + "logps/chosen": -0.28422626852989197, + "logps/rejected": -0.28308025002479553, + "loss": 0.10617043823003769, + "loss/core": 0.10617043823003769, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.355593681335449, + "rewards/margins": 1.2501784563064575, + "rewards/rejected": 1.1054149866104126, + "step": 525 + }, + { + "epoch": 0.5061175768427335, + "grad_norm": 34.25, + "ht_mnpo/logit": 0.17451244592666626, + "ht_mnpo/residual": 0.16701245307922363, + "ht_mnpo/residual_abs": 0.16843153536319733, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.1712813430593434e-07, + "logits/chosen": -2.0963895320892334, + "logits/rejected": -2.3358659744262695, + "logps/chosen": -0.28125277161598206, + "logps/rejected": -0.2825583219528198, + "loss": 0.38109883666038513, + "loss/core": 0.38109883666038513, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.497371196746826, + "rewards/margins": 1.7451244592666626, + "rewards/rejected": 0.752246618270874, + "step": 530 + }, + { + "epoch": 0.5108922709638913, + "grad_norm": 68.5, + "ht_mnpo/logit": 0.2884078621864319, + "ht_mnpo/residual": 0.28090786933898926, + "ht_mnpo/residual_abs": 0.2828427851200104, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.123285719376292e-07, + "logits/chosen": -1.8652563095092773, + "logits/rejected": -2.1189205646514893, + "logps/chosen": -0.3185581564903259, + "logps/rejected": -0.3078560531139374, + "loss": 0.5867989659309387, + "loss/core": 0.5867989659309387, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.6412525177001953, + "rewards/margins": 2.8840785026550293, + "rewards/rejected": 0.7571739554405212, + "step": 535 + }, + { + "epoch": 0.5156669650850493, + "grad_norm": 6.09375, + "ht_mnpo/logit": 0.10569143295288086, + "ht_mnpo/residual": 0.09819144755601883, + "ht_mnpo/residual_abs": 0.10155366361141205, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.0754317625276982e-07, + "logits/chosen": -2.0045483112335205, + "logits/rejected": -2.270784854888916, + "logps/chosen": -0.31180429458618164, + "logps/rejected": -0.3156093955039978, + "loss": 0.04411657899618149, + "loss/core": 0.04411657899618149, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.5413364171981812, + "rewards/margins": 1.0569144487380981, + "rewards/rejected": 0.48442205786705017, + "step": 540 + }, + { + "epoch": 0.5204416592062071, + "grad_norm": 580.0, + "ht_mnpo/logit": 0.19196265935897827, + "ht_mnpo/residual": 0.18446265161037445, + "ht_mnpo/residual_abs": 0.186975359916687, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.0277374684296498e-07, + "logits/chosen": -2.0267083644866943, + "logits/rejected": -2.356262683868408, + "logps/chosen": -0.29003921151161194, + "logps/rejected": -0.3435795307159424, + "loss": 0.3793261647224426, + "loss/core": 0.3793261647224426, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.015592575073242, + "rewards/margins": 1.9196265935897827, + "rewards/rejected": 1.095966100692749, + "step": 545 + }, + { + "epoch": 0.5252163533273649, + "grad_norm": 7.8125, + "ht_mnpo/logit": 0.23898443579673767, + "ht_mnpo/residual": 0.23148444294929504, + "ht_mnpo/residual_abs": 0.2341163158416748, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.980220772955602e-07, + "logits/chosen": -1.8253529071807861, + "logits/rejected": -2.06758713722229, + "logps/chosen": -0.27169501781463623, + "logps/rejected": -0.2784484326839447, + "loss": 0.5674008131027222, + "loss/core": 0.5674008131027222, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.5318641662597656, + "rewards/margins": 2.3898441791534424, + "rewards/rejected": 1.142019510269165, + "step": 550 + }, + { + "epoch": 0.5299910474485229, + "grad_norm": 2.96875, + "ht_mnpo/logit": 0.19051536917686462, + "ht_mnpo/residual": 0.183015376329422, + "ht_mnpo/residual_abs": 0.18519142270088196, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.932899545191433e-07, + "logits/chosen": -2.0828499794006348, + "logits/rejected": -2.2646288871765137, + "logps/chosen": -0.31175822019577026, + "logps/rejected": -0.28370457887649536, + "loss": 0.45988917350769043, + "loss/core": 0.45988917350769043, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.769913911819458, + "rewards/margins": 1.9051536321640015, + "rewards/rejected": 0.8647602200508118, + "step": 555 + }, + { + "epoch": 0.5347657415696807, + "grad_norm": 2.28125, + "ht_mnpo/logit": 0.17307063937187195, + "ht_mnpo/residual": 0.16557064652442932, + "ht_mnpo/residual_abs": 0.16744212806224823, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.885791580715609e-07, + "logits/chosen": -1.9716126918792725, + "logits/rejected": -2.2994656562805176, + "logps/chosen": -0.301216185092926, + "logps/rejected": -0.27583974599838257, + "loss": 0.45512303709983826, + "loss/core": 0.45512303709983826, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.420431137084961, + "rewards/margins": 1.7307065725326538, + "rewards/rejected": 0.6897248029708862, + "step": 560 + }, + { + "epoch": 0.5395404356908385, + "grad_norm": 4.4375, + "ht_mnpo/logit": 0.34825602173805237, + "ht_mnpo/residual": 0.34075599908828735, + "ht_mnpo/residual_abs": 0.34296515583992004, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8389145949069951e-07, + "logits/chosen": -2.1434173583984375, + "logits/rejected": -2.416426420211792, + "logps/chosen": -0.3156982362270355, + "logps/rejected": -0.2996530830860138, + "loss": 2.206251382827759, + "loss/core": 2.206251382827759, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 4.403435707092285, + "rewards/margins": 3.4825599193573, + "rewards/rejected": 0.9208753705024719, + "step": 565 + }, + { + "epoch": 0.5443151298119964, + "grad_norm": 60.0, + "ht_mnpo/logit": 0.24878111481666565, + "ht_mnpo/residual": 0.24128107726573944, + "ht_mnpo/residual_abs": 0.246558278799057, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.792286216282824e-07, + "logits/chosen": -1.9773708581924438, + "logits/rejected": -2.3607583045959473, + "logps/chosen": -0.28530240058898926, + "logps/rejected": -0.27613160014152527, + "loss": 0.4744301736354828, + "loss/core": 0.4744301736354828, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.3544468879699707, + "rewards/margins": 2.487811326980591, + "rewards/rejected": 0.8666356801986694, + "step": 570 + }, + { + "epoch": 0.5490898239331543, + "grad_norm": 2.078125, + "ht_mnpo/logit": 0.31033894419670105, + "ht_mnpo/residual": 0.3028389513492584, + "ht_mnpo/residual_abs": 0.30536240339279175, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.745923979869336e-07, + "logits/chosen": -2.0291919708251953, + "logits/rejected": -2.281064510345459, + "logps/chosen": -0.32846799492836, + "logps/rejected": -0.3118685483932495, + "loss": 1.0607415437698364, + "loss/core": 1.0607415437698364, + "rewards/accuracies": 0.9375, + "rewards/chosen": 4.091516971588135, + "rewards/margins": 3.1033897399902344, + "rewards/rejected": 0.9881270527839661, + "step": 575 + }, + { + "epoch": 0.5538645180543121, + "grad_norm": 3.6875, + "ht_mnpo/logit": 0.4819414019584656, + "ht_mnpo/residual": 0.47444137930870056, + "ht_mnpo/residual_abs": 0.4764309823513031, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.6998453206075708e-07, + "logits/chosen": -1.7918148040771484, + "logits/rejected": -1.9814106225967407, + "logps/chosen": -0.30701881647109985, + "logps/rejected": -0.25892844796180725, + "loss": 2.3531346321105957, + "loss/core": 2.3531346321105957, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 6.004878520965576, + "rewards/margins": 4.819413661956787, + "rewards/rejected": 1.18546462059021, + "step": 580 + }, + { + "epoch": 0.55863921217547, + "grad_norm": 50.75, + "ht_mnpo/logit": 0.156865194439888, + "ht_mnpo/residual": 0.14936518669128418, + "ht_mnpo/residual_abs": 0.15104547142982483, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.6540675667967973e-07, + "logits/chosen": -2.02787709236145, + "logits/rejected": -2.223013401031494, + "logps/chosen": -0.29150041937828064, + "logps/rejected": -0.2901824414730072, + "loss": 0.09419430792331696, + "loss/core": 0.09419430792331696, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.5935516357421875, + "rewards/margins": 1.5686519145965576, + "rewards/rejected": 1.0248996019363403, + "step": 585 + }, + { + "epoch": 0.5634139062966279, + "grad_norm": 888.0, + "ht_mnpo/logit": 0.2369958609342575, + "ht_mnpo/residual": 0.2294958084821701, + "ht_mnpo/residual_abs": 0.2326071560382843, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.60860793357805e-07, + "logits/chosen": -1.8807750940322876, + "logits/rejected": -2.03248929977417, + "logps/chosen": -0.29985108971595764, + "logps/rejected": -0.3031771183013916, + "loss": 0.7982430458068848, + "loss/core": 0.7982430458068848, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 3.11862850189209, + "rewards/margins": 2.3699584007263184, + "rewards/rejected": 0.7486701011657715, + "step": 590 + }, + { + "epoch": 0.5681886004177857, + "grad_norm": 2.515625, + "ht_mnpo/logit": 0.4987797141075134, + "ht_mnpo/residual": 0.4912796914577484, + "ht_mnpo/residual_abs": 0.4936184287071228, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.5634835164602196e-07, + "logits/chosen": -1.8091611862182617, + "logits/rejected": -2.182551622390747, + "logps/chosen": -0.3149603307247162, + "logps/rejected": -0.2931450605392456, + "loss": 2.4295339584350586, + "loss/core": 2.4295339584350586, + "rewards/accuracies": 0.9375, + "rewards/chosen": 5.907982349395752, + "rewards/margins": 4.987797260284424, + "rewards/rejected": 0.9201849102973938, + "step": 595 + }, + { + "epoch": 0.5729632945389436, + "grad_norm": 2.09375, + "ht_mnpo/logit": 0.24387526512145996, + "ht_mnpo/residual": 0.23637528717517853, + "ht_mnpo/residual_abs": 0.23978765308856964, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.518711284891132e-07, + "logits/chosen": -1.8736908435821533, + "logits/rejected": -2.1749606132507324, + "logps/chosen": -0.29759305715560913, + "logps/rejected": -0.2989031970500946, + "loss": 0.559294581413269, + "loss/core": 0.559294581413269, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 4.058932304382324, + "rewards/margins": 2.4387524127960205, + "rewards/rejected": 1.6201797723770142, + "step": 600 + }, + { + "epoch": 0.5777379886601015, + "grad_norm": 16.375, + "ht_mnpo/logit": 0.27251681685447693, + "ht_mnpo/residual": 0.2650168240070343, + "ht_mnpo/residual_abs": 0.2667366862297058, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.47430807587603e-07, + "logits/chosen": -1.8784462213516235, + "logits/rejected": -2.1159400939941406, + "logps/chosen": -0.29141777753829956, + "logps/rejected": -0.3019111752510071, + "loss": 0.662787914276123, + "loss/core": 0.662787914276123, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.4891610145568848, + "rewards/margins": 2.725167989730835, + "rewards/rejected": 0.7639929056167603, + "step": 605 + }, + { + "epoch": 0.5825126827812593, + "grad_norm": 3.5625, + "ht_mnpo/logit": 0.23614993691444397, + "ht_mnpo/residual": 0.22864992916584015, + "ht_mnpo/residual_abs": 0.23106643557548523, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.430290587645865e-07, + "logits/chosen": -1.9317668676376343, + "logits/rejected": -2.19699764251709, + "logps/chosen": -0.301189124584198, + "logps/rejected": -0.2828231751918793, + "loss": 0.7685016393661499, + "loss/core": 0.7685016393661499, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.8618009090423584, + "rewards/margins": 2.361499309539795, + "rewards/rejected": 0.5003016591072083, + "step": 610 + }, + { + "epoch": 0.5872873769024172, + "grad_norm": 11.1875, + "ht_mnpo/logit": 0.3057871460914612, + "ht_mnpo/residual": 0.2982870936393738, + "ht_mnpo/residual_abs": 0.30613571405410767, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.3866753733777765e-07, + "logits/chosen": -1.7794278860092163, + "logits/rejected": -2.1755802631378174, + "logps/chosen": -0.2987704873085022, + "logps/rejected": -0.2674489915370941, + "loss": 1.0454192161560059, + "loss/core": 1.0454192161560059, + "rewards/accuracies": 0.9375, + "rewards/chosen": 4.152740001678467, + "rewards/margins": 3.0578713417053223, + "rewards/rejected": 1.094868779182434, + "step": 615 + }, + { + "epoch": 0.592062071023575, + "grad_norm": 7.90625, + "ht_mnpo/logit": 0.16309209167957306, + "ht_mnpo/residual": 0.15559211373329163, + "ht_mnpo/residual_abs": 0.1573321521282196, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.343478834970121e-07, + "logits/chosen": -2.0946414470672607, + "logits/rejected": -2.3167364597320557, + "logps/chosen": -0.2965165972709656, + "logps/rejected": -0.2895964980125427, + "loss": 0.220075324177742, + "loss/core": 0.220075324177742, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.6535112857818604, + "rewards/margins": 1.6309210062026978, + "rewards/rejected": 1.0225903987884521, + "step": 620 + }, + { + "epoch": 0.5968367651447329, + "grad_norm": 168.0, + "ht_mnpo/logit": 0.26800763607025146, + "ht_mnpo/residual": 0.26050764322280884, + "ht_mnpo/residual_abs": 0.26258984208106995, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.3007172168743852e-07, + "logits/chosen": -1.8491462469100952, + "logits/rejected": -2.066373586654663, + "logps/chosen": -0.2881318926811218, + "logps/rejected": -0.30786803364753723, + "loss": 0.5212100744247437, + "loss/core": 0.5212100744247437, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.430001735687256, + "rewards/margins": 2.6800761222839355, + "rewards/rejected": 0.7499252557754517, + "step": 625 + }, + { + "epoch": 0.6016114592658908, + "grad_norm": 190.0, + "ht_mnpo/logit": 0.3151053786277771, + "ht_mnpo/residual": 0.3076053857803345, + "ht_mnpo/residual_abs": 0.3096749186515808, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.25840659998631e-07, + "logits/chosen": -1.9944003820419312, + "logits/rejected": -2.25925350189209, + "logps/chosen": -0.33351221680641174, + "logps/rejected": -0.30969151854515076, + "loss": 1.3126461505889893, + "loss/core": 1.3126461505889893, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 4.146966457366943, + "rewards/margins": 3.1510536670684814, + "rewards/rejected": 0.9959124326705933, + "step": 630 + }, + { + "epoch": 0.6063861533870486, + "grad_norm": 2.15625, + "ht_mnpo/logit": 0.16642612218856812, + "ht_mnpo/residual": 0.15892614424228668, + "ht_mnpo/residual_abs": 0.1607307344675064, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.2165628955985313e-07, + "logits/chosen": -2.1082916259765625, + "logits/rejected": -2.336918354034424, + "logps/chosen": -0.28919678926467896, + "logps/rejected": -0.2757853865623474, + "loss": 0.32006698846817017, + "loss/core": 0.32006698846817017, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.2848565578460693, + "rewards/margins": 1.6642612218856812, + "rewards/rejected": 0.6205950975418091, + "step": 635 + }, + { + "epoch": 0.6111608475082065, + "grad_norm": 188.0, + "ht_mnpo/logit": 0.13086314499378204, + "ht_mnpo/residual": 0.12336313724517822, + "ht_mnpo/residual_abs": 0.1262132078409195, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.175201839416988e-07, + "logits/chosen": -2.047877550125122, + "logits/rejected": -2.198028564453125, + "logps/chosen": -0.30865031480789185, + "logps/rejected": -0.29963168501853943, + "loss": 0.12744709849357605, + "loss/core": 0.12744709849357605, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.1879143714904785, + "rewards/margins": 1.308631420135498, + "rewards/rejected": 0.87928307056427, + "step": 640 + }, + { + "epoch": 0.6159355416293644, + "grad_norm": 12.75, + "ht_mnpo/logit": 0.35795003175735474, + "ht_mnpo/residual": 0.3504500985145569, + "ht_mnpo/residual_abs": 0.3530126214027405, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.1343389856433658e-07, + "logits/chosen": -1.7279043197631836, + "logits/rejected": -2.076632022857666, + "logps/chosen": -0.309658944606781, + "logps/rejected": -0.2867974638938904, + "loss": 1.6379516124725342, + "loss/core": 1.6379516124725342, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 5.011632919311523, + "rewards/margins": 3.579500675201416, + "rewards/rejected": 1.4321321249008179, + "step": 645 + }, + { + "epoch": 0.6207102357505222, + "grad_norm": 78.5, + "ht_mnpo/logit": 0.2575874626636505, + "ht_mnpo/residual": 0.2500874698162079, + "ht_mnpo/residual_abs": 0.25217220187187195, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0939897011258e-07, + "logits/chosen": -1.9400348663330078, + "logits/rejected": -2.297685146331787, + "logps/chosen": -0.2735014259815216, + "logps/rejected": -0.2798306345939636, + "loss": 0.5699511170387268, + "loss/core": 0.5699511170387268, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.7315871715545654, + "rewards/margins": 2.5758748054504395, + "rewards/rejected": 1.1557127237319946, + "step": 650 + }, + { + "epoch": 0.6254849298716801, + "grad_norm": 4.4375, + "ht_mnpo/logit": 0.11060944944620132, + "ht_mnpo/residual": 0.1031094565987587, + "ht_mnpo/residual_abs": 0.10582210123538971, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0541691595800336e-07, + "logits/chosen": -2.1760499477386475, + "logits/rejected": -2.4464058876037598, + "logps/chosen": -0.263247013092041, + "logps/rejected": -0.2721119225025177, + "loss": 0.043946050107479095, + "loss/core": 0.043946050107479095, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.126974105834961, + "rewards/margins": 1.1060945987701416, + "rewards/rejected": 1.0208795070648193, + "step": 655 + }, + { + "epoch": 0.630259623992838, + "grad_norm": 286.0, + "ht_mnpo/logit": 0.3428904116153717, + "ht_mnpo/residual": 0.33539044857025146, + "ht_mnpo/residual_abs": 0.33829253911972046, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0148923358832021e-07, + "logits/chosen": -1.8339145183563232, + "logits/rejected": -2.0472755432128906, + "logps/chosen": -0.33507493138313293, + "logps/rejected": -0.31190431118011475, + "loss": 1.6195424795150757, + "loss/core": 1.6195424795150757, + "rewards/accuracies": 0.9375, + "rewards/chosen": 4.361727237701416, + "rewards/margins": 3.4289040565490723, + "rewards/rejected": 0.9328228831291199, + "step": 660 + }, + { + "epoch": 0.6350343181139958, + "grad_norm": 3.875, + "ht_mnpo/logit": 0.3288431465625763, + "ht_mnpo/residual": 0.3213431239128113, + "ht_mnpo/residual_abs": 0.323604017496109, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.761740004423926e-08, + "logits/chosen": -1.9675363302230835, + "logits/rejected": -2.2226428985595703, + "logps/chosen": -0.33497220277786255, + "logps/rejected": -0.2824486792087555, + "loss": 1.085584044456482, + "loss/core": 1.085584044456482, + "rewards/accuracies": 0.9375, + "rewards/chosen": 4.039675712585449, + "rewards/margins": 3.288431167602539, + "rewards/rejected": 0.7512443661689758, + "step": 665 + }, + { + "epoch": 0.6398090122351537, + "grad_norm": 49.5, + "ht_mnpo/logit": 0.26619061827659607, + "ht_mnpo/residual": 0.25869062542915344, + "ht_mnpo/residual_abs": 0.2621300220489502, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.380287136400999e-08, + "logits/chosen": -1.9468028545379639, + "logits/rejected": -2.044259548187256, + "logps/chosen": -0.32372915744781494, + "logps/rejected": -0.32650211453437805, + "loss": 1.0328400135040283, + "loss/core": 1.0328400135040283, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 4.102653503417969, + "rewards/margins": 2.6619060039520264, + "rewards/rejected": 1.440747618675232, + "step": 670 + }, + { + "epoch": 0.6445837063563116, + "grad_norm": 155.0, + "ht_mnpo/logit": 0.1919471025466919, + "ht_mnpo/residual": 0.18444707989692688, + "ht_mnpo/residual_abs": 0.18674826622009277, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.004708203586628e-08, + "logits/chosen": -2.0553970336914062, + "logits/rejected": -2.277501344680786, + "logps/chosen": -0.30528566241264343, + "logps/rejected": -0.30857595801353455, + "loss": 0.44716134667396545, + "loss/core": 0.44716134667396545, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.826671838760376, + "rewards/margins": 1.9194707870483398, + "rewards/rejected": 0.9072009325027466, + "step": 675 + }, + { + "epoch": 0.6493584004774694, + "grad_norm": 1408.0, + "ht_mnpo/logit": 0.5156992077827454, + "ht_mnpo/residual": 0.5081992149353027, + "ht_mnpo/residual_abs": 0.5100849270820618, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.635144445857407e-08, + "logits/chosen": -1.9224817752838135, + "logits/rejected": -2.295820713043213, + "logps/chosen": -0.3082597255706787, + "logps/rejected": -0.28871044516563416, + "loss": 2.1461853981018066, + "loss/core": 2.1461853981018066, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 6.397282600402832, + "rewards/margins": 5.1569929122924805, + "rewards/rejected": 1.2402900457382202, + "step": 680 + }, + { + "epoch": 0.6541330945986272, + "grad_norm": 11.875, + "ht_mnpo/logit": 0.2202470600605011, + "ht_mnpo/residual": 0.21274705231189728, + "ht_mnpo/residual_abs": 0.21405646204948425, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.271734841028552e-08, + "logits/chosen": -1.833820104598999, + "logits/rejected": -2.1766836643218994, + "logps/chosen": -0.2879013121128082, + "logps/rejected": -0.2877139449119568, + "loss": 0.39130714535713196, + "loss/core": 0.39130714535713196, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 3.120842456817627, + "rewards/margins": 2.202470541000366, + "rewards/rejected": 0.9183722734451294, + "step": 685 + }, + { + "epoch": 0.6589077887197852, + "grad_norm": 2.484375, + "ht_mnpo/logit": 0.33171364665031433, + "ht_mnpo/residual": 0.3242136240005493, + "ht_mnpo/residual_abs": 0.32709458470344543, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 7.91461605259007e-08, + "logits/chosen": -1.9676786661148071, + "logits/rejected": -2.1830246448516846, + "logps/chosen": -0.31910890340805054, + "logps/rejected": -0.2641559839248657, + "loss": 1.0723878145217896, + "loss/core": 1.0723878145217896, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.9963231086730957, + "rewards/margins": 3.317136287689209, + "rewards/rejected": 0.6791867613792419, + "step": 690 + }, + { + "epoch": 0.663682482840943, + "grad_norm": 54.75, + "ht_mnpo/logit": 0.35695138573646545, + "ht_mnpo/residual": 0.3494513928890228, + "ht_mnpo/residual_abs": 0.35167449712753296, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 7.563922378313217e-08, + "logits/chosen": -1.966403603553772, + "logits/rejected": -2.119166851043701, + "logps/chosen": -0.3139706552028656, + "logps/rejected": -0.3095807731151581, + "loss": 1.3685941696166992, + "loss/core": 1.3685941696166992, + "rewards/accuracies": 0.9375, + "rewards/chosen": 4.306251525878906, + "rewards/margins": 3.569514513015747, + "rewards/rejected": 0.7367372512817383, + "step": 695 + }, + { + "epoch": 0.6684571769621008, + "grad_norm": 828.0, + "ht_mnpo/logit": 0.3428606390953064, + "ht_mnpo/residual": 0.335360586643219, + "ht_mnpo/residual_abs": 0.337501585483551, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 7.219785699746572e-08, + "logits/chosen": -1.8964332342147827, + "logits/rejected": -2.292275905609131, + "logps/chosen": -0.30034637451171875, + "logps/rejected": -0.2862694263458252, + "loss": 1.527578592300415, + "loss/core": 1.527578592300415, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 4.155306816101074, + "rewards/margins": 3.428605556488037, + "rewards/rejected": 0.7267010807991028, + "step": 700 + }, + { + "epoch": 0.6732318710832588, + "grad_norm": 14.6875, + "ht_mnpo/logit": 0.2360476702451706, + "ht_mnpo/residual": 0.22854766249656677, + "ht_mnpo/residual_abs": 0.2305394411087036, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.882335432620779e-08, + "logits/chosen": -1.8612133264541626, + "logits/rejected": -2.1879470348358154, + "logps/chosen": -0.29452234506607056, + "logps/rejected": -0.2882519066333771, + "loss": 0.5280643701553345, + "loss/core": 0.5280643701553345, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 3.631542205810547, + "rewards/margins": 2.3604767322540283, + "rewards/rejected": 1.271065592765808, + "step": 705 + }, + { + "epoch": 0.6780065652044166, + "grad_norm": 496.0, + "ht_mnpo/logit": 0.27381792664527893, + "ht_mnpo/residual": 0.2663179337978363, + "ht_mnpo/residual_abs": 0.2707618176937103, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.551698478180589e-08, + "logits/chosen": -1.8524887561798096, + "logits/rejected": -2.209779739379883, + "logps/chosen": -0.3093031048774719, + "logps/rejected": -0.28702372312545776, + "loss": 1.0192804336547852, + "loss/core": 1.0192804336547852, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.5826401710510254, + "rewards/margins": 2.7381794452667236, + "rewards/rejected": 0.8444606065750122, + "step": 710 + }, + { + "epoch": 0.6827812593255744, + "grad_norm": 13.5, + "ht_mnpo/logit": 0.17260591685771942, + "ht_mnpo/residual": 0.1651059240102768, + "ht_mnpo/residual_abs": 0.16933459043502808, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.22799917546252e-08, + "logits/chosen": -1.8810266256332397, + "logits/rejected": -2.1536669731140137, + "logps/chosen": -0.3034297525882721, + "logps/rejected": -0.29463475942611694, + "loss": 0.3258531093597412, + "loss/core": 0.3258531093597412, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.0421574115753174, + "rewards/margins": 1.7260591983795166, + "rewards/rejected": 1.3160979747772217, + "step": 715 + }, + { + "epoch": 0.6875559534467324, + "grad_norm": 79.5, + "ht_mnpo/logit": 0.241688534617424, + "ht_mnpo/residual": 0.234188511967659, + "ht_mnpo/residual_abs": 0.23640303313732147, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.9113592545359944e-08, + "logits/chosen": -2.146207332611084, + "logits/rejected": -2.2886502742767334, + "logps/chosen": -0.309707373380661, + "logps/rejected": -0.309203565120697, + "loss": 0.458344042301178, + "loss/core": 0.458344042301178, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.303243637084961, + "rewards/margins": 2.4168851375579834, + "rewards/rejected": 0.886358380317688, + "step": 720 + }, + { + "epoch": 0.6923306475678902, + "grad_norm": 26.25, + "ht_mnpo/logit": 0.16217423975467682, + "ht_mnpo/residual": 0.15467426180839539, + "ht_mnpo/residual_abs": 0.15796169638633728, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.601897790725643e-08, + "logits/chosen": -1.9593452215194702, + "logits/rejected": -2.2494146823883057, + "logps/chosen": -0.275308758020401, + "logps/rejected": -0.27825742959976196, + "loss": 0.21573790907859802, + "loss/core": 0.21573790907859802, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.375974178314209, + "rewards/margins": 1.621742606163025, + "rewards/rejected": 0.7542315125465393, + "step": 725 + }, + { + "epoch": 0.697105341689048, + "grad_norm": 332.0, + "ht_mnpo/logit": 0.381821870803833, + "ht_mnpo/residual": 0.374321848154068, + "ht_mnpo/residual_abs": 0.3764488399028778, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.299731159831952e-08, + "logits/chosen": -2.0782310962677, + "logits/rejected": -2.2460265159606934, + "logps/chosen": -0.31326770782470703, + "logps/rejected": -0.3076656758785248, + "loss": 1.6463673114776611, + "loss/core": 1.6463673114776611, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 4.482171535491943, + "rewards/margins": 3.81821870803833, + "rewards/rejected": 0.6639531254768372, + "step": 730 + }, + { + "epoch": 0.7018800358102059, + "grad_norm": 5.34375, + "ht_mnpo/logit": 0.15911749005317688, + "ht_mnpo/residual": 0.15161748230457306, + "ht_mnpo/residual_abs": 0.17043578624725342, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.0049729943671013e-08, + "logits/chosen": -1.9329124689102173, + "logits/rejected": -2.1531574726104736, + "logps/chosen": -0.32668760418891907, + "logps/rejected": -0.3379434049129486, + "loss": 0.3836316466331482, + "loss/core": 0.3836316466331482, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.7718982696533203, + "rewards/margins": 1.591174840927124, + "rewards/rejected": 1.1807234287261963, + "step": 735 + }, + { + "epoch": 0.7066547299313638, + "grad_norm": 187.0, + "ht_mnpo/logit": 0.21543124318122864, + "ht_mnpo/residual": 0.20793123543262482, + "ht_mnpo/residual_abs": 0.21036949753761292, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.7177341408223994e-08, + "logits/chosen": -1.9197518825531006, + "logits/rejected": -2.1591908931732178, + "logps/chosen": -0.2565429210662842, + "logps/rejected": -0.2755061984062195, + "loss": 0.30916422605514526, + "loss/core": 0.30916422605514526, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.222672939300537, + "rewards/margins": 2.1543123722076416, + "rewards/rejected": 1.068360686302185, + "step": 740 + }, + { + "epoch": 0.7114294240525216, + "grad_norm": 500.0, + "ht_mnpo/logit": 0.26379165053367615, + "ht_mnpo/residual": 0.2562916874885559, + "ht_mnpo/residual_abs": 0.25983983278274536, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.438122617983442e-08, + "logits/chosen": -1.9132686853408813, + "logits/rejected": -2.159632921218872, + "logps/chosen": -0.26915454864501953, + "logps/rejected": -0.26063108444213867, + "loss": 0.9998035430908203, + "loss/core": 0.9998035430908203, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 3.6523070335388184, + "rewards/margins": 2.6379168033599854, + "rewards/rejected": 1.0143901109695435, + "step": 745 + }, + { + "epoch": 0.7162041181736795, + "grad_norm": 3.765625, + "ht_mnpo/logit": 0.2811947762966156, + "ht_mnpo/residual": 0.273694783449173, + "ht_mnpo/residual_abs": 0.2760910987854004, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.1662435763087114e-08, + "logits/chosen": -2.0892138481140137, + "logits/rejected": -2.2944881916046143, + "logps/chosen": -0.2737082839012146, + "logps/rejected": -0.259973406791687, + "loss": 1.173396110534668, + "loss/core": 1.173396110534668, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.8202099800109863, + "rewards/margins": 2.8119475841522217, + "rewards/rejected": 1.0082619190216064, + "step": 750 + }, + { + "epoch": 0.7209788122948374, + "grad_norm": 2.140625, + "ht_mnpo/logit": 0.2567535936832428, + "ht_mnpo/residual": 0.24925358593463898, + "ht_mnpo/residual_abs": 0.25218337774276733, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.902199258386732e-08, + "logits/chosen": -2.019989252090454, + "logits/rejected": -2.1991806030273438, + "logps/chosen": -0.3405812978744507, + "logps/rejected": -0.31987202167510986, + "loss": 1.182205319404602, + "loss/core": 1.182205319404602, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.823350429534912, + "rewards/margins": 2.567535877227783, + "rewards/rejected": 1.255814790725708, + "step": 755 + }, + { + "epoch": 0.7257535064159952, + "grad_norm": 82.5, + "ht_mnpo/logit": 0.40269652009010315, + "ht_mnpo/residual": 0.39519649744033813, + "ht_mnpo/residual_abs": 0.39851316809654236, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.646088960486862e-08, + "logits/chosen": -1.9100509881973267, + "logits/rejected": -2.2545628547668457, + "logps/chosen": -0.31178393959999084, + "logps/rejected": -0.28806573152542114, + "loss": 1.7591040134429932, + "loss/core": 1.7591040134429932, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 5.450934410095215, + "rewards/margins": 4.026965141296387, + "rewards/rejected": 1.4239691495895386, + "step": 760 + }, + { + "epoch": 0.7305282005371531, + "grad_norm": 5.9375, + "ht_mnpo/logit": 0.24737465381622314, + "ht_mnpo/residual": 0.23987463116645813, + "ht_mnpo/residual_abs": 0.24287545680999756, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.398008995217988e-08, + "logits/chosen": -1.9502792358398438, + "logits/rejected": -2.0989506244659424, + "logps/chosen": -0.2815241813659668, + "logps/rejected": -0.29228392243385315, + "loss": 0.46429914236068726, + "loss/core": 0.46429914236068726, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.6968235969543457, + "rewards/margins": 2.4737465381622314, + "rewards/rejected": 1.2230769395828247, + "step": 765 + }, + { + "epoch": 0.735302894658311, + "grad_norm": 88.0, + "ht_mnpo/logit": 0.365441232919693, + "ht_mnpo/residual": 0.357941210269928, + "ht_mnpo/residual_abs": 0.3601204752922058, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.1580526553093315e-08, + "logits/chosen": -1.895281195640564, + "logits/rejected": -2.2292673587799072, + "logps/chosen": -0.3453708291053772, + "logps/rejected": -0.3260127305984497, + "loss": 1.5065122842788696, + "loss/core": 1.5065122842788696, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 4.30310583114624, + "rewards/margins": 3.6544125080108643, + "rewards/rejected": 0.6486933827400208, + "step": 770 + }, + { + "epoch": 0.7400775887794688, + "grad_norm": 10.25, + "ht_mnpo/logit": 0.21786002814769745, + "ht_mnpo/residual": 0.21036005020141602, + "ht_mnpo/residual_abs": 0.2132169008255005, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.9263101785268252e-08, + "logits/chosen": -1.9512803554534912, + "logits/rejected": -2.2073264122009277, + "logps/chosen": -0.3291786313056946, + "logps/rejected": -0.31446442008018494, + "loss": 0.46908074617385864, + "loss/core": 0.46908074617385864, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.122516393661499, + "rewards/margins": 2.178600549697876, + "rewards/rejected": 0.9439161419868469, + "step": 775 + }, + { + "epoch": 0.7448522829006267, + "grad_norm": 5.03125, + "ht_mnpo/logit": 0.2005649358034134, + "ht_mnpo/residual": 0.19306494295597076, + "ht_mnpo/residual_abs": 0.19553616642951965, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.7028687137384264e-08, + "logits/chosen": -2.147618293762207, + "logits/rejected": -2.329775333404541, + "logps/chosen": -0.2661969065666199, + "logps/rejected": -0.2787913680076599, + "loss": 0.28196778893470764, + "loss/core": 0.28196778893470764, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.99574613571167, + "rewards/margins": 2.0056493282318115, + "rewards/rejected": 0.9900972247123718, + "step": 780 + }, + { + "epoch": 0.7496269770217845, + "grad_norm": 5.625, + "ht_mnpo/logit": 0.23682251572608948, + "ht_mnpo/residual": 0.22932252287864685, + "ht_mnpo/residual_abs": 0.23593783378601074, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4878122881409447e-08, + "logits/chosen": -1.9615819454193115, + "logits/rejected": -2.3150079250335693, + "logps/chosen": -0.3067627549171448, + "logps/rejected": -0.2893941402435303, + "loss": 0.7686271667480469, + "loss/core": 0.7686271667480469, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.3052401542663574, + "rewards/margins": 2.36822509765625, + "rewards/rejected": 0.9370150566101074, + "step": 785 + }, + { + "epoch": 0.7544016711429424, + "grad_norm": 91.5, + "ht_mnpo/logit": 0.26232343912124634, + "ht_mnpo/residual": 0.2548234462738037, + "ht_mnpo/residual_abs": 0.25668424367904663, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.2812217756608937e-08, + "logits/chosen": -1.9566657543182373, + "logits/rejected": -2.261164426803589, + "logps/chosen": -0.32755106687545776, + "logps/rejected": -0.3085654377937317, + "loss": 0.5578528642654419, + "loss/core": 0.5578528642654419, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 3.1903016567230225, + "rewards/margins": 2.623234272003174, + "rewards/rejected": 0.5670672059059143, + "step": 790 + }, + { + "epoch": 0.7591763652641003, + "grad_norm": 724.0, + "ht_mnpo/logit": 0.31201422214508057, + "ht_mnpo/residual": 0.30451422929763794, + "ht_mnpo/residual_abs": 0.30678969621658325, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.0831748665410763e-08, + "logits/chosen": -2.053274393081665, + "logits/rejected": -2.2694168090820312, + "logps/chosen": -0.3137114942073822, + "logps/rejected": -0.29724258184432983, + "loss": 0.9311115145683289, + "loss/core": 0.9311115145683289, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 4.055318832397461, + "rewards/margins": 3.1201424598693848, + "rewards/rejected": 0.9351763725280762, + "step": 795 + }, + { + "epoch": 0.7639510593852581, + "grad_norm": 23.625, + "ht_mnpo/logit": 0.20176801085472107, + "ht_mnpo/residual": 0.19426801800727844, + "ht_mnpo/residual_abs": 0.19569215178489685, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8937460381244967e-08, + "logits/chosen": -2.0264482498168945, + "logits/rejected": -2.240051746368408, + "logps/chosen": -0.2773614525794983, + "logps/rejected": -0.2761766314506531, + "loss": 0.31296461820602417, + "loss/core": 0.31296461820602417, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 3.1320345401763916, + "rewards/margins": 2.0176799297332764, + "rewards/rejected": 1.1143543720245361, + "step": 800 + }, + { + "epoch": 0.768725753506416, + "grad_norm": 338.0, + "ht_mnpo/logit": 0.31727129220962524, + "ht_mnpo/residual": 0.3097712993621826, + "ht_mnpo/residual_abs": 0.3111569285392761, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.713006526846439e-08, + "logits/chosen": -2.038196086883545, + "logits/rejected": -2.4574263095855713, + "logps/chosen": -0.30098554491996765, + "logps/rejected": -0.29372772574424744, + "loss": 0.8073376417160034, + "loss/core": 0.8073376417160034, + "rewards/accuracies": 0.987500011920929, + "rewards/chosen": 4.151384353637695, + "rewards/margins": 3.172712802886963, + "rewards/rejected": 0.9786712527275085, + "step": 805 + }, + { + "epoch": 0.7735004476275739, + "grad_norm": 540.0, + "ht_mnpo/logit": 0.21397161483764648, + "ht_mnpo/residual": 0.20647163689136505, + "ht_mnpo/residual_abs": 0.21008792519569397, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.541024301445404e-08, + "logits/chosen": -1.9568876028060913, + "logits/rejected": -2.162205457687378, + "logps/chosen": -0.32081520557403564, + "logps/rejected": -0.28566688299179077, + "loss": 0.5576066970825195, + "loss/core": 0.5576066970825195, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.346921443939209, + "rewards/margins": 2.139716386795044, + "rewards/rejected": 1.2072051763534546, + "step": 810 + }, + { + "epoch": 0.7782751417487317, + "grad_norm": 120.5, + "ht_mnpo/logit": 0.34778210520744324, + "ht_mnpo/residual": 0.34028205275535583, + "ht_mnpo/residual_abs": 0.34224414825439453, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.3778640374027983e-08, + "logits/chosen": -1.81075119972229, + "logits/rejected": -2.0091278553009033, + "logps/chosen": -0.28327834606170654, + "logps/rejected": -0.2862473726272583, + "loss": 0.8907841444015503, + "loss/core": 0.8907841444015503, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 4.21142578125, + "rewards/margins": 3.477820873260498, + "rewards/rejected": 0.7336045503616333, + "step": 815 + }, + { + "epoch": 0.7830498358698896, + "grad_norm": 14.1875, + "ht_mnpo/logit": 0.15176530182361603, + "ht_mnpo/residual": 0.1442653089761734, + "ht_mnpo/residual_abs": 0.14901027083396912, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.2235870926211616e-08, + "logits/chosen": -1.9298378229141235, + "logits/rejected": -2.1080105304718018, + "logps/chosen": -0.2902945578098297, + "logps/rejected": -0.309215247631073, + "loss": 0.4051854610443115, + "loss/core": 0.4051854610443115, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.702075481414795, + "rewards/margins": 1.517652988433838, + "rewards/rejected": 1.184422492980957, + "step": 820 + }, + { + "epoch": 0.7878245299910475, + "grad_norm": 1312.0, + "ht_mnpo/logit": 0.3178713321685791, + "ht_mnpo/residual": 0.3103713393211365, + "ht_mnpo/residual_abs": 0.3130297064781189, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0782514843499652e-08, + "logits/chosen": -2.0223171710968018, + "logits/rejected": -2.2735869884490967, + "logps/chosen": -0.34258824586868286, + "logps/rejected": -0.2813429832458496, + "loss": 0.8753134608268738, + "loss/core": 0.8753134608268738, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.9592788219451904, + "rewards/margins": 3.178713321685791, + "rewards/rejected": 0.7805658578872681, + "step": 825 + }, + { + "epoch": 0.7925992241122053, + "grad_norm": 29.625, + "ht_mnpo/logit": 0.232127383351326, + "ht_mnpo/residual": 0.22462734580039978, + "ht_mnpo/residual_abs": 0.22618567943572998, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.419118673676924e-09, + "logits/chosen": -2.1135377883911133, + "logits/rejected": -2.3391060829162598, + "logps/chosen": -0.27336445450782776, + "logps/rejected": -0.28452619910240173, + "loss": 0.5279203653335571, + "loss/core": 0.5279203653335571, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 3.090513229370117, + "rewards/margins": 2.3212738037109375, + "rewards/rejected": 0.7692393660545349, + "step": 830 + }, + { + "epoch": 0.7973739182333631, + "grad_norm": 31.75, + "ht_mnpo/logit": 0.23250722885131836, + "ht_mnpo/residual": 0.22500717639923096, + "ht_mnpo/residual_abs": 0.22754648327827454, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.14619513428405e-09, + "logits/chosen": -1.7653303146362305, + "logits/rejected": -1.9559898376464844, + "logps/chosen": -0.2960887551307678, + "logps/rejected": -0.3074343502521515, + "loss": 0.5747458934783936, + "loss/core": 0.5747458934783936, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.5430405139923096, + "rewards/margins": 2.3250718116760254, + "rewards/rejected": 1.217968463897705, + "step": 835 + }, + { + "epoch": 0.8021486123545211, + "grad_norm": 434.0, + "ht_mnpo/logit": 0.22585304081439972, + "ht_mnpo/residual": 0.2183530032634735, + "ht_mnpo/residual_abs": 0.2201291024684906, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.96422291980539e-09, + "logits/chosen": -2.053584337234497, + "logits/rejected": -2.245476722717285, + "logps/chosen": -0.3269621729850769, + "logps/rejected": -0.26414385437965393, + "loss": 0.5430350303649902, + "loss/core": 0.5430350303649902, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 3.252704620361328, + "rewards/margins": 2.2585301399230957, + "rewards/rejected": 0.9941746592521667, + "step": 840 + }, + { + "epoch": 0.8069233064756789, + "grad_norm": 112.5, + "ht_mnpo/logit": 0.2180916965007782, + "ht_mnpo/residual": 0.21059170365333557, + "ht_mnpo/residual_abs": 0.21333935856819153, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.8736465216517594e-09, + "logits/chosen": -1.9658924341201782, + "logits/rejected": -2.281589984893799, + "logps/chosen": -0.30036720633506775, + "logps/rejected": -0.2967119812965393, + "loss": 0.3782276511192322, + "loss/core": 0.3782276511192322, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.758944511413574, + "rewards/margins": 2.1809170246124268, + "rewards/rejected": 0.5780273079872131, + "step": 845 + }, + { + "epoch": 0.8116980005968367, + "grad_norm": 2.140625, + "ht_mnpo/logit": 0.2448008507490158, + "ht_mnpo/residual": 0.23730087280273438, + "ht_mnpo/residual_abs": 0.23997625708580017, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.874876061005173e-09, + "logits/chosen": -1.8780618906021118, + "logits/rejected": -2.181715488433838, + "logps/chosen": -0.2794502377510071, + "logps/rejected": -0.26828446984291077, + "loss": 0.8545548319816589, + "loss/core": 0.8545548319816589, + "rewards/accuracies": 0.9375, + "rewards/chosen": 3.7230312824249268, + "rewards/margins": 2.4480087757110596, + "rewards/rejected": 1.2750225067138672, + "step": 850 + }, + { + "epoch": 0.8164726947179947, + "grad_norm": 11.0625, + "ht_mnpo/logit": 0.1709764301776886, + "ht_mnpo/residual": 0.16347643733024597, + "ht_mnpo/residual_abs": 0.1738419234752655, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.968287134589188e-09, + "logits/chosen": -2.113243579864502, + "logits/rejected": -2.4351394176483154, + "logps/chosen": -0.26266947388648987, + "logps/rejected": -0.2733474373817444, + "loss": 0.12632545828819275, + "loss/core": 0.12632545828819275, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 2.9499731063842773, + "rewards/margins": 1.7097641229629517, + "rewards/rejected": 1.2402087450027466, + "step": 855 + }, + { + "epoch": 0.8212473888391525, + "grad_norm": 3.296875, + "ht_mnpo/logit": 0.14397737383842468, + "ht_mnpo/residual": 0.13647738099098206, + "ht_mnpo/residual_abs": 0.13811519742012024, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.154220673422192e-09, + "logits/chosen": -1.9785963296890259, + "logits/rejected": -2.251053810119629, + "logps/chosen": -0.27199432253837585, + "logps/rejected": -0.27827268838882446, + "loss": 0.15808339416980743, + "loss/core": 0.15808339416980743, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 2.0167627334594727, + "rewards/margins": 1.4397737979888916, + "rewards/rejected": 0.5769888758659363, + "step": 860 + }, + { + "epoch": 0.8260220829603103, + "grad_norm": 161.0, + "ht_mnpo/logit": 0.3069954812526703, + "ht_mnpo/residual": 0.2994953989982605, + "ht_mnpo/residual_abs": 0.3049822747707367, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4329828146074096e-09, + "logits/chosen": -1.8511345386505127, + "logits/rejected": -2.2174289226531982, + "logps/chosen": -0.3130944073200226, + "logps/rejected": -0.3080425560474396, + "loss": 1.1950414180755615, + "loss/core": 1.1950414180755615, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.894388198852539, + "rewards/margins": 3.0699546337127686, + "rewards/rejected": 0.8244336247444153, + "step": 865 + }, + { + "epoch": 0.8307967770814683, + "grad_norm": 29.75, + "ht_mnpo/logit": 0.09861830621957779, + "ht_mnpo/residual": 0.09111831337213516, + "ht_mnpo/residual_abs": 0.09437834471464157, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8048447862070714e-09, + "logits/chosen": -2.1397294998168945, + "logits/rejected": -2.3984568119049072, + "logps/chosen": -0.28324785828590393, + "logps/rejected": -0.28805121779441833, + "loss": 0.06719032675027847, + "loss/core": 0.06719032675027847, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.770001769065857, + "rewards/margins": 0.9861831665039062, + "rewards/rejected": 0.7838188409805298, + "step": 870 + }, + { + "epoch": 0.8355714712026261, + "grad_norm": 3.078125, + "ht_mnpo/logit": 0.11863557994365692, + "ht_mnpo/residual": 0.1111355796456337, + "ht_mnpo/residual_abs": 0.11609260737895966, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.2700428052447033e-09, + "logits/chosen": -1.9720087051391602, + "logits/rejected": -2.288149356842041, + "logps/chosen": -0.2549072802066803, + "logps/rejected": -0.27585774660110474, + "loss": 0.08893564343452454, + "loss/core": 0.08893564343452454, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 1.8031947612762451, + "rewards/margins": 1.1863558292388916, + "rewards/rejected": 0.6168391108512878, + "step": 875 + }, + { + "epoch": 0.8403461653237839, + "grad_norm": 50.75, + "ht_mnpo/logit": 0.13304957747459412, + "ht_mnpo/residual": 0.1255495846271515, + "ht_mnpo/residual_abs": 0.12660455703735352, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.287779888734858e-10, + "logits/chosen": -2.260289430618286, + "logits/rejected": -2.4413468837738037, + "logps/chosen": -0.304781973361969, + "logps/rejected": -0.30884766578674316, + "loss": 0.14404062926769257, + "loss/core": 0.14404062926769257, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.8423153162002563, + "rewards/margins": 1.3304959535598755, + "rewards/rejected": 0.5118193626403809, + "step": 880 + }, + { + "epoch": 0.8451208594449419, + "grad_norm": 2.796875, + "ht_mnpo/logit": 0.2581387162208557, + "ht_mnpo/residual": 0.2506386935710907, + "ht_mnpo/residual_abs": 0.2524881660938263, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.812162787445062e-10, + "logits/chosen": -1.9322468042373657, + "logits/rejected": -2.3333423137664795, + "logps/chosen": -0.319403737783432, + "logps/rejected": -0.3051341474056244, + "loss": 0.6756513714790344, + "loss/core": 0.6756513714790344, + "rewards/accuracies": 0.9750000238418579, + "rewards/chosen": 3.50068736076355, + "rewards/margins": 2.5813872814178467, + "rewards/rejected": 0.9193001985549927, + "step": 885 + }, + { + "epoch": 0.8498955535660997, + "grad_norm": 12.8125, + "ht_mnpo/logit": 0.22012737393379211, + "ht_mnpo/residual": 0.2126273661851883, + "ht_mnpo/residual_abs": 0.2149299830198288, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.2748837860270265e-10, + "logits/chosen": -1.9026530981063843, + "logits/rejected": -2.1633448600769043, + "logps/chosen": -0.29086828231811523, + "logps/rejected": -0.27807530760765076, + "loss": 0.35099947452545166, + "loss/core": 0.35099947452545166, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 2.9746203422546387, + "rewards/margins": 2.2012736797332764, + "rewards/rejected": 0.7733467817306519, + "step": 890 + }, + { + "epoch": 0.8546702476872575, + "grad_norm": 15.8125, + "ht_mnpo/logit": 0.3148799538612366, + "ht_mnpo/residual": 0.3073800206184387, + "ht_mnpo/residual_abs": 0.3098318874835968, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.768970513457151e-11, + "logits/chosen": -1.93404221534729, + "logits/rejected": -2.2042999267578125, + "logps/chosen": -0.32174813747406006, + "logps/rejected": -0.2906070351600647, + "loss": 0.975996196269989, + "loss/core": 0.975996196269989, + "rewards/accuracies": 0.9375, + "rewards/chosen": 4.408227443695068, + "rewards/margins": 3.1487996578216553, + "rewards/rejected": 1.259427785873413, + "step": 895 + }, + { + "epoch": 0.8594449418084154, + "grad_norm": 16.5, + "ht_mnpo/logit": 0.22211775183677673, + "ht_mnpo/residual": 0.2146177738904953, + "ht_mnpo/residual_abs": 0.26752087473869324, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8803520859811406e-12, + "logits/chosen": -1.908017873764038, + "logits/rejected": -2.071946382522583, + "logps/chosen": -0.30949825048446655, + "logps/rejected": -0.33156439661979675, + "loss": 0.5480181574821472, + "loss/core": 0.5480181574821472, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 4.108867645263672, + "rewards/margins": 2.221177577972412, + "rewards/rejected": 1.8876899480819702, + "step": 900 + }, + { + "epoch": 0.8594449418084154, + "step": 900, + "total_flos": 0.0, + "train_loss": 0.7468786084651947, + "train_runtime": 6934.2752, + "train_samples_per_second": 2.077, + "train_steps_per_second": 0.13 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..6df379c --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1f238ba516ceace5ec9048a217e4d497cc9d5b574a564215405530b1bd130c5 +size 7057