commit 8b9fbab47a5b265222fe948e426ed8156a3266ea Author: ModelHub XC Date: Thu Jul 23 10:44:10 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: promotion/qwen3-8b-aaai27-flagship-ht-mnpo-safety-s42 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..d5ed155 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +--- +library_name: transformers +base_model: Qwen/Qwen3-8B +tags: +- ronpo +- mnpo +- preference-optimization +--- + +# qwen3-8b-aaai27-flagship-ht-mnpo-safety-s42 + +Research checkpoint for the RONPO AAAI revision experiments. + +- Method: ht_mnpo_safety +- Base model: `Qwen/Qwen3-8B` +- Seed: 42 +- Local source at upload time: `/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/full/ht_mnpo_safety/seed42/attempt1` +- Uploaded at UTC: 2026-07-13T15:59:32Z +- Run status metadata: `{"attempt": 1, "effective_batch_size": 16, "method": "ht_mnpo_safety", "objective_protocol": "objective_protocol.json", "optimizer_steps": 900, "resource_profile": "resource_profile.json", "scope_amendment": "scope_amendment_20260713.json", "seed": 42, "split_manifest": "split_manifest.json", "stability_gate": "stability_gate.json", "status": "completed", "wandb_run_id": "152fbc211857", "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/152fbc211857"}` + +AAAI-27 P1 matched budget: 900 optimizer steps, effective batch 16; passed non-thinking and collapse stability gates; sealed-test metrics are not used for checkpoint selection. + +Intended use: reproducibility and evaluation for the RONPO paper. This +checkpoint is not intended as a production assistant. diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..c77fa53 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8589835361488904, + "total_flos": 0.0, + "train_loss": 0.3534543008564247, + "train_runtime": 7070.5487, + "train_samples": 16764, + "train_samples_per_second": 2.037, + "train_steps_per_second": 0.127 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..769e30f --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..139ea2e --- /dev/null +++ b/config.yaml @@ -0,0 +1,58 @@ +model_name_or_path: Qwen/Qwen3-8B +torch_dtype: null +attn_implementation: sdpa +dataset_mixer: + /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/ht_mnpo_safety: 1.0 +dataset_splits: +- train +- test +preprocessing_num_workers: 4 +bf16: true +loss_type: ht_mnpo +eta: 0.0075 +ratio: 0.3333 +max_history_t: 1 +history_weights: +- 1.0 +dpo_beta: 0.05 +simpo_beta: 2.0 +simpo_gamma: 0.6 +ronpo_alpha: 0.5 +ronpo_tau: 0.05 +ronpo_target_column: ronpo_target +reference_anchor_weight: 0.0 +preference_sft_weight: 0.0 +ht_target_column: ht_target +ht_target_scale: 1.0 +beta: 10.0 +learning_rate: 5.0e-07 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 1.0 +seed: 42 +gradient_accumulation_steps: 16 +gradient_checkpointing: true +num_train_epochs: 1 +max_steps: 900 +per_device_train_batch_size: 1 +per_device_eval_batch_size: 1 +max_length: 2048 +max_prompt_length: 1800 +do_eval: false +eval_strategy: 'no' +logging_steps: 5 +log_level: info +generate_during_eval: false +load_best_model_at_end: false +save_strategy: steps +save_steps: 900 +save_total_limit: 1 +save_only_model: true +save_safetensors: true +push_to_hub: false +report_to: +- wandb +output_dir: /NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/full/ht_mnpo_safety/seed42/attempt1 +run_name: aaai27-flagship-full-ht_mnpo_safety-s42-a1 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..afeddcb --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.13.0" +} diff --git a/job_status.json b/job_status.json new file mode 100644 index 0000000..b29f1d0 --- /dev/null +++ b/job_status.json @@ -0,0 +1,23 @@ +{ + "status": "completed", + "stage": "full", + "method": "ht_mnpo_safety", + "seed": 42, + "attempt": 1, + "gpu": 2, + "gpus": [ + 2 + ], + "returncode": 0, + "finite_metrics": true, + "oom": false, + "traceback": false, + "reason": "passed", + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "152fbc211857", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/152fbc211857", + "output_dir": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/full/ht_mnpo_safety/seed42/attempt1", + "log": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/logs/full_ht_mnpo_safety_s42_a1.log", + "completed_at": "2026-07-13T15:54:49Z" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..aa8c1dc --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8c967585e216e40b6974843f1c22d8226fb793a1594e6033a1e06f69ac89fb3 +size 16381517208 diff --git a/objective_protocol.json b/objective_protocol.json new file mode 100644 index 0000000..35b3b44 --- /dev/null +++ b/objective_protocol.json @@ -0,0 +1,69 @@ +{ + "schema_version": 1, + "frozen_at_kst": "2026-07-12T21:40:00+09:00", + "base_model": "Qwen/Qwen3-8B", + "primary_triple": { + "helpfulness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "ultrafeedback-helpfulness", + "transform": "identity" + }, + "safety": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "beavertails-is_safe", + "transform": "identity" + }, + "conciseness": { + "model": "RLHFlow/ArmoRM-Llama3-8B-v0.1", + "head": "helpsteer-verbosity", + "transform": "negate" + } + }, + "gate_thresholds": { + "raw_score_std_strictly_greater_than": 0.01, + "aggregate_prompt_pair_spearman_median_strictly_less_than": 0.0, + "aggregate_pairwise_top1_mismatch_at_least": 0.5 + }, + "gate_pool": "source-train Qwen3-8B response pool split into deterministic train/validation; sealed source-test excluded", + "training_seeds": [42, 43, 44], + "optimizer_steps": 900, + "effective_batch_size": 16, + "decode": { + "enable_thinking": false, + "temperature": 0.7, + "top_p": 0.9, + "max_new_tokens": 2048, + "decode_seed": 42 + }, + "methods": [ + "ronpo_full_expect", + "ronpo_k_only", + "dpo", + "ipo", + "simpo", + "kto", + "sppo_avg", + "inpo_avg", + "ht_mnpo_helpfulness", + "ht_mnpo_safety", + "ht_mnpo_conciseness" + ], + "selection_policy": "No sealed-test selection. One preregistered configuration per method; failures use only the declared stability retry recipe, at most three attempts.", + "stability_gate": { + "validation_prompts": 128, + "min_candidate_base_mean_word_ratio": 0.33, + "max_candidate_base_mean_word_ratio": 2.0, + "max_consecutive_identical_word_run": 20, + "think_tag_count": 0, + "empty_response_count": 0, + "retry_recipe": "attempt 2 halves learning rate, enables gradient checkpointing, and adds reference=0.02/SFT=0.002 anchoring to unanchored unified losses; attempt 3 quarters learning rate and uses reference=0.05/SFT=0.005; RONPO uses reference=0.05/SFT=0.005 from attempt 1" + }, + "primary_metric": "mean prompt-level minimum of per-prompt min-max-normalized helpfulness, safety, conciseness", + "bootstrap": { + "unit": "prompt", + "paired": true, + "resamples": 2000, + "interval": "percentile_95", + "seed": 42 + } +} diff --git a/provenance.json b/provenance.json new file mode 100644 index 0000000..2444952 --- /dev/null +++ b/provenance.json @@ -0,0 +1,14 @@ +{ + "method": "ht_mnpo_safety", + "seed": 42, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "152fbc211857", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/152fbc211857", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json", + "scope_amendment": "scope_amendment_20260713.json" +} diff --git a/resource_profile.json b/resource_profile.json new file mode 100644 index 0000000..18eec47 --- /dev/null +++ b/resource_profile.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "measured_at_kst": "2026-07-12T22:38:00+09:00", + "hardware": "NVIDIA B200 183359 MiB reported total; 178.35 GiB PyTorch-visible capacity", + "smoke_evidence": { + "attempted_profile": {"per_device_batch_size": 2, "gradient_accumulation_steps": 8, "gradient_checkpointing": false, "effective_batch_size": 16}, + "outcome": "OOM before completing the 20-step smoke across every launched unified baseline; representative process used 176.87 GiB and failed a 1.62 GiB allocation. KTO also OOMed at the optimizer step.", + "paper_metric": false + }, + "selected_full_training_profile": { + "per_device_batch_size": 2, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": false, + "attention_implementation": "PyTorch SDPA with cuDNN SDPA disabled", + "effective_batch_size": 16, + "applies_symmetrically_to_all_unified_pairwise_methods": true, + "kto_implementation_constraint": "TRL KTO requires actual batch size > 1. KTO uses batch 2 / accumulation 8 and identical AdamW with foreach=False. PyTorch SDPA is used with cuDNN SDPA disabled. The fixed base reference model is placed on a second B200 and its exact logits are returned to the policy device; the policy/optimizer remain on the first B200. This avoids a measured ~6-hour reference-precompute path while leaving the KTO loss, actual/effective batch, optimizer, and 900-step budget unchanged." + }, + "retry_semantics": "Resource selection is separated from method-stability attempts. Full attempt 1 uses the smoke-validated SDPA batch-2 profile and retains the preregistered method LR/anchor; attempts 2 and 3 fall back to batch 1 / accumulation 16 / gradient checkpointing and apply the declared LR/anchor stabilization recipe.", + "provenance_note": "No reward or sealed-evaluation result was available when this hardware profile was selected." +} diff --git a/run_status.json b/run_status.json new file mode 100644 index 0000000..d5795b2 --- /dev/null +++ b/run_status.json @@ -0,0 +1,15 @@ +{ + "method": "ht_mnpo_safety", + "seed": 42, + "attempt": 1, + "optimizer_steps": 900, + "effective_batch_size": 16, + "wandb_run_id": "152fbc211857", + "wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/152fbc211857", + "split_manifest": "split_manifest.json", + "objective_protocol": "objective_protocol.json", + "resource_profile": "resource_profile.json", + "stability_gate": "stability_gate.json", + "scope_amendment": "scope_amendment_20260713.json", + "status": "completed" +} diff --git a/split_manifest.json b/split_manifest.json new file mode 100644 index 0000000..4fb96bd --- /dev/null +++ b/split_manifest.json @@ -0,0 +1,23 @@ +{ + "schema_version": 1, + "split_rule": "sort source-train prompts by sha256('flagship-v1||'+prompt_id); first round(10%) validation", + "validation_fraction": 0.1, + "source_dir": "/NHNHOME/WORKSPACE/26msit001_A/BASE/aipr_lab_sjkim_eval/revision_qwen3_8b/full_iter1/precomputed/avg_oracle", + "base_model": "Qwen/Qwen3-8B", + "counts": { + "train": 16780, + "validation": 1864, + "sealed_test": 604 + }, + "prompt_id_sha256": { + "train": "fcf6f2ea5a0b7bcd41179e6f5288f4d850b2d04bdb729da91907bd18bf733a69", + "validation": "a87ba6f519192164daa487b109a6547dd8cbde46a27a373d6da201ca2d061b8f", + "sealed_test": "b398ced6af6cf73d731e1a5f32a0631d8060323d51eda1e375695566bf5f1867" + }, + "overlaps": { + "train_validation": 0, + "train_sealed_test": 0, + "validation_sealed_test": 0 + }, + "sealed_test_policy": "prompts only until S4a; no objective scores or model generations used in S0-S3" +} diff --git a/stability_gate.json b/stability_gate.json new file mode 100644 index 0000000..c56ffd6 --- /dev/null +++ b/stability_gate.json @@ -0,0 +1,38 @@ +{ + "status": "passed", + "passed": true, + "base": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 388.1484375, + "max_words": 1379, + "max_repeat_run": 4 + }, + "candidate": { + "records": 128, + "empty_count": 0, + "think_leak_count": 0, + "think_leak_indices": [], + "mean_words": 404.4765625, + "max_words": 1283, + "max_repeat_run": 4 + }, + "candidate_base_mean_word_ratio": 1.042066702896363, + "thresholds": { + "min_length_ratio": 0.33, + "max_length_ratio": 2.0, + "max_repeat_run": 20, + "expected_records": 128, + "think_leakage": 0, + "empty_count": 0 + }, + "checks": { + "complete_records": true, + "zero_empty": true, + "zero_think_leakage": true, + "length_near_base": true, + "repeat_run_below_threshold": true + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..ea3c9e2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 2048, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..c77fa53 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 0.8589835361488904, + "total_flos": 0.0, + "train_loss": 0.3534543008564247, + "train_runtime": 7070.5487, + "train_samples": 16764, + "train_samples_per_second": 2.037, + "train_steps_per_second": 0.127 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..09ce8a3 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,3823 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8589835361488904, + "eval_steps": 500, + "global_step": 900, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.004772130756382725, + "grad_norm": 7.90625, + "ht_mnpo/logit": 0.12129320949316025, + "ht_mnpo/residual": 0.11379321664571762, + "ht_mnpo/residual_abs": 0.14974038302898407, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.222222222222222e-08, + "logits/chosen": -2.319894313812256, + "logits/rejected": -2.253976345062256, + "logps/chosen": -0.2934279143810272, + "logps/rejected": -0.2908848226070404, + "loss": 0.16081146895885468, + "loss/core": 0.16081146895885468, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.294299602508545, + "rewards/margins": 1.2129321098327637, + "rewards/rejected": 1.0813673734664917, + "step": 5 + }, + { + "epoch": 0.00954426151276545, + "grad_norm": 912.0, + "ht_mnpo/logit": 0.1957351267337799, + "ht_mnpo/residual": 0.18823513388633728, + "ht_mnpo/residual_abs": 0.20851953327655792, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5e-08, + "logits/chosen": -2.0424561500549316, + "logits/rejected": -1.9713897705078125, + "logps/chosen": -0.27538567781448364, + "logps/rejected": -0.28329208493232727, + "loss": 0.67027348279953, + "loss/core": 0.67027348279953, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.2881271839141846, + "rewards/margins": 1.9573514461517334, + "rewards/rejected": 1.3307758569717407, + "step": 10 + }, + { + "epoch": 0.014316392269148175, + "grad_norm": 2.28125, + "ht_mnpo/logit": 0.22769351303577423, + "ht_mnpo/residual": 0.22019347548484802, + "ht_mnpo/residual_abs": 0.23837217688560486, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 7.777777777777778e-08, + "logits/chosen": -2.1736812591552734, + "logits/rejected": -2.193058967590332, + "logps/chosen": -0.3110750913619995, + "logps/rejected": -0.31340306997299194, + "loss": 0.7280125617980957, + "loss/core": 0.7280125617980957, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.035675048828125, + "rewards/margins": 2.27693510055542, + "rewards/rejected": 0.7587399482727051, + "step": 15 + }, + { + "epoch": 0.0190885230255309, + "grad_norm": 188.0, + "ht_mnpo/logit": 0.17653341591358185, + "ht_mnpo/residual": 0.16903340816497803, + "ht_mnpo/residual_abs": 0.19080820679664612, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0555555555555555e-07, + "logits/chosen": -2.2002100944519043, + "logits/rejected": -2.175318717956543, + "logps/chosen": -0.30930784344673157, + "logps/rejected": -0.32532697916030884, + "loss": 0.6724044680595398, + "loss/core": 0.6724044680595398, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.882935047149658, + "rewards/margins": 1.765333890914917, + "rewards/rejected": 1.117600917816162, + "step": 20 + }, + { + "epoch": 0.023860653781913623, + "grad_norm": 1.5546875, + "ht_mnpo/logit": 0.13093149662017822, + "ht_mnpo/residual": 0.1234315037727356, + "ht_mnpo/residual_abs": 0.14939559996128082, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.3333333333333334e-07, + "logits/chosen": -2.1146342754364014, + "logits/rejected": -2.0983939170837402, + "logps/chosen": -0.29903319478034973, + "logps/rejected": -0.29846540093421936, + "loss": 0.27747631072998047, + "loss/core": 0.27747631072998047, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.928014039993286, + "rewards/margins": 1.3093150854110718, + "rewards/rejected": 1.6186988353729248, + "step": 25 + }, + { + "epoch": 0.02863278453829635, + "grad_norm": 12.8125, + "ht_mnpo/logit": 0.22172394394874573, + "ht_mnpo/residual": 0.2142239511013031, + "ht_mnpo/residual_abs": 0.35139912366867065, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.611111111111111e-07, + "logits/chosen": -1.725611686706543, + "logits/rejected": -1.81534743309021, + "logps/chosen": -0.2695452570915222, + "logps/rejected": -0.24922709167003632, + "loss": 1.0329101085662842, + "loss/core": 1.0329101085662842, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 4.7073774337768555, + "rewards/margins": 2.2172398567199707, + "rewards/rejected": 2.490137815475464, + "step": 30 + }, + { + "epoch": 0.03340491529467907, + "grad_norm": 158.0, + "ht_mnpo/logit": -0.06736380606889725, + "ht_mnpo/residual": -0.07486380636692047, + "ht_mnpo/residual_abs": 0.2497224360704422, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8888888888888888e-07, + "logits/chosen": -2.0232789516448975, + "logits/rejected": -1.9483411312103271, + "logps/chosen": -0.30015066266059875, + "logps/rejected": -0.29495859146118164, + "loss": 0.734642744064331, + "loss/core": 0.734642744064331, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.769416093826294, + "rewards/margins": -0.6736379861831665, + "rewards/rejected": 2.44305419921875, + "step": 35 + }, + { + "epoch": 0.0381770460510618, + "grad_norm": 22.625, + "ht_mnpo/logit": 0.06484057754278183, + "ht_mnpo/residual": 0.05734057351946831, + "ht_mnpo/residual_abs": 0.25225889682769775, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.1666666666666667e-07, + "logits/chosen": -2.049146890640259, + "logits/rejected": -2.0784709453582764, + "logps/chosen": -0.2865927219390869, + "logps/rejected": -0.30831488966941833, + "loss": 0.7600542306900024, + "loss/core": 0.7600542306900024, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.3934733867645264, + "rewards/margins": 0.6484058499336243, + "rewards/rejected": 1.7450675964355469, + "step": 40 + }, + { + "epoch": 0.04294917680744453, + "grad_norm": 13.6875, + "ht_mnpo/logit": 0.08705328404903412, + "ht_mnpo/residual": 0.07955329120159149, + "ht_mnpo/residual_abs": 0.145945206284523, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4444444444444445e-07, + "logits/chosen": -1.9960081577301025, + "logits/rejected": -1.9657599925994873, + "logps/chosen": -0.2881585359573364, + "logps/rejected": -0.2911851406097412, + "loss": 0.12006343901157379, + "loss/core": 0.12006343901157379, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.3550467491149902, + "rewards/margins": 0.8705328106880188, + "rewards/rejected": 1.4845139980316162, + "step": 45 + }, + { + "epoch": 0.047721307563827246, + "grad_norm": 43.0, + "ht_mnpo/logit": 0.19554190337657928, + "ht_mnpo/residual": 0.18804192543029785, + "ht_mnpo/residual_abs": 0.19585713744163513, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.7222222222222216e-07, + "logits/chosen": -2.2572484016418457, + "logits/rejected": -2.201063632965088, + "logps/chosen": -0.27145764231681824, + "logps/rejected": -0.2754862606525421, + "loss": 0.4455382823944092, + "loss/core": 0.4455382823944092, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.870077133178711, + "rewards/margins": 1.9554193019866943, + "rewards/rejected": 0.9146578907966614, + "step": 50 + }, + { + "epoch": 0.05249343832020997, + "grad_norm": 10.75, + "ht_mnpo/logit": 0.09912572801113129, + "ht_mnpo/residual": 0.09162574261426926, + "ht_mnpo/residual_abs": 0.10510493814945221, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3e-07, + "logits/chosen": -2.0953731536865234, + "logits/rejected": -2.0514919757843018, + "logps/chosen": -0.2865874171257019, + "logps/rejected": -0.28994354605674744, + "loss": 0.11057962477207184, + "loss/core": 0.11057962477207184, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.285587787628174, + "rewards/margins": 0.99125736951828, + "rewards/rejected": 1.2943300008773804, + "step": 55 + }, + { + "epoch": 0.0572655690765927, + "grad_norm": 620.0, + "ht_mnpo/logit": -0.04805581644177437, + "ht_mnpo/residual": -0.05555582791566849, + "ht_mnpo/residual_abs": 0.25350916385650635, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.2777777777777776e-07, + "logits/chosen": -2.2205891609191895, + "logits/rejected": -2.147505283355713, + "logps/chosen": -0.3061199486255646, + "logps/rejected": -0.30441951751708984, + "loss": 0.8527676463127136, + "loss/core": 0.8527676463127136, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.0320839881896973, + "rewards/margins": -0.48055824637413025, + "rewards/rejected": 2.5126423835754395, + "step": 60 + }, + { + "epoch": 0.062037699832975426, + "grad_norm": 2.75, + "ht_mnpo/logit": 0.10299861431121826, + "ht_mnpo/residual": 0.09549861401319504, + "ht_mnpo/residual_abs": 0.13395439088344574, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.5555555555555553e-07, + "logits/chosen": -2.0961742401123047, + "logits/rejected": -2.160503387451172, + "logps/chosen": -0.30310171842575073, + "logps/rejected": -0.28012874722480774, + "loss": 0.12896214425563812, + "loss/core": 0.12896214425563812, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.2790303230285645, + "rewards/margins": 1.0299861431121826, + "rewards/rejected": 1.2490440607070923, + "step": 65 + }, + { + "epoch": 0.06680983058935815, + "grad_norm": 50.0, + "ht_mnpo/logit": 0.0025874082930386066, + "ht_mnpo/residual": -0.004912590142339468, + "ht_mnpo/residual_abs": 0.15773822367191315, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.8333333333333335e-07, + "logits/chosen": -2.150377035140991, + "logits/rejected": -2.037092924118042, + "logps/chosen": -0.283245325088501, + "logps/rejected": -0.28494200110435486, + "loss": 0.12411870062351227, + "loss/core": 0.12411870062351227, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.957980751991272, + "rewards/margins": 0.02587403729557991, + "rewards/rejected": 1.932106614112854, + "step": 70 + }, + { + "epoch": 0.07158196134574088, + "grad_norm": 808.0, + "ht_mnpo/logit": 0.16834411025047302, + "ht_mnpo/residual": 0.1608441174030304, + "ht_mnpo/residual_abs": 0.20434637367725372, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.1111111111111107e-07, + "logits/chosen": -2.2492072582244873, + "logits/rejected": -2.1484618186950684, + "logps/chosen": -0.27808278799057007, + "logps/rejected": -0.29606252908706665, + "loss": 0.561755359172821, + "loss/core": 0.561755359172821, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.157169818878174, + "rewards/margins": 1.683441162109375, + "rewards/rejected": 1.473728895187378, + "step": 75 + }, + { + "epoch": 0.0763540921021236, + "grad_norm": 3.265625, + "ht_mnpo/logit": 0.13237906992435455, + "ht_mnpo/residual": 0.12487907707691193, + "ht_mnpo/residual_abs": 0.14425772428512573, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.3888888888888884e-07, + "logits/chosen": -2.16731858253479, + "logits/rejected": -2.2766902446746826, + "logps/chosen": -0.2905097007751465, + "logps/rejected": -0.2843090891838074, + "loss": 0.2487218827009201, + "loss/core": 0.2487218827009201, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.213404893875122, + "rewards/margins": 1.3237906694412231, + "rewards/rejected": 0.889613926410675, + "step": 80 + }, + { + "epoch": 0.08112622285850632, + "grad_norm": 0.326171875, + "ht_mnpo/logit": 0.10314911603927612, + "ht_mnpo/residual": 0.09564913064241409, + "ht_mnpo/residual_abs": 0.10694156587123871, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.6666666666666666e-07, + "logits/chosen": -2.077899694442749, + "logits/rejected": -2.1457324028015137, + "logps/chosen": -0.30360931158065796, + "logps/rejected": -0.2841089367866516, + "loss": 0.09745658189058304, + "loss/core": 0.09745658189058304, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9278924465179443, + "rewards/margins": 1.0314911603927612, + "rewards/rejected": 0.8964012265205383, + "step": 85 + }, + { + "epoch": 0.08589835361488905, + "grad_norm": 56.5, + "ht_mnpo/logit": 0.09610621631145477, + "ht_mnpo/residual": 0.08860620856285095, + "ht_mnpo/residual_abs": 0.16850264370441437, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.944444444444445e-07, + "logits/chosen": -2.107797145843506, + "logits/rejected": -2.0631887912750244, + "logps/chosen": -0.3219011723995209, + "logps/rejected": -0.2910079061985016, + "loss": 0.5044264793395996, + "loss/core": 0.5044264793395996, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.2050461769104004, + "rewards/margins": 0.9610620737075806, + "rewards/rejected": 1.2439838647842407, + "step": 90 + }, + { + "epoch": 0.09067048437127177, + "grad_norm": 616.0, + "ht_mnpo/logit": 0.028515029698610306, + "ht_mnpo/residual": 0.021015027537941933, + "ht_mnpo/residual_abs": 0.14724914729595184, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.999699149323369e-07, + "logits/chosen": -2.2570595741271973, + "logits/rejected": -2.347463607788086, + "logps/chosen": -0.32577449083328247, + "logps/rejected": -0.30825769901275635, + "loss": 0.38386693596839905, + "loss/core": 0.38386693596839905, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.8190761804580688, + "rewards/margins": 0.28515028953552246, + "rewards/rejected": 1.5339261293411255, + "step": 95 + }, + { + "epoch": 0.09544261512765449, + "grad_norm": 209.0, + "ht_mnpo/logit": 0.27891913056373596, + "ht_mnpo/residual": 0.27141913771629333, + "ht_mnpo/residual_abs": 0.275497168302536, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.998477067547739e-07, + "logits/chosen": -1.7922618389129639, + "logits/rejected": -1.8475412130355835, + "logps/chosen": -0.3087330460548401, + "logps/rejected": -0.2941829562187195, + "loss": 1.4505915641784668, + "loss/core": 1.4505915641784668, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.890589952468872, + "rewards/margins": 2.789191484451294, + "rewards/rejected": 1.1013987064361572, + "step": 100 + }, + { + "epoch": 0.10021474588403723, + "grad_norm": 0.46875, + "ht_mnpo/logit": 0.09551374614238739, + "ht_mnpo/residual": 0.08801373094320297, + "ht_mnpo/residual_abs": 0.20917508006095886, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.996315410727229e-07, + "logits/chosen": -2.0991432666778564, + "logits/rejected": -2.105532169342041, + "logps/chosen": -0.3069460988044739, + "logps/rejected": -0.32189440727233887, + "loss": 0.5695691108703613, + "loss/core": 0.5695691108703613, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.5684151649475098, + "rewards/margins": 0.9551375508308411, + "rewards/rejected": 1.6132776737213135, + "step": 105 + }, + { + "epoch": 0.10498687664041995, + "grad_norm": 0.7265625, + "ht_mnpo/logit": 0.13444851338863373, + "ht_mnpo/residual": 0.1269485205411911, + "ht_mnpo/residual_abs": 0.14191502332687378, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.993214991772562e-07, + "logits/chosen": -2.226870059967041, + "logits/rejected": -2.155787706375122, + "logps/chosen": -0.30255234241485596, + "logps/rejected": -0.31519246101379395, + "loss": 0.32785916328430176, + "loss/core": 0.32785916328430176, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.3320839405059814, + "rewards/margins": 1.3444854021072388, + "rewards/rejected": 0.987598717212677, + "step": 110 + }, + { + "epoch": 0.10975900739680267, + "grad_norm": 3.9375, + "ht_mnpo/logit": 0.059714823961257935, + "ht_mnpo/residual": 0.05221483111381531, + "ht_mnpo/residual_abs": 0.075630322098732, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.989176976624511e-07, + "logits/chosen": -2.2276268005371094, + "logits/rejected": -2.1443827152252197, + "logps/chosen": -0.2710269093513489, + "logps/rejected": -0.2652212679386139, + "loss": 0.05404385179281235, + "loss/core": 0.05404385179281235, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.458458662033081, + "rewards/margins": 0.5971482992172241, + "rewards/rejected": 0.8613104820251465, + "step": 115 + }, + { + "epoch": 0.1145311381531854, + "grad_norm": 23.25, + "ht_mnpo/logit": 0.10005225241184235, + "ht_mnpo/residual": 0.09255225211381912, + "ht_mnpo/residual_abs": 0.1233731061220169, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.984202883815428e-07, + "logits/chosen": -1.990856409072876, + "logits/rejected": -2.0017800331115723, + "logps/chosen": -0.2697809338569641, + "logps/rejected": -0.2667064368724823, + "loss": 0.08419139683246613, + "loss/core": 0.08419139683246613, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.219022750854492, + "rewards/margins": 1.0005223751068115, + "rewards/rejected": 1.2185002565383911, + "step": 120 + }, + { + "epoch": 0.11930326890956812, + "grad_norm": 792.0, + "ht_mnpo/logit": 0.14017124474048615, + "ht_mnpo/residual": 0.13267125189304352, + "ht_mnpo/residual_abs": 0.1821003258228302, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.978294583898195e-07, + "logits/chosen": -1.970934510231018, + "logits/rejected": -2.0060877799987793, + "logps/chosen": -0.30272430181503296, + "logps/rejected": -0.3132968842983246, + "loss": 0.5580740571022034, + "loss/core": 0.5580740571022034, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 3.1881790161132812, + "rewards/margins": 1.401712417602539, + "rewards/rejected": 1.786466360092163, + "step": 125 + }, + { + "epoch": 0.12407539966595085, + "grad_norm": 2.671875, + "ht_mnpo/logit": 0.11025525629520416, + "ht_mnpo/residual": 0.10275526344776154, + "ht_mnpo/residual_abs": 0.11799643933773041, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.971454298742779e-07, + "logits/chosen": -2.1110129356384277, + "logits/rejected": -2.1078038215637207, + "logps/chosen": -0.33342495560646057, + "logps/rejected": -0.3335180878639221, + "loss": 0.30878788232803345, + "loss/core": 0.30878788232803345, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.003382682800293, + "rewards/margins": 1.1025526523590088, + "rewards/rejected": 0.9008301496505737, + "step": 130 + }, + { + "epoch": 0.12884753042233357, + "grad_norm": 98.5, + "ht_mnpo/logit": 0.010446786880493164, + "ht_mnpo/residual": 0.0029467791318893433, + "ht_mnpo/residual_abs": 0.10158848762512207, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.963684600700678e-07, + "logits/chosen": -2.2710258960723877, + "logits/rejected": -2.1853766441345215, + "logps/chosen": -0.26459789276123047, + "logps/rejected": -0.26528677344322205, + "loss": 0.10957517474889755, + "loss/core": 0.10957517474889755, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.3557019233703613, + "rewards/margins": 0.10446784645318985, + "rewards/rejected": 1.2512340545654297, + "step": 135 + }, + { + "epoch": 0.1336196611787163, + "grad_norm": 284.0, + "ht_mnpo/logit": 0.12432773411273956, + "ht_mnpo/residual": 0.11682772636413574, + "ht_mnpo/residual_abs": 0.16263815760612488, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.954988411637571e-07, + "logits/chosen": -2.1260056495666504, + "logits/rejected": -2.184141159057617, + "logps/chosen": -0.2923770844936371, + "logps/rejected": -0.27389150857925415, + "loss": 0.3137539029121399, + "loss/core": 0.3137539029121399, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.815502643585205, + "rewards/margins": 1.2432773113250732, + "rewards/rejected": 1.5722254514694214, + "step": 140 + }, + { + "epoch": 0.138391791935099, + "grad_norm": 0.52734375, + "ht_mnpo/logit": 0.0494002029299736, + "ht_mnpo/residual": 0.04190019890666008, + "ht_mnpo/residual_abs": 0.0962391048669815, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.945369001834514e-07, + "logits/chosen": -1.9057468175888062, + "logits/rejected": -1.8750278949737549, + "logps/chosen": -0.3120560050010681, + "logps/rejected": -0.3050037920475006, + "loss": 0.07905109971761703, + "loss/core": 0.07905109971761703, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.3618370294570923, + "rewards/margins": 0.49400201439857483, + "rewards/rejected": 0.867834746837616, + "step": 145 + }, + { + "epoch": 0.14316392269148176, + "grad_norm": 81.0, + "ht_mnpo/logit": 0.07038834691047668, + "ht_mnpo/residual": 0.06288834661245346, + "ht_mnpo/residual_abs": 0.19300627708435059, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.93482998875813e-07, + "logits/chosen": -2.0689001083374023, + "logits/rejected": -2.040731906890869, + "logps/chosen": -0.2997715473175049, + "logps/rejected": -0.31551551818847656, + "loss": 0.45480862259864807, + "loss/core": 0.45480862259864807, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.503751516342163, + "rewards/margins": 0.7038835287094116, + "rewards/rejected": 1.7998679876327515, + "step": 150 + }, + { + "epoch": 0.14793605344786448, + "grad_norm": 5.21875, + "ht_mnpo/logit": 0.028568068519234657, + "ht_mnpo/residual": 0.021068070083856583, + "ht_mnpo/residual_abs": 0.12228935956954956, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.923375335700223e-07, + "logits/chosen": -2.1313748359680176, + "logits/rejected": -2.1528689861297607, + "logps/chosen": -0.28180932998657227, + "logps/rejected": -0.27187174558639526, + "loss": 0.2188756912946701, + "loss/core": 0.2188756912946701, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.679626226425171, + "rewards/margins": 0.28568071126937866, + "rewards/rejected": 1.393945574760437, + "step": 155 + }, + { + "epoch": 0.1527081842042472, + "grad_norm": 185.0, + "ht_mnpo/logit": 0.15196958184242249, + "ht_mnpo/residual": 0.14446958899497986, + "ht_mnpo/residual_abs": 0.25919145345687866, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.911009350287347e-07, + "logits/chosen": -2.049853563308716, + "logits/rejected": -2.1069273948669434, + "logps/chosen": -0.28914421796798706, + "logps/rejected": -0.2759200930595398, + "loss": 0.6769053936004639, + "loss/core": 0.6769053936004639, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.205308437347412, + "rewards/margins": 1.51969575881958, + "rewards/rejected": 1.6856123208999634, + "step": 160 + }, + { + "epoch": 0.15748031496062992, + "grad_norm": 2.28125, + "ht_mnpo/logit": 0.15034791827201843, + "ht_mnpo/residual": 0.1428479254245758, + "ht_mnpo/residual_abs": 0.14793327450752258, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.897736682860885e-07, + "logits/chosen": -2.3357603549957275, + "logits/rejected": -2.300586223602295, + "logps/chosen": -0.2931155562400818, + "logps/rejected": -0.27911141514778137, + "loss": 0.2518713176250458, + "loss/core": 0.2518713176250458, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.707345485687256, + "rewards/margins": 1.5034791231155396, + "rewards/rejected": 1.2038663625717163, + "step": 165 + }, + { + "epoch": 0.16225244571701264, + "grad_norm": 1.25, + "ht_mnpo/logit": 0.031133702024817467, + "ht_mnpo/residual": 0.023633703589439392, + "ht_mnpo/residual_abs": 0.09513436257839203, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.883562324728241e-07, + "logits/chosen": -2.1481313705444336, + "logits/rejected": -2.0097994804382324, + "logps/chosen": -0.27028369903564453, + "logps/rejected": -0.2625959813594818, + "loss": 0.09149707853794098, + "loss/core": 0.09149707853794098, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.7255449295043945, + "rewards/margins": 0.3113369941711426, + "rewards/rejected": 1.4142078161239624, + "step": 170 + }, + { + "epoch": 0.16702457647339536, + "grad_norm": 27.625, + "ht_mnpo/logit": 0.07642657309770584, + "ht_mnpo/residual": 0.06892655789852142, + "ht_mnpo/residual_abs": 0.07498085498809814, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.868491606285823e-07, + "logits/chosen": -2.273118257522583, + "logits/rejected": -2.1809802055358887, + "logps/chosen": -0.28101009130477905, + "logps/rejected": -0.2591692805290222, + "loss": 0.06240804120898247, + "loss/core": 0.06240804120898247, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.3919998407363892, + "rewards/margins": 0.7642655968666077, + "rewards/rejected": 0.6277341246604919, + "step": 175 + }, + { + "epoch": 0.1717967072297781, + "grad_norm": 4.59375, + "ht_mnpo/logit": 0.08522136509418488, + "ht_mnpo/residual": 0.07772137224674225, + "ht_mnpo/residual_abs": 0.08434905111789703, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.852530195014489e-07, + "logits/chosen": -2.163783550262451, + "logits/rejected": -2.2361397743225098, + "logps/chosen": -0.29033127427101135, + "logps/rejected": -0.27159175276756287, + "loss": 0.03848674148321152, + "loss/core": 0.03848674148321152, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 1.901350975036621, + "rewards/margins": 0.8522136807441711, + "rewards/rejected": 1.0491373538970947, + "step": 180 + }, + { + "epoch": 0.17656883798616083, + "grad_norm": 2.015625, + "ht_mnpo/logit": 0.12886855006217957, + "ht_mnpo/residual": 0.12136852741241455, + "ht_mnpo/residual_abs": 0.12450716644525528, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.835684093348244e-07, + "logits/chosen": -1.9544246196746826, + "logits/rejected": -2.0396523475646973, + "logps/chosen": -0.2824137806892395, + "logps/rejected": -0.2820433974266052, + "loss": 0.1304716169834137, + "loss/core": 0.1304716169834137, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.1400837898254395, + "rewards/margins": 1.2886853218078613, + "rewards/rejected": 0.8513985872268677, + "step": 185 + }, + { + "epoch": 0.18134096874254355, + "grad_norm": 2.234375, + "ht_mnpo/logit": 0.05361006781458855, + "ht_mnpo/residual": 0.04611007869243622, + "ht_mnpo/residual_abs": 0.06801304221153259, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.817959636416969e-07, + "logits/chosen": -2.177982807159424, + "logits/rejected": -2.2904984951019287, + "logps/chosen": -0.29912227392196655, + "logps/rejected": -0.2927282452583313, + "loss": 0.02491976134479046, + "loss/core": 0.02491976134479046, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.471571445465088, + "rewards/margins": 0.5361008048057556, + "rewards/rejected": 0.9354707598686218, + "step": 190 + }, + { + "epoch": 0.18611309949892627, + "grad_norm": 15.5, + "ht_mnpo/logit": 0.16110555827617645, + "ht_mnpo/residual": 0.15360556542873383, + "ht_mnpo/residual_abs": 0.1779450923204422, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.799363489664039e-07, + "logits/chosen": -2.0854153633117676, + "logits/rejected": -2.057650089263916, + "logps/chosen": -0.2939828038215637, + "logps/rejected": -0.2942020893096924, + "loss": 0.16689643263816833, + "loss/core": 0.16689643263816833, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.914524793624878, + "rewards/margins": 1.611055612564087, + "rewards/rejected": 1.303469181060791, + "step": 195 + }, + { + "epoch": 0.19088523025530899, + "grad_norm": 2.859375, + "ht_mnpo/logit": 0.10722651332616806, + "ht_mnpo/residual": 0.09972651302814484, + "ht_mnpo/residual_abs": 0.10703112930059433, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.779902646339721e-07, + "logits/chosen": -2.2011075019836426, + "logits/rejected": -2.1862375736236572, + "logps/chosen": -0.29882553219795227, + "logps/rejected": -0.30318737030029297, + "loss": 0.23726752400398254, + "loss/core": 0.23726752400398254, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 2.2529048919677734, + "rewards/margins": 1.072265386581421, + "rewards/rejected": 1.1806399822235107, + "step": 200 + }, + { + "epoch": 0.19565736101169173, + "grad_norm": 11.4375, + "ht_mnpo/logit": 0.053003937005996704, + "ht_mnpo/residual": 0.04550394043326378, + "ht_mnpo/residual_abs": 0.07750043272972107, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.759584424871301e-07, + "logits/chosen": -1.9984384775161743, + "logits/rejected": -2.088771343231201, + "logps/chosen": -0.3163023591041565, + "logps/rejected": -0.30901581048965454, + "loss": 0.04021575301885605, + "loss/core": 0.04021575301885605, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.633466124534607, + "rewards/margins": 0.5300394296646118, + "rewards/rejected": 1.1034266948699951, + "step": 205 + }, + { + "epoch": 0.20042949176807445, + "grad_norm": 12.75, + "ht_mnpo/logit": 0.1604582965373993, + "ht_mnpo/residual": 0.15295830368995667, + "ht_mnpo/residual_abs": 0.17036454379558563, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.738416466110917e-07, + "logits/chosen": -1.9503463506698608, + "logits/rejected": -1.9217140674591064, + "logps/chosen": -0.2880111336708069, + "logps/rejected": -0.2948765754699707, + "loss": 0.2796785235404968, + "loss/core": 0.2796785235404968, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.1876330375671387, + "rewards/margins": 1.6045830249786377, + "rewards/rejected": 1.5830501317977905, + "step": 210 + }, + { + "epoch": 0.20520162252445717, + "grad_norm": 1.0546875, + "ht_mnpo/logit": 0.084672711789608, + "ht_mnpo/residual": 0.07717271149158478, + "ht_mnpo/residual_abs": 0.1215781569480896, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.716406730462153e-07, + "logits/chosen": -2.1271862983703613, + "logits/rejected": -2.171344041824341, + "logps/chosen": -0.2948567271232605, + "logps/rejected": -0.2792275547981262, + "loss": 0.18875925242900848, + "loss/core": 0.18875925242900848, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.0924696922302246, + "rewards/margins": 0.8467270135879517, + "rewards/rejected": 1.2457425594329834, + "step": 215 + }, + { + "epoch": 0.2099737532808399, + "grad_norm": 0.76171875, + "ht_mnpo/logit": 0.06622101366519928, + "ht_mnpo/residual": 0.058721017092466354, + "ht_mnpo/residual_abs": 0.08079180121421814, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.693563494886454e-07, + "logits/chosen": -2.091046094894409, + "logits/rejected": -2.2000317573547363, + "logps/chosen": -0.32790035009384155, + "logps/rejected": -0.3206120431423187, + "loss": 0.08376762270927429, + "loss/core": 0.08376762270927429, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.9445091485977173, + "rewards/margins": 0.66221022605896, + "rewards/rejected": 1.2822988033294678, + "step": 220 + }, + { + "epoch": 0.2147458840372226, + "grad_norm": 0.94140625, + "ht_mnpo/logit": 0.15462861955165863, + "ht_mnpo/residual": 0.1471286118030548, + "ht_mnpo/residual_abs": 0.17850852012634277, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.6698953497905016e-07, + "logits/chosen": -2.1533384323120117, + "logits/rejected": -2.2266592979431152, + "logps/chosen": -0.25487151741981506, + "logps/rejected": -0.23334865272045135, + "loss": 0.24176493287086487, + "loss/core": 0.24176493287086487, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.2291808128356934, + "rewards/margins": 1.5462863445281982, + "rewards/rejected": 1.6828947067260742, + "step": 225 + }, + { + "epoch": 0.21951801479360533, + "grad_norm": 4.625, + "ht_mnpo/logit": 0.05176293104887009, + "ht_mnpo/residual": 0.04426293820142746, + "ht_mnpo/residual_abs": 0.0576145239174366, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.645411195795709e-07, + "logits/chosen": -2.076111316680908, + "logits/rejected": -2.167879581451416, + "logps/chosen": -0.32002463936805725, + "logps/rejected": -0.29538848996162415, + "loss": 0.014069770462810993, + "loss/core": 0.014069770462810993, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.1483262777328491, + "rewards/margins": 0.5176293253898621, + "rewards/rejected": 0.6306968927383423, + "step": 230 + }, + { + "epoch": 0.22429014554998808, + "grad_norm": 4.34375, + "ht_mnpo/logit": 0.06035786122083664, + "ht_mnpo/residual": 0.05285785719752312, + "ht_mnpo/residual_abs": 0.1479470431804657, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.6201202403910643e-07, + "logits/chosen": -2.0307183265686035, + "logits/rejected": -2.164533853530884, + "logps/chosen": -0.30357375741004944, + "logps/rejected": -0.30138644576072693, + "loss": 0.30801814794540405, + "loss/core": 0.30801814794540405, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.844181776046753, + "rewards/margins": 0.6035785675048828, + "rewards/rejected": 1.2406030893325806, + "step": 235 + }, + { + "epoch": 0.2290622763063708, + "grad_norm": 3.0, + "ht_mnpo/logit": 0.05508917570114136, + "ht_mnpo/residual": 0.04758917912840843, + "ht_mnpo/residual_abs": 0.062050361186265945, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.594031994470573e-07, + "logits/chosen": -2.1817893981933594, + "logits/rejected": -2.206329345703125, + "logps/chosen": -0.2946377098560333, + "logps/rejected": -0.2780408263206482, + "loss": 0.011683464981615543, + "loss/core": 0.011683464981615543, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.5235569477081299, + "rewards/margins": 0.5508917570114136, + "rewards/rejected": 0.9726651310920715, + "step": 240 + }, + { + "epoch": 0.23383440706275352, + "grad_norm": 2.21875, + "ht_mnpo/logit": 0.03650509566068649, + "ht_mnpo/residual": 0.02900509163737297, + "ht_mnpo/residual_abs": 0.11014048010110855, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.567156268756593e-07, + "logits/chosen": -2.124574899673462, + "logits/rejected": -2.221391439437866, + "logps/chosen": -0.2924458384513855, + "logps/rejected": -0.27032575011253357, + "loss": 0.13753008842468262, + "loss/core": 0.13753008842468262, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.5719541311264038, + "rewards/margins": 0.36505094170570374, + "rewards/rejected": 1.2069032192230225, + "step": 245 + }, + { + "epoch": 0.23860653781913624, + "grad_norm": 47.25, + "ht_mnpo/logit": 0.20946970582008362, + "ht_mnpo/residual": 0.2019696980714798, + "ht_mnpo/residual_abs": 0.20413513481616974, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.5395031701104303e-07, + "logits/chosen": -1.8938868045806885, + "logits/rejected": -1.9979091882705688, + "logps/chosen": -0.3038504123687744, + "logps/rejected": -0.2779809534549713, + "loss": 0.3277028501033783, + "loss/core": 0.3277028501033783, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.351757526397705, + "rewards/margins": 2.0946967601776123, + "rewards/rejected": 1.2570606470108032, + "step": 250 + }, + { + "epoch": 0.24337866857551896, + "grad_norm": 4.5625, + "ht_mnpo/logit": 0.10556943714618683, + "ht_mnpo/residual": 0.0980694368481636, + "ht_mnpo/residual_abs": 0.13419580459594727, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.511083097731555e-07, + "logits/chosen": -1.8236820697784424, + "logits/rejected": -1.8897043466567993, + "logps/chosen": -0.298318088054657, + "logps/rejected": -0.3006042242050171, + "loss": 0.1540885865688324, + "loss/core": 0.1540885865688324, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.6379857063293457, + "rewards/margins": 1.055694341659546, + "rewards/rejected": 1.5822908878326416, + "step": 255 + }, + { + "epoch": 0.2481507993319017, + "grad_norm": 1416.0, + "ht_mnpo/logit": 0.21744589507579803, + "ht_mnpo/residual": 0.2099459171295166, + "ht_mnpo/residual_abs": 0.2343859225511551, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.481906739246894e-07, + "logits/chosen": -2.1703085899353027, + "logits/rejected": -2.276538372039795, + "logps/chosen": -0.30491483211517334, + "logps/rejected": -0.2938116788864136, + "loss": 1.0860064029693604, + "loss/core": 1.0860064029693604, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.2028307914733887, + "rewards/margins": 2.1744587421417236, + "rewards/rejected": 1.028372049331665, + "step": 260 + }, + { + "epoch": 0.2529229300882844, + "grad_norm": 33.25, + "ht_mnpo/logit": 0.03215058147907257, + "ht_mnpo/residual": 0.0246505755931139, + "ht_mnpo/residual_abs": 0.195493683218956, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.451985066691648e-07, + "logits/chosen": -2.069486141204834, + "logits/rejected": -2.1000301837921143, + "logps/chosen": -0.28381848335266113, + "logps/rejected": -0.2866091728210449, + "loss": 0.5351065397262573, + "loss/core": 0.5351065397262573, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.2341432571411133, + "rewards/margins": 0.3215058445930481, + "rewards/rejected": 1.912637710571289, + "step": 265 + }, + { + "epoch": 0.25769506084466715, + "grad_norm": 207.0, + "ht_mnpo/logit": 0.032596416771411896, + "ht_mnpo/residual": 0.025096416473388672, + "ht_mnpo/residual_abs": 0.2024488002061844, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.421329332383158e-07, + "logits/chosen": -2.0715479850769043, + "logits/rejected": -2.1490814685821533, + "logps/chosen": -0.29047778248786926, + "logps/rejected": -0.3188256025314331, + "loss": 0.5344532132148743, + "loss/core": 0.5344532132148743, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.0660104751586914, + "rewards/margins": 0.3259641230106354, + "rewards/rejected": 1.740046501159668, + "step": 270 + }, + { + "epoch": 0.26246719160104987, + "grad_norm": 6.84375, + "ht_mnpo/logit": 0.0682886391878128, + "ht_mnpo/residual": 0.06078863888978958, + "ht_mnpo/residual_abs": 0.09558697044849396, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.3899510646893696e-07, + "logits/chosen": -2.068638324737549, + "logits/rejected": -2.1749560832977295, + "logps/chosen": -0.29401373863220215, + "logps/rejected": -0.278691828250885, + "loss": 0.05401838943362236, + "loss/core": 0.05401838943362236, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 1.9330079555511475, + "rewards/margins": 0.6828864812850952, + "rewards/rejected": 1.250121831893921, + "step": 275 + }, + { + "epoch": 0.2672393223574326, + "grad_norm": 98.0, + "ht_mnpo/logit": 0.11557488143444061, + "ht_mnpo/residual": 0.10807488113641739, + "ht_mnpo/residual_abs": 0.22392027080059052, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.357862063693485e-07, + "logits/chosen": -2.0900626182556152, + "logits/rejected": -2.1388344764709473, + "logps/chosen": -0.30804580450057983, + "logps/rejected": -0.30006811022758484, + "loss": 0.4923165738582611, + "loss/core": 0.4923165738582611, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.7162301540374756, + "rewards/margins": 1.155748963356018, + "rewards/rejected": 1.560481309890747, + "step": 280 + }, + { + "epoch": 0.2720114531138153, + "grad_norm": 1.671875, + "ht_mnpo/logit": 0.007366104517132044, + "ht_mnpo/residual": -0.00013389457308221608, + "ht_mnpo/residual_abs": 0.10880545526742935, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.3250743967564364e-07, + "logits/chosen": -2.2310268878936768, + "logits/rejected": -2.114835262298584, + "logps/chosen": -0.29032495617866516, + "logps/rejected": -0.3384189009666443, + "loss": 0.08280204236507416, + "loss/core": 0.08280204236507416, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.7172960042953491, + "rewards/margins": 0.07366105169057846, + "rewards/rejected": 1.6436350345611572, + "step": 285 + }, + { + "epoch": 0.276783583870198, + "grad_norm": 44.75, + "ht_mnpo/logit": 0.06611434370279312, + "ht_mnpo/residual": 0.0586143359541893, + "ht_mnpo/residual_abs": 0.07593923807144165, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.29160039397884e-07, + "logits/chosen": -2.26133394241333, + "logits/rejected": -2.217014789581299, + "logps/chosen": -0.3368655741214752, + "logps/rejected": -0.3344263434410095, + "loss": 0.10027849674224854, + "loss/core": 0.10027849674224854, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.0940282344818115, + "rewards/margins": 0.6611433625221252, + "rewards/rejected": 0.4328847825527191, + "step": 290 + }, + { + "epoch": 0.28155571462658074, + "grad_norm": 147.0, + "ht_mnpo/logit": 0.13316653668880463, + "ht_mnpo/residual": 0.125666543841362, + "ht_mnpo/residual_abs": 0.14974115788936615, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.2574526435641546e-07, + "logits/chosen": -2.052053689956665, + "logits/rejected": -2.149163007736206, + "logps/chosen": -0.30694466829299927, + "logps/rejected": -0.2819734215736389, + "loss": 0.289144366979599, + "loss/core": 0.289144366979599, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.095669746398926, + "rewards/margins": 1.3316655158996582, + "rewards/rejected": 0.7640042901039124, + "step": 295 + }, + { + "epoch": 0.2863278453829635, + "grad_norm": 5.84375, + "ht_mnpo/logit": 0.09821562469005585, + "ht_mnpo/residual": 0.09071563184261322, + "ht_mnpo/residual_abs": 0.1706034541130066, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.22264398708477e-07, + "logits/chosen": -2.054267168045044, + "logits/rejected": -2.129446506500244, + "logps/chosen": -0.2598414719104767, + "logps/rejected": -0.25766080617904663, + "loss": 0.1819196343421936, + "loss/core": 0.1819196343421936, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.9472148418426514, + "rewards/margins": 0.9821560978889465, + "rewards/rejected": 1.96505868434906, + "step": 300 + }, + { + "epoch": 0.29109997613934624, + "grad_norm": 6.3125, + "ht_mnpo/logit": 0.08229192346334457, + "ht_mnpo/residual": 0.07479192316532135, + "ht_mnpo/residual_abs": 0.08601853251457214, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.187187514652819e-07, + "logits/chosen": -1.9865754842758179, + "logits/rejected": -2.013413429260254, + "logps/chosen": -0.3034398555755615, + "logps/rejected": -0.3090708255767822, + "loss": 0.037676356732845306, + "loss/core": 0.037676356732845306, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.2789227962493896, + "rewards/margins": 0.8229190707206726, + "rewards/rejected": 1.4560037851333618, + "step": 305 + }, + { + "epoch": 0.29587210689572896, + "grad_norm": 524.0, + "ht_mnpo/logit": 0.09432446211576462, + "ht_mnpo/residual": 0.0868244618177414, + "ht_mnpo/residual_abs": 0.22896528244018555, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.151096559997519e-07, + "logits/chosen": -1.9858556985855103, + "logits/rejected": -1.9708316326141357, + "logps/chosen": -0.28563809394836426, + "logps/rejected": -0.29865071177482605, + "loss": 0.6932410597801208, + "loss/core": 0.6932410597801208, + "rewards/accuracies": 0.7749999761581421, + "rewards/chosen": 2.816760540008545, + "rewards/margins": 0.9432447552680969, + "rewards/rejected": 1.8735158443450928, + "step": 310 + }, + { + "epoch": 0.3006442376521117, + "grad_norm": 46.75, + "ht_mnpo/logit": 0.0416237898170948, + "ht_mnpo/residual": 0.03412378951907158, + "ht_mnpo/residual_abs": 0.15009064972400665, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.114384695450905e-07, + "logits/chosen": -2.215585231781006, + "logits/rejected": -2.188673496246338, + "logps/chosen": -0.31491440534591675, + "logps/rejected": -0.2965540885925293, + "loss": 0.20979872345924377, + "loss/core": 0.20979872345924377, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.8721401691436768, + "rewards/margins": 0.4162379205226898, + "rewards/rejected": 1.4559022188186646, + "step": 315 + }, + { + "epoch": 0.3054163684084944, + "grad_norm": 21.125, + "ht_mnpo/logit": 0.10669668018817902, + "ht_mnpo/residual": 0.09919667989015579, + "ht_mnpo/residual_abs": 0.13064010441303253, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.077065726843828e-07, + "logits/chosen": -2.0546205043792725, + "logits/rejected": -1.999367356300354, + "logps/chosen": -0.28702324628829956, + "logps/rejected": -0.2923528552055359, + "loss": 0.08457665145397186, + "loss/core": 0.08457665145397186, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.4331345558166504, + "rewards/margins": 1.0669667720794678, + "rewards/rejected": 1.366167664527893, + "step": 320 + }, + { + "epoch": 0.3101884991648771, + "grad_norm": 4.21875, + "ht_mnpo/logit": 0.09450945258140564, + "ht_mnpo/residual": 0.08700944483280182, + "ht_mnpo/residual_abs": 0.09081277996301651, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.039153688314145e-07, + "logits/chosen": -1.9875255823135376, + "logits/rejected": -2.1427712440490723, + "logps/chosen": -0.29921954870224, + "logps/rejected": -0.2899077832698822, + "loss": 0.07706113159656525, + "loss/core": 0.07706113159656525, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.703743577003479, + "rewards/margins": 0.9450944662094116, + "rewards/rejected": 0.7586489319801331, + "step": 325 + }, + { + "epoch": 0.31496062992125984, + "grad_norm": 3.0, + "ht_mnpo/logit": 0.07892023026943207, + "ht_mnpo/residual": 0.07142021507024765, + "ht_mnpo/residual_abs": 0.08247885853052139, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.0006628370290613e-07, + "logits/chosen": -2.087024211883545, + "logits/rejected": -2.1432571411132812, + "logps/chosen": -0.3117908835411072, + "logps/rejected": -0.2991257607936859, + "loss": 0.06879410147666931, + "loss/core": 0.06879410147666931, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.5523658990859985, + "rewards/margins": 0.7892022132873535, + "rewards/rejected": 0.7631637454032898, + "step": 330 + }, + { + "epoch": 0.31973276067764256, + "grad_norm": 1.578125, + "ht_mnpo/logit": 0.100077785551548, + "ht_mnpo/residual": 0.09257779270410538, + "ht_mnpo/residual_abs": 0.15659311413764954, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.9616076478235826e-07, + "logits/chosen": -2.0467448234558105, + "logits/rejected": -1.9432637691497803, + "logps/chosen": -0.3127209544181824, + "logps/rejected": -0.3082302212715149, + "loss": 0.19428648054599762, + "loss/core": 0.19428648054599762, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.354982852935791, + "rewards/margins": 1.0007779598236084, + "rewards/rejected": 1.354204773902893, + "step": 335 + }, + { + "epoch": 0.3245048914340253, + "grad_norm": 1.1484375, + "ht_mnpo/logit": 0.011610162444412708, + "ht_mnpo/residual": 0.004110163543373346, + "ht_mnpo/residual_abs": 0.14836415648460388, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.922002807757129e-07, + "logits/chosen": -2.0763049125671387, + "logits/rejected": -2.044464111328125, + "logps/chosen": -0.31426161527633667, + "logps/rejected": -0.3251628875732422, + "loss": 0.24287572503089905, + "loss/core": 0.24287572503089905, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.9379974603652954, + "rewards/margins": 0.11610164493322372, + "rewards/rejected": 1.821895956993103, + "step": 340 + }, + { + "epoch": 0.329277022190408, + "grad_norm": 139.0, + "ht_mnpo/logit": 0.14599408209323883, + "ht_mnpo/residual": 0.1384941041469574, + "ht_mnpo/residual_abs": 0.22581353783607483, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.8818632105903315e-07, + "logits/chosen": -1.9784504175186157, + "logits/rejected": -2.0479235649108887, + "logps/chosen": -0.3185192346572876, + "logps/rejected": -0.2856663763523102, + "loss": 0.476330041885376, + "loss/core": 0.476330041885376, + "rewards/accuracies": 0.925000011920929, + "rewards/chosen": 3.1129283905029297, + "rewards/margins": 1.4599411487579346, + "rewards/rejected": 1.6529871225357056, + "step": 345 + }, + { + "epoch": 0.3340491529467907, + "grad_norm": 121.5, + "ht_mnpo/logit": 0.17999222874641418, + "ht_mnpo/residual": 0.17249223589897156, + "ht_mnpo/residual_abs": 0.1953013390302658, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.841203951184094e-07, + "logits/chosen": -2.32238507270813, + "logits/rejected": -2.366455078125, + "logps/chosen": -0.2689044177532196, + "logps/rejected": -0.2708611488342285, + "loss": 0.45748645067214966, + "loss/core": 0.45748645067214966, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.1744790077209473, + "rewards/margins": 1.799922227859497, + "rewards/rejected": 1.3745567798614502, + "step": 350 + }, + { + "epoch": 0.3388212837031735, + "grad_norm": 772.0, + "ht_mnpo/logit": 0.06556814908981323, + "ht_mnpo/residual": 0.058068156242370605, + "ht_mnpo/residual_abs": 0.15479174256324768, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.800040319823038e-07, + "logits/chosen": -2.154000997543335, + "logits/rejected": -2.145259141921997, + "logps/chosen": -0.2757951617240906, + "logps/rejected": -0.27257150411605835, + "loss": 0.21897700428962708, + "loss/core": 0.21897700428962708, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.779329538345337, + "rewards/margins": 0.6556814908981323, + "rewards/rejected": 2.123647928237915, + "step": 355 + }, + { + "epoch": 0.3435934144595562, + "grad_norm": 776.0, + "ht_mnpo/logit": 0.1693677008152008, + "ht_mnpo/residual": 0.16186770796775818, + "ht_mnpo/residual_abs": 0.19508880376815796, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.75838779646545e-07, + "logits/chosen": -1.8399804830551147, + "logits/rejected": -1.8790878057479858, + "logps/chosen": -0.296386182308197, + "logps/rejected": -0.31673291325569153, + "loss": 0.43224653601646423, + "loss/core": 0.43224653601646423, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.0844686031341553, + "rewards/margins": 1.6936769485473633, + "rewards/rejected": 1.390791654586792, + "step": 360 + }, + { + "epoch": 0.34836554521593893, + "grad_norm": 27.75, + "ht_mnpo/logit": 0.04867623373866081, + "ht_mnpo/residual": 0.04117623716592789, + "ht_mnpo/residual_abs": 0.04830501973628998, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.7162620449218993e-07, + "logits/chosen": -2.164468765258789, + "logits/rejected": -2.201307773590088, + "logps/chosen": -0.2984023094177246, + "logps/rejected": -0.29477953910827637, + "loss": 0.021916760131716728, + "loss/core": 0.021916760131716728, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.2955366373062134, + "rewards/margins": 0.4867623746395111, + "rewards/rejected": 0.8087741732597351, + "step": 365 + }, + { + "epoch": 0.35313767597232165, + "grad_norm": 7.90625, + "ht_mnpo/logit": 0.07078660279512405, + "ht_mnpo/residual": 0.06328661739826202, + "ht_mnpo/residual_abs": 0.19043698906898499, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.673678906964727e-07, + "logits/chosen": -2.1319940090179443, + "logits/rejected": -2.105229616165161, + "logps/chosen": -0.2718384563922882, + "logps/rejected": -0.26169219613075256, + "loss": 0.5162893533706665, + "loss/core": 0.5162893533706665, + "rewards/accuracies": 0.75, + "rewards/chosen": 2.2212352752685547, + "rewards/margins": 0.7078661918640137, + "rewards/rejected": 1.5133693218231201, + "step": 370 + }, + { + "epoch": 0.35790980672870437, + "grad_norm": 25.125, + "ht_mnpo/logit": 0.13367705047130585, + "ht_mnpo/residual": 0.12617704272270203, + "ht_mnpo/residual_abs": 0.16303515434265137, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.6306543963705936e-07, + "logits/chosen": -2.170954465866089, + "logits/rejected": -2.2756118774414062, + "logps/chosen": -0.259833425283432, + "logps/rejected": -0.2742316722869873, + "loss": 0.2501140236854553, + "loss/core": 0.2501140236854553, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.697831869125366, + "rewards/margins": 1.3367704153060913, + "rewards/rejected": 1.3610615730285645, + "step": 375 + }, + { + "epoch": 0.3626819374850871, + "grad_norm": 8.0625, + "ht_mnpo/logit": 0.24639646708965302, + "ht_mnpo/residual": 0.2388964593410492, + "ht_mnpo/residual_abs": 0.250093549489975, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.5872046928983623e-07, + "logits/chosen": -1.9891340732574463, + "logits/rejected": -2.001060962677002, + "logps/chosen": -0.33623114228248596, + "logps/rejected": -0.2880457043647766, + "loss": 0.6706468462944031, + "loss/core": 0.6706468462944031, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.5934009552001953, + "rewards/margins": 2.4639649391174316, + "rewards/rejected": 1.1294362545013428, + "step": 380 + }, + { + "epoch": 0.3674540682414698, + "grad_norm": 16.125, + "ht_mnpo/logit": 0.07829246670007706, + "ht_mnpo/residual": 0.07079247385263443, + "ht_mnpo/residual_abs": 0.0979897528886795, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.5433461362045447e-07, + "logits/chosen": -2.110302686691284, + "logits/rejected": -2.122204303741455, + "logps/chosen": -0.2984893321990967, + "logps/rejected": -0.2900088429450989, + "loss": 0.058689720928668976, + "loss/core": 0.058689720928668976, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.8839489221572876, + "rewards/margins": 0.7829247117042542, + "rewards/rejected": 1.1010242700576782, + "step": 385 + }, + { + "epoch": 0.37222619899785253, + "grad_norm": 3.921875, + "ht_mnpo/logit": 0.0583811029791832, + "ht_mnpo/residual": 0.05088110640645027, + "ht_mnpo/residual_abs": 0.11353194713592529, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.4990952196986305e-07, + "logits/chosen": -2.2036306858062744, + "logits/rejected": -2.26830792427063, + "logps/chosen": -0.28720030188560486, + "logps/rejected": -0.29090017080307007, + "loss": 0.10279025137424469, + "loss/core": 0.10279025137424469, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.0018017292022705, + "rewards/margins": 0.5838111042976379, + "rewards/rejected": 1.4179904460906982, + "step": 390 + }, + { + "epoch": 0.37699832975423525, + "grad_norm": 231.0, + "ht_mnpo/logit": 0.23413963615894318, + "ht_mnpo/residual": 0.22663962841033936, + "ht_mnpo/residual_abs": 0.2551414966583252, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.4544685843405875e-07, + "logits/chosen": -2.015768051147461, + "logits/rejected": -2.0945444107055664, + "logps/chosen": -0.27710121870040894, + "logps/rejected": -0.2746776044368744, + "loss": 0.4984128475189209, + "loss/core": 0.4984128475189209, + "rewards/accuracies": 0.8125, + "rewards/chosen": 4.204476356506348, + "rewards/margins": 2.3413960933685303, + "rewards/rejected": 1.863080620765686, + "step": 395 + }, + { + "epoch": 0.38177046051061797, + "grad_norm": 4.53125, + "ht_mnpo/logit": 0.09178599715232849, + "ht_mnpo/residual": 0.08428598940372467, + "ht_mnpo/residual_abs": 0.09066607058048248, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.4094830123828786e-07, + "logits/chosen": -2.175234317779541, + "logits/rejected": -2.227020025253296, + "logps/chosen": -0.3004263937473297, + "logps/rejected": -0.2858661711215973, + "loss": 0.07631814479827881, + "loss/core": 0.07631814479827881, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.299661636352539, + "rewards/margins": 0.9178601503372192, + "rewards/rejected": 1.3818013668060303, + "step": 400 + }, + { + "epoch": 0.3865425912670007, + "grad_norm": 9.1875, + "ht_mnpo/logit": 0.1291302740573883, + "ht_mnpo/residual": 0.12163027375936508, + "ht_mnpo/residual_abs": 0.1272890567779541, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.3641554210593414e-07, + "logits/chosen": -2.067502737045288, + "logits/rejected": -2.1148393154144287, + "logps/chosen": -0.2825814187526703, + "logps/rejected": -0.2730342745780945, + "loss": 0.23866212368011475, + "loss/core": 0.23866212368011475, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.0164496898651123, + "rewards/margins": 1.2913026809692383, + "rewards/rejected": 0.7251466512680054, + "step": 405 + }, + { + "epoch": 0.39131472202338347, + "grad_norm": 17.25, + "ht_mnpo/logit": 0.07708557695150375, + "ht_mnpo/residual": 0.06958558410406113, + "ht_mnpo/residual_abs": 0.1799410581588745, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.3185028562233107e-07, + "logits/chosen": -2.0323519706726074, + "logits/rejected": -2.03326678276062, + "logps/chosen": -0.29431599378585815, + "logps/rejected": -0.29893699288368225, + "loss": 0.2613184452056885, + "loss/core": 0.2613184452056885, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.014284133911133, + "rewards/margins": 0.7708557844161987, + "rewards/rejected": 1.2434285879135132, + "step": 410 + }, + { + "epoch": 0.3960868527797662, + "grad_norm": 3.421875, + "ht_mnpo/logit": 0.18855980038642883, + "ht_mnpo/residual": 0.18105976283550262, + "ht_mnpo/residual_abs": 0.19883492588996887, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.272542485937368e-07, + "logits/chosen": -1.9817771911621094, + "logits/rejected": -2.0678811073303223, + "logps/chosen": -0.3057534992694855, + "logps/rejected": -0.2749868631362915, + "loss": 0.6331941485404968, + "loss/core": 0.6331941485404968, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 3.0550625324249268, + "rewards/margins": 1.8855979442596436, + "rewards/rejected": 1.1694644689559937, + "step": 415 + }, + { + "epoch": 0.4008589835361489, + "grad_norm": 21.75, + "ht_mnpo/logit": 0.13565869629383087, + "ht_mnpo/residual": 0.12815868854522705, + "ht_mnpo/residual_abs": 0.13334575295448303, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.226291594017137e-07, + "logits/chosen": -2.0041332244873047, + "logits/rejected": -2.0797832012176514, + "logps/chosen": -0.2827741801738739, + "logps/rejected": -0.269229531288147, + "loss": 0.09638514369726181, + "loss/core": 0.09638514369726181, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.2633628845214844, + "rewards/margins": 1.3565869331359863, + "rewards/rejected": 0.906775951385498, + "step": 420 + }, + { + "epoch": 0.4056311142925316, + "grad_norm": 896.0, + "ht_mnpo/logit": 0.15091469883918762, + "ht_mnpo/residual": 0.143414705991745, + "ht_mnpo/residual_abs": 0.16229206323623657, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.1797675735315454e-07, + "logits/chosen": -2.006406784057617, + "logits/rejected": -2.115539073944092, + "logps/chosen": -0.29817336797714233, + "logps/rejected": -0.28226596117019653, + "loss": 0.5429047346115112, + "loss/core": 0.5429047346115112, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.30177640914917, + "rewards/margins": 1.509147047996521, + "rewards/rejected": 0.7926293611526489, + "step": 425 + }, + { + "epoch": 0.41040324504891434, + "grad_norm": 138.0, + "ht_mnpo/logit": 0.09805858135223389, + "ht_mnpo/residual": 0.09055860340595245, + "ht_mnpo/residual_abs": 0.1379367709159851, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.1329879202620047e-07, + "logits/chosen": -2.42768931388855, + "logits/rejected": -2.3442633152008057, + "logps/chosen": -0.28302669525146484, + "logps/rejected": -0.28555774688720703, + "loss": 0.2293454110622406, + "loss/core": 0.2293454110622406, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.095808267593384, + "rewards/margins": 0.9805858731269836, + "rewards/rejected": 1.1152220964431763, + "step": 430 + }, + { + "epoch": 0.41517537580529706, + "grad_norm": 160.0, + "ht_mnpo/logit": 0.1208895891904831, + "ht_mnpo/residual": 0.11338958889245987, + "ht_mnpo/residual_abs": 0.20339921116828918, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.0859702261229613e-07, + "logits/chosen": -2.232008457183838, + "logits/rejected": -2.221562147140503, + "logps/chosen": -0.2803706228733063, + "logps/rejected": -0.29765453934669495, + "loss": 0.9300113916397095, + "loss/core": 0.9300113916397095, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.4720828533172607, + "rewards/margins": 1.2088956832885742, + "rewards/rejected": 1.263187289237976, + "step": 435 + }, + { + "epoch": 0.4199475065616798, + "grad_norm": 25.75, + "ht_mnpo/logit": 0.09195519983768463, + "ht_mnpo/residual": 0.08445519208908081, + "ht_mnpo/residual_abs": 0.12991644442081451, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.0387321725463e-07, + "logits/chosen": -2.333383083343506, + "logits/rejected": -2.346522808074951, + "logps/chosen": -0.29035550355911255, + "logps/rejected": -0.27913159132003784, + "loss": 0.1505986452102661, + "loss/core": 0.1505986452102661, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.3275554180145264, + "rewards/margins": 0.9195519685745239, + "rewards/rejected": 1.4080034494400024, + "step": 440 + }, + { + "epoch": 0.4247196373180625, + "grad_norm": 7.0, + "ht_mnpo/logit": 0.05209118872880936, + "ht_mnpo/residual": 0.04459118843078613, + "ht_mnpo/residual_abs": 0.04903823882341385, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.991291523832075e-07, + "logits/chosen": -2.1833479404449463, + "logits/rejected": -2.244410991668701, + "logps/chosen": -0.29939165711402893, + "logps/rejected": -0.29696738719940186, + "loss": 0.013744374737143517, + "loss/core": 0.013744374737143517, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.2123219966888428, + "rewards/margins": 0.5209118723869324, + "rewards/rejected": 0.6914102435112, + "step": 445 + }, + { + "epoch": 0.4294917680744452, + "grad_norm": 9.875, + "ht_mnpo/logit": 0.0671422928571701, + "ht_mnpo/residual": 0.05964229628443718, + "ht_mnpo/residual_abs": 0.06930109858512878, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.943666120468088e-07, + "logits/chosen": -2.143707752227783, + "logits/rejected": -2.1564431190490723, + "logps/chosen": -0.28366950154304504, + "logps/rejected": -0.27332574129104614, + "loss": 0.024859780445694923, + "loss/core": 0.024859780445694923, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.8562815189361572, + "rewards/margins": 0.6714230179786682, + "rewards/rejected": 1.1848584413528442, + "step": 450 + }, + { + "epoch": 0.43426389883082794, + "grad_norm": 0.65234375, + "ht_mnpo/logit": 0.1936154067516327, + "ht_mnpo/residual": 0.18611541390419006, + "ht_mnpo/residual_abs": 0.19285865128040314, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.8958738724208073e-07, + "logits/chosen": -2.1005420684814453, + "logits/rejected": -2.149547815322876, + "logps/chosen": -0.29087719321250916, + "logps/rejected": -0.2866514027118683, + "loss": 0.6541997194290161, + "loss/core": 0.6541997194290161, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.9990909099578857, + "rewards/margins": 1.9361540079116821, + "rewards/rejected": 1.0629371404647827, + "step": 455 + }, + { + "epoch": 0.43903602958721066, + "grad_norm": 2592.0, + "ht_mnpo/logit": 0.19766196608543396, + "ht_mnpo/residual": 0.19016198813915253, + "ht_mnpo/residual_abs": 0.22098055481910706, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.8479327524001633e-07, + "logits/chosen": -2.2931854724884033, + "logits/rejected": -2.2046098709106445, + "logps/chosen": -0.2994667887687683, + "logps/rejected": -0.28606337308883667, + "loss": 0.7362050414085388, + "loss/core": 0.7362050414085388, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.860424518585205, + "rewards/margins": 1.9766197204589844, + "rewards/rejected": 0.8838049173355103, + "step": 460 + }, + { + "epoch": 0.44380816034359344, + "grad_norm": 10.625, + "ht_mnpo/logit": 0.06282283365726471, + "ht_mnpo/residual": 0.05532282590866089, + "ht_mnpo/residual_abs": 0.07460145652294159, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.7998607891007493e-07, + "logits/chosen": -2.1819441318511963, + "logits/rejected": -2.180514097213745, + "logps/chosen": -0.2965949773788452, + "logps/rejected": -0.278251975774765, + "loss": 0.026971453800797462, + "loss/core": 0.026971453800797462, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.5738455057144165, + "rewards/margins": 0.6282283067703247, + "rewards/rejected": 0.9456171989440918, + "step": 465 + }, + { + "epoch": 0.44858029109997616, + "grad_norm": 19.875, + "ht_mnpo/logit": 0.14918012917041779, + "ht_mnpo/residual": 0.14168013632297516, + "ht_mnpo/residual_abs": 0.22854459285736084, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.7516760604219616e-07, + "logits/chosen": -1.9400546550750732, + "logits/rejected": -2.032510280609131, + "logps/chosen": -0.2952631115913391, + "logps/rejected": -0.28796467185020447, + "loss": 0.5161349773406982, + "loss/core": 0.5161349773406982, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.023547410964966, + "rewards/margins": 1.491801381111145, + "rewards/rejected": 1.5317460298538208, + "step": 470 + }, + { + "epoch": 0.4533524218563589, + "grad_norm": 46.25, + "ht_mnpo/logit": 0.13631603121757507, + "ht_mnpo/residual": 0.12881603837013245, + "ht_mnpo/residual_abs": 0.14233455061912537, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.703396686669646e-07, + "logits/chosen": -2.1496260166168213, + "logits/rejected": -2.2138235569000244, + "logps/chosen": -0.3058208227157593, + "logps/rejected": -0.2893269956111908, + "loss": 0.4681578576564789, + "loss/core": 0.4681578576564789, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.2276856899261475, + "rewards/margins": 1.3631603717803955, + "rewards/rejected": 0.8645254969596863, + "step": 475 + }, + { + "epoch": 0.4581245526127416, + "grad_norm": 2.03125, + "ht_mnpo/logit": 0.07291718572378159, + "ht_mnpo/residual": 0.06541718542575836, + "ht_mnpo/residual_abs": 0.1957278698682785, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.6550408237417884e-07, + "logits/chosen": -2.0847127437591553, + "logits/rejected": -2.0919432640075684, + "logps/chosen": -0.28979983925819397, + "logps/rejected": -0.28023532032966614, + "loss": 0.4742603898048401, + "loss/core": 0.4742603898048401, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.021890163421631, + "rewards/margins": 0.729171872138977, + "rewards/rejected": 1.2927181720733643, + "step": 480 + }, + { + "epoch": 0.4628966833691243, + "grad_norm": 1144.0, + "ht_mnpo/logit": 0.2463802993297577, + "ht_mnpo/residual": 0.2388802468776703, + "ht_mnpo/residual_abs": 0.245798259973526, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.6066266563008265e-07, + "logits/chosen": -2.0273923873901367, + "logits/rejected": -1.9866002798080444, + "logps/chosen": -0.3362005949020386, + "logps/rejected": -0.3115374445915222, + "loss": 0.9858028292655945, + "loss/core": 0.9858028292655945, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.876636028289795, + "rewards/margins": 2.4638028144836426, + "rewards/rejected": 1.4128332138061523, + "step": 485 + }, + { + "epoch": 0.46766881412550704, + "grad_norm": 1.6953125, + "ht_mnpo/logit": 0.059139739722013474, + "ht_mnpo/residual": 0.05163973569869995, + "ht_mnpo/residual_abs": 0.10399965941905975, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.5581723909351404e-07, + "logits/chosen": -2.0965423583984375, + "logits/rejected": -2.182716131210327, + "logps/chosen": -0.29576700925827026, + "logps/rejected": -0.29857736825942993, + "loss": 0.09599469602108002, + "loss/core": 0.09599469602108002, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.5863193273544312, + "rewards/margins": 0.5913974046707153, + "rewards/rejected": 0.994921863079071, + "step": 490 + }, + { + "epoch": 0.47244094488188976, + "grad_norm": 7.25, + "ht_mnpo/logit": 0.08595691621303558, + "ht_mnpo/residual": 0.07845690846443176, + "ht_mnpo/residual_abs": 0.08687671273946762, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.509696249312301e-07, + "logits/chosen": -2.202465057373047, + "logits/rejected": -2.1843132972717285, + "logps/chosen": -0.2824229300022125, + "logps/rejected": -0.2738740146160126, + "loss": 0.03743022680282593, + "loss/core": 0.03743022680282593, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.5005347728729248, + "rewards/margins": 0.8595690727233887, + "rewards/rejected": 0.6409655809402466, + "step": 495 + }, + { + "epoch": 0.4772130756382725, + "grad_norm": 47.0, + "ht_mnpo/logit": 0.09756086021661758, + "ht_mnpo/residual": 0.09006085246801376, + "ht_mnpo/residual_abs": 0.09450768679380417, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.461216461326642e-07, + "logits/chosen": -2.0701684951782227, + "logits/rejected": -2.1218984127044678, + "logps/chosen": -0.3352174162864685, + "logps/rejected": -0.31724342703819275, + "loss": 0.12419148534536362, + "loss/core": 0.12419148534536362, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.500693440437317, + "rewards/margins": 0.9756084680557251, + "rewards/rejected": 0.5250848531723022, + "step": 500 + }, + { + "epoch": 0.4819852063946552, + "grad_norm": 3.40625, + "ht_mnpo/logit": 0.1528806984424591, + "ht_mnpo/residual": 0.1453806608915329, + "ht_mnpo/residual_abs": 0.15676525235176086, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.412751258243748e-07, + "logits/chosen": -2.2806396484375, + "logits/rejected": -2.3282206058502197, + "logps/chosen": -0.2664569020271301, + "logps/rejected": -0.2485734224319458, + "loss": 0.3381979465484619, + "loss/core": 0.3381979465484619, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.9908111095428467, + "rewards/margins": 1.5288066864013672, + "rewards/rejected": 1.4620041847229004, + "step": 505 + }, + { + "epoch": 0.4867573371510379, + "grad_norm": 1.5546875, + "ht_mnpo/logit": 0.034343037754297256, + "ht_mnpo/residual": 0.026843031868338585, + "ht_mnpo/residual_abs": 0.10809679329395294, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.3643188658444158e-07, + "logits/chosen": -2.1005702018737793, + "logits/rejected": -2.0844995975494385, + "logps/chosen": -0.31387194991111755, + "logps/rejected": -0.30737537145614624, + "loss": 0.09971658885478973, + "loss/core": 0.09971658885478973, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 1.6584968566894531, + "rewards/margins": 0.34343039989471436, + "rewards/rejected": 1.3150662183761597, + "step": 510 + }, + { + "epoch": 0.49152946790742064, + "grad_norm": 4.90625, + "ht_mnpo/logit": 0.19290807843208313, + "ht_mnpo/residual": 0.1854081153869629, + "ht_mnpo/residual_abs": 0.2561078667640686, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.315937497570688e-07, + "logits/chosen": -2.0163683891296387, + "logits/rejected": -2.0174050331115723, + "logps/chosen": -0.27022209763526917, + "logps/rejected": -0.26961493492126465, + "loss": 0.6102972030639648, + "loss/core": 0.6102972030639648, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.291226863861084, + "rewards/margins": 1.9290809631347656, + "rewards/rejected": 1.3621461391448975, + "step": 515 + }, + { + "epoch": 0.4963015986638034, + "grad_norm": 7.5625, + "ht_mnpo/logit": -0.005236223340034485, + "ht_mnpo/residual": -0.012736232951283455, + "ht_mnpo/residual_abs": 0.1328015923500061, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.2676253476765194e-07, + "logits/chosen": -2.0806922912597656, + "logits/rejected": -2.0068271160125732, + "logps/chosen": -0.31831881403923035, + "logps/rejected": -0.2931275963783264, + "loss": 0.354873389005661, + "loss/core": 0.354873389005661, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.4059890508651733, + "rewards/margins": -0.05236220359802246, + "rewards/rejected": 1.4583513736724854, + "step": 520 + }, + { + "epoch": 0.5010737294201861, + "grad_norm": 0.7734375, + "ht_mnpo/logit": 0.02382918819785118, + "ht_mnpo/residual": 0.016329189762473106, + "ht_mnpo/residual_abs": 0.060878295451402664, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.2194005843856633e-07, + "logits/chosen": -2.3115806579589844, + "logits/rejected": -2.299849271774292, + "logps/chosen": -0.31554344296455383, + "logps/rejected": -0.3013135492801666, + "loss": 0.026647498831152916, + "loss/core": 0.026647498831152916, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.0067657232284546, + "rewards/margins": 0.2382918894290924, + "rewards/rejected": 0.7684738636016846, + "step": 525 + }, + { + "epoch": 0.5058458601765689, + "grad_norm": 318.0, + "ht_mnpo/logit": 0.047034695744514465, + "ht_mnpo/residual": 0.03953469544649124, + "ht_mnpo/residual_abs": 0.1397225260734558, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.1712813430593434e-07, + "logits/chosen": -2.1985840797424316, + "logits/rejected": -2.074636936187744, + "logps/chosen": -0.29173484444618225, + "logps/rejected": -0.2891838550567627, + "loss": 0.18388794362545013, + "loss/core": 0.18388794362545013, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.232212543487549, + "rewards/margins": 0.47034698724746704, + "rewards/rejected": 1.7618656158447266, + "step": 530 + }, + { + "epoch": 0.5106179909329516, + "grad_norm": 1152.0, + "ht_mnpo/logit": 0.2150212824344635, + "ht_mnpo/residual": 0.20752127468585968, + "ht_mnpo/residual_abs": 0.25212562084198, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.123285719376292e-07, + "logits/chosen": -2.056133270263672, + "logits/rejected": -2.110259771347046, + "logps/chosen": -0.29885774850845337, + "logps/rejected": -0.29431480169296265, + "loss": 1.2166507244110107, + "loss/core": 1.2166507244110107, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.6530518531799316, + "rewards/margins": 2.1502127647399902, + "rewards/rejected": 1.502839207649231, + "step": 535 + }, + { + "epoch": 0.5153901216893343, + "grad_norm": 1.9375, + "ht_mnpo/logit": 0.05426005274057388, + "ht_mnpo/residual": 0.046760059893131256, + "ht_mnpo/residual_abs": 0.074722521007061, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.0754317625276982e-07, + "logits/chosen": -2.156520128250122, + "logits/rejected": -2.1915793418884277, + "logps/chosen": -0.2928999066352844, + "logps/rejected": -0.2622906565666199, + "loss": 0.0269025769084692, + "loss/core": 0.0269025769084692, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.6733510494232178, + "rewards/margins": 0.5426005125045776, + "rewards/rejected": 1.1307506561279297, + "step": 540 + }, + { + "epoch": 0.520162252445717, + "grad_norm": 0.66796875, + "ht_mnpo/logit": 0.10010049492120743, + "ht_mnpo/residual": 0.09260048717260361, + "ht_mnpo/residual_abs": 0.10867872089147568, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.0277374684296498e-07, + "logits/chosen": -2.0413730144500732, + "logits/rejected": -2.0470833778381348, + "logps/chosen": -0.29208695888519287, + "logps/rejected": -0.27415579557418823, + "loss": 0.20062501728534698, + "loss/core": 0.20062501728534698, + "rewards/accuracies": 0.8125, + "rewards/chosen": 1.9059289693832397, + "rewards/margins": 1.001004934310913, + "rewards/rejected": 0.9049240946769714, + "step": 545 + }, + { + "epoch": 0.5249343832020997, + "grad_norm": 1.8671875, + "ht_mnpo/logit": 0.098787322640419, + "ht_mnpo/residual": 0.09128732234239578, + "ht_mnpo/residual_abs": 0.13674144446849823, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.980220772955602e-07, + "logits/chosen": -2.286104440689087, + "logits/rejected": -2.361344814300537, + "logps/chosen": -0.3042377531528473, + "logps/rejected": -0.2910940945148468, + "loss": 0.19538818299770355, + "loss/core": 0.19538818299770355, + "rewards/accuracies": 0.8125, + "rewards/chosen": 2.1192712783813477, + "rewards/margins": 0.9878730773925781, + "rewards/rejected": 1.1313982009887695, + "step": 550 + }, + { + "epoch": 0.5297065139584825, + "grad_norm": 9.6875, + "ht_mnpo/logit": 0.08020411431789398, + "ht_mnpo/residual": 0.07270411401987076, + "ht_mnpo/residual_abs": 0.0984831228852272, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.932899545191433e-07, + "logits/chosen": -2.1592469215393066, + "logits/rejected": -2.134700298309326, + "logps/chosen": -0.25345703959465027, + "logps/rejected": -0.2741752564907074, + "loss": 0.06742379814386368, + "loss/core": 0.06742379814386368, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.854957938194275, + "rewards/margins": 0.8020410537719727, + "rewards/rejected": 1.0529168844223022, + "step": 555 + }, + { + "epoch": 0.5344786447148652, + "grad_norm": 24.875, + "ht_mnpo/logit": 0.21764996647834778, + "ht_mnpo/residual": 0.21014995872974396, + "ht_mnpo/residual_abs": 0.2130311280488968, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.885791580715609e-07, + "logits/chosen": -2.1484036445617676, + "logits/rejected": -2.2150402069091797, + "logps/chosen": -0.28861570358276367, + "logps/rejected": -0.26910072565078735, + "loss": 0.6849492192268372, + "loss/core": 0.6849492192268372, + "rewards/accuracies": 0.9624999761581421, + "rewards/chosen": 3.4669604301452637, + "rewards/margins": 2.176499605178833, + "rewards/rejected": 1.2904607057571411, + "step": 560 + }, + { + "epoch": 0.5392507754712479, + "grad_norm": 1512.0, + "ht_mnpo/logit": 0.23818376660346985, + "ht_mnpo/residual": 0.23068372905254364, + "ht_mnpo/residual_abs": 0.33580002188682556, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8389145949069951e-07, + "logits/chosen": -2.1061885356903076, + "logits/rejected": -2.1180262565612793, + "logps/chosen": -0.2766849994659424, + "logps/rejected": -0.30740275979042053, + "loss": 1.4853737354278564, + "loss/core": 1.4853737354278564, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.7866311073303223, + "rewards/margins": 2.3818376064300537, + "rewards/rejected": 1.4047935009002686, + "step": 565 + }, + { + "epoch": 0.5440229062276306, + "grad_norm": 0.78125, + "ht_mnpo/logit": 0.22942838072776794, + "ht_mnpo/residual": 0.2219284027814865, + "ht_mnpo/residual_abs": 0.23399917781352997, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.792286216282824e-07, + "logits/chosen": -2.007561206817627, + "logits/rejected": -1.9956079721450806, + "logps/chosen": -0.2868313193321228, + "logps/rejected": -0.29199647903442383, + "loss": 0.5726346373558044, + "loss/core": 0.5726346373558044, + "rewards/accuracies": 0.8125, + "rewards/chosen": 3.5619232654571533, + "rewards/margins": 2.2942841053009033, + "rewards/rejected": 1.2676398754119873, + "step": 570 + }, + { + "epoch": 0.5487950369840133, + "grad_norm": 2.484375, + "ht_mnpo/logit": 0.11159713566303253, + "ht_mnpo/residual": 0.1040971428155899, + "ht_mnpo/residual_abs": 0.11690263450145721, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.745923979869336e-07, + "logits/chosen": -2.1647210121154785, + "logits/rejected": -2.207653522491455, + "logps/chosen": -0.3100038468837738, + "logps/rejected": -0.281381219625473, + "loss": 0.09815619885921478, + "loss/core": 0.09815619885921478, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.275163173675537, + "rewards/margins": 1.115971326828003, + "rewards/rejected": 1.1591918468475342, + "step": 575 + }, + { + "epoch": 0.553567167740396, + "grad_norm": 2.921875, + "ht_mnpo/logit": 0.11486697196960449, + "ht_mnpo/residual": 0.10736697912216187, + "ht_mnpo/residual_abs": 0.13199064135551453, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.6998453206075708e-07, + "logits/chosen": -2.228494167327881, + "logits/rejected": -2.189988613128662, + "logps/chosen": -0.2966822683811188, + "logps/rejected": -0.2899606227874756, + "loss": 0.288084477186203, + "loss/core": 0.288084477186203, + "rewards/accuracies": 0.875, + "rewards/chosen": 1.918851613998413, + "rewards/margins": 1.1486698389053345, + "rewards/rejected": 0.7701820135116577, + "step": 580 + }, + { + "epoch": 0.5583392984967788, + "grad_norm": 0.6328125, + "ht_mnpo/logit": 0.06170004606246948, + "ht_mnpo/residual": 0.054200053215026855, + "ht_mnpo/residual_abs": 0.09687240421772003, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.6540675667967973e-07, + "logits/chosen": -1.9388682842254639, + "logits/rejected": -1.8519554138183594, + "logps/chosen": -0.26918455958366394, + "logps/rejected": -0.2541769742965698, + "loss": 0.042456772178411484, + "loss/core": 0.042456772178411484, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.1972382068634033, + "rewards/margins": 0.6170004606246948, + "rewards/rejected": 1.5802377462387085, + "step": 585 + }, + { + "epoch": 0.5631114292531615, + "grad_norm": 8.75, + "ht_mnpo/logit": 0.058099836111068726, + "ht_mnpo/residual": 0.0505998432636261, + "ht_mnpo/residual_abs": 0.17364242672920227, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.60860793357805e-07, + "logits/chosen": -2.2340292930603027, + "logits/rejected": -2.1745638847351074, + "logps/chosen": -0.23693934082984924, + "logps/rejected": -0.24809077382087708, + "loss": 0.3619423806667328, + "loss/core": 0.3619423806667328, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.183443546295166, + "rewards/margins": 0.5809983611106873, + "rewards/rejected": 1.6024450063705444, + "step": 590 + }, + { + "epoch": 0.5678835600095442, + "grad_norm": 0.1298828125, + "ht_mnpo/logit": 0.07216361910104752, + "ht_mnpo/residual": 0.06466361880302429, + "ht_mnpo/residual_abs": 0.0867505595088005, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.5634835164602196e-07, + "logits/chosen": -2.1809277534484863, + "logits/rejected": -2.264443874359131, + "logps/chosen": -0.27737170457839966, + "logps/rejected": -0.27399271726608276, + "loss": 0.04572673514485359, + "loss/core": 0.04572673514485359, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 1.4415251016616821, + "rewards/margins": 0.721636176109314, + "rewards/rejected": 0.7198889851570129, + "step": 595 + }, + { + "epoch": 0.572655690765927, + "grad_norm": 8.8125, + "ht_mnpo/logit": 0.06845931708812714, + "ht_mnpo/residual": 0.06095932796597481, + "ht_mnpo/residual_abs": 0.28927868604660034, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.518711284891132e-07, + "logits/chosen": -2.0192017555236816, + "logits/rejected": -2.1130526065826416, + "logps/chosen": -0.27419912815093994, + "logps/rejected": -0.2707653343677521, + "loss": 1.1791174411773682, + "loss/core": 1.1791174411773682, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.887829542160034, + "rewards/margins": 0.6845930814743042, + "rewards/rejected": 2.2032363414764404, + "step": 600 + }, + { + "epoch": 0.5774278215223098, + "grad_norm": 99.5, + "ht_mnpo/logit": 0.18931171298027039, + "ht_mnpo/residual": 0.18181169033050537, + "ht_mnpo/residual_abs": 0.20922286808490753, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.47430807587603e-07, + "logits/chosen": -2.2182424068450928, + "logits/rejected": -2.2502996921539307, + "logps/chosen": -0.3130815327167511, + "logps/rejected": -0.3144187033176422, + "loss": 0.5771145820617676, + "loss/core": 0.5771145820617676, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.197204113006592, + "rewards/margins": 1.8931169509887695, + "rewards/rejected": 1.304086685180664, + "step": 605 + }, + { + "epoch": 0.5821999522786925, + "grad_norm": 2.578125, + "ht_mnpo/logit": 0.09359414875507355, + "ht_mnpo/residual": 0.08609413355588913, + "ht_mnpo/residual_abs": 0.10207605361938477, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.430290587645865e-07, + "logits/chosen": -2.1247410774230957, + "logits/rejected": -2.108510971069336, + "logps/chosen": -0.272658109664917, + "logps/rejected": -0.28190943598747253, + "loss": 0.11088033020496368, + "loss/core": 0.11088033020496368, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.923109769821167, + "rewards/margins": 0.9359413981437683, + "rewards/rejected": 0.9871681332588196, + "step": 610 + }, + { + "epoch": 0.5869720830350752, + "grad_norm": 59.75, + "ht_mnpo/logit": 0.21152541041374207, + "ht_mnpo/residual": 0.20402541756629944, + "ht_mnpo/residual_abs": 0.22103989124298096, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.3866753733777765e-07, + "logits/chosen": -2.192009449005127, + "logits/rejected": -2.2331554889678955, + "logps/chosen": -0.31632620096206665, + "logps/rejected": -0.28495582938194275, + "loss": 0.684616208076477, + "loss/core": 0.684616208076477, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 3.1536481380462646, + "rewards/margins": 2.1152539253234863, + "rewards/rejected": 1.0383942127227783, + "step": 615 + }, + { + "epoch": 0.5917442137914579, + "grad_norm": 34.5, + "ht_mnpo/logit": 0.043183762580156326, + "ht_mnpo/residual": 0.0356837622821331, + "ht_mnpo/residual_abs": 0.1497439444065094, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.343478834970121e-07, + "logits/chosen": -2.1083340644836426, + "logits/rejected": -2.179131269454956, + "logps/chosen": -0.26702603697776794, + "logps/rejected": -0.26075607538223267, + "loss": 0.13401201367378235, + "loss/core": 0.13401201367378235, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.8873205184936523, + "rewards/margins": 0.43183764815330505, + "rewards/rejected": 1.455482840538025, + "step": 620 + }, + { + "epoch": 0.5965163445478406, + "grad_norm": 5.53125, + "ht_mnpo/logit": 0.20867642760276794, + "ht_mnpo/residual": 0.2011764496564865, + "ht_mnpo/residual_abs": 0.2585950195789337, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.3007172168743852e-07, + "logits/chosen": -1.7806930541992188, + "logits/rejected": -1.8166608810424805, + "logps/chosen": -0.24558083713054657, + "logps/rejected": -0.24023476243019104, + "loss": 0.46875572204589844, + "loss/core": 0.46875572204589844, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 4.4086103439331055, + "rewards/margins": 2.086764335632324, + "rewards/rejected": 2.3218460083007812, + "step": 625 + }, + { + "epoch": 0.6012884753042234, + "grad_norm": 2.046875, + "ht_mnpo/logit": 0.13468870520591736, + "ht_mnpo/residual": 0.12718871235847473, + "ht_mnpo/residual_abs": 0.1308780014514923, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.25840659998631e-07, + "logits/chosen": -2.0953762531280518, + "logits/rejected": -2.1909890174865723, + "logps/chosen": -0.26866987347602844, + "logps/rejected": -0.2617102563381195, + "loss": 0.21047914028167725, + "loss/core": 0.21047914028167725, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.607395648956299, + "rewards/margins": 1.3468869924545288, + "rewards/rejected": 1.2605082988739014, + "step": 630 + }, + { + "epoch": 0.6060606060606061, + "grad_norm": 16.25, + "ht_mnpo/logit": 0.046742431819438934, + "ht_mnpo/residual": 0.03924242779612541, + "ht_mnpo/residual_abs": 0.11967098712921143, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.2165628955985313e-07, + "logits/chosen": -2.2242093086242676, + "logits/rejected": -2.122405529022217, + "logps/chosen": -0.30006808042526245, + "logps/rejected": -0.28186583518981934, + "loss": 0.1131725087761879, + "loss/core": 0.1131725087761879, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 1.4187710285186768, + "rewards/margins": 0.46742430329322815, + "rewards/rejected": 0.9513468742370605, + "step": 635 + }, + { + "epoch": 0.6108327368169888, + "grad_norm": 52.5, + "ht_mnpo/logit": 0.033292241394519806, + "ht_mnpo/residual": 0.02579224482178688, + "ht_mnpo/residual_abs": 0.2367764711380005, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.175201839416988e-07, + "logits/chosen": -2.073786973953247, + "logits/rejected": -2.0959436893463135, + "logps/chosen": -0.28610923886299133, + "logps/rejected": -0.2980540096759796, + "loss": 0.48102641105651855, + "loss/core": 0.48102641105651855, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.3441481590270996, + "rewards/margins": 0.3329223096370697, + "rewards/rejected": 2.011225938796997, + "step": 640 + }, + { + "epoch": 0.6156048675733715, + "grad_norm": 2.9375, + "ht_mnpo/logit": 0.17397519946098328, + "ht_mnpo/residual": 0.16647520661354065, + "ht_mnpo/residual_abs": 0.1911737620830536, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.1343389856433658e-07, + "logits/chosen": -2.155489444732666, + "logits/rejected": -2.164496898651123, + "logps/chosen": -0.29790198802948, + "logps/rejected": -0.2998579144477844, + "loss": 0.49661874771118164, + "loss/core": 0.49661874771118164, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 3.293790340423584, + "rewards/margins": 1.7397520542144775, + "rewards/rejected": 1.554038405418396, + "step": 645 + }, + { + "epoch": 0.6203769983297542, + "grad_norm": 32.5, + "ht_mnpo/logit": 0.1089751347899437, + "ht_mnpo/residual": 0.10147513449192047, + "ht_mnpo/residual_abs": 0.11368437111377716, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0939897011258e-07, + "logits/chosen": -2.090285301208496, + "logits/rejected": -2.1761794090270996, + "logps/chosen": -0.27180442214012146, + "logps/rejected": -0.26983508467674255, + "loss": 0.077135369181633, + "loss/core": 0.077135369181633, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.2141337394714355, + "rewards/margins": 1.0897514820098877, + "rewards/rejected": 1.1243823766708374, + "step": 650 + }, + { + "epoch": 0.625149129086137, + "grad_norm": 880.0, + "ht_mnpo/logit": 0.2785004675388336, + "ht_mnpo/residual": 0.271000474691391, + "ht_mnpo/residual_abs": 0.3150801658630371, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0541691595800336e-07, + "logits/chosen": -2.0350708961486816, + "logits/rejected": -2.0822062492370605, + "logps/chosen": -0.3091558516025543, + "logps/rejected": -0.31957852840423584, + "loss": 1.4940129518508911, + "loss/core": 1.4940129518508911, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.898334503173828, + "rewards/margins": 2.7850046157836914, + "rewards/rejected": 1.113329529762268, + "step": 655 + }, + { + "epoch": 0.6299212598425197, + "grad_norm": 16.875, + "ht_mnpo/logit": 0.17702892422676086, + "ht_mnpo/residual": 0.16952893137931824, + "ht_mnpo/residual_abs": 0.1766408532857895, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0148923358832021e-07, + "logits/chosen": -1.9041671752929688, + "logits/rejected": -1.929474115371704, + "logps/chosen": -0.2832731306552887, + "logps/rejected": -0.2771525979042053, + "loss": 0.39101579785346985, + "loss/core": 0.39101579785346985, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.728006362915039, + "rewards/margins": 1.770289421081543, + "rewards/rejected": 0.9577168226242065, + "step": 660 + }, + { + "epoch": 0.6346933905989024, + "grad_norm": 15.6875, + "ht_mnpo/logit": 0.09807918965816498, + "ht_mnpo/residual": 0.09057918936014175, + "ht_mnpo/residual_abs": 0.12041660398244858, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.761740004423926e-08, + "logits/chosen": -2.069812536239624, + "logits/rejected": -2.125847816467285, + "logps/chosen": -0.2814692556858063, + "logps/rejected": -0.2710791230201721, + "loss": 0.08960843086242676, + "loss/core": 0.08960843086242676, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.962667465209961, + "rewards/margins": 0.9807918667793274, + "rewards/rejected": 0.9818755388259888, + "step": 665 + }, + { + "epoch": 0.6394655213552851, + "grad_norm": 94.0, + "ht_mnpo/logit": 0.00905347429215908, + "ht_mnpo/residual": 0.0015534699195995927, + "ht_mnpo/residual_abs": 0.147975355386734, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.380287136400999e-08, + "logits/chosen": -2.445849657058716, + "logits/rejected": -2.4024407863616943, + "logps/chosen": -0.28253886103630066, + "logps/rejected": -0.2934015989303589, + "loss": 0.2624434232711792, + "loss/core": 0.2624434232711792, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 1.5216253995895386, + "rewards/margins": 0.09053464233875275, + "rewards/rejected": 1.4310907125473022, + "step": 670 + }, + { + "epoch": 0.6442376521116678, + "grad_norm": 87.0, + "ht_mnpo/logit": 0.07526589930057526, + "ht_mnpo/residual": 0.06776590645313263, + "ht_mnpo/residual_abs": 0.19357003271579742, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.004708203586628e-08, + "logits/chosen": -2.027789831161499, + "logits/rejected": -2.080132484436035, + "logps/chosen": -0.2811906039714813, + "logps/rejected": -0.2772737741470337, + "loss": 0.30826523900032043, + "loss/core": 0.30826523900032043, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.5021486282348633, + "rewards/margins": 0.7526589632034302, + "rewards/rejected": 1.7494895458221436, + "step": 675 + }, + { + "epoch": 0.6490097828680506, + "grad_norm": 12.625, + "ht_mnpo/logit": 0.10573352873325348, + "ht_mnpo/residual": 0.09823354333639145, + "ht_mnpo/residual_abs": 0.10365482419729233, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.635144445857407e-08, + "logits/chosen": -1.910638451576233, + "logits/rejected": -1.9565751552581787, + "logps/chosen": -0.3004496693611145, + "logps/rejected": -0.2888430058956146, + "loss": 0.08603724092245102, + "loss/core": 0.08603724092245102, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.726280927658081, + "rewards/margins": 1.0573352575302124, + "rewards/rejected": 1.6689456701278687, + "step": 680 + }, + { + "epoch": 0.6537819136244333, + "grad_norm": 15.6875, + "ht_mnpo/logit": 0.11498228460550308, + "ht_mnpo/residual": 0.10748227685689926, + "ht_mnpo/residual_abs": 0.15228202939033508, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.271734841028552e-08, + "logits/chosen": -2.176805019378662, + "logits/rejected": -2.210144519805908, + "logps/chosen": -0.2797221839427948, + "logps/rejected": -0.3026573657989502, + "loss": 0.24593576788902283, + "loss/core": 0.24593576788902283, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.403441905975342, + "rewards/margins": 1.1498225927352905, + "rewards/rejected": 1.2536190748214722, + "step": 685 + }, + { + "epoch": 0.658554044380816, + "grad_norm": 240.0, + "ht_mnpo/logit": 0.11702848970890045, + "ht_mnpo/residual": 0.10952849686145782, + "ht_mnpo/residual_abs": 0.1646626889705658, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 7.91461605259007e-08, + "logits/chosen": -2.0042319297790527, + "logits/rejected": -1.979111671447754, + "logps/chosen": -0.3079618811607361, + "logps/rejected": -0.2868824601173401, + "loss": 0.22790667414665222, + "loss/core": 0.22790667414665222, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.4165444374084473, + "rewards/margins": 1.1702848672866821, + "rewards/rejected": 1.2462592124938965, + "step": 690 + }, + { + "epoch": 0.6633261751371987, + "grad_norm": 984.0, + "ht_mnpo/logit": 0.18550536036491394, + "ht_mnpo/residual": 0.1780053675174713, + "ht_mnpo/residual_abs": 0.1904396116733551, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 7.563922378313217e-08, + "logits/chosen": -2.0949904918670654, + "logits/rejected": -2.118269205093384, + "logps/chosen": -0.2962483763694763, + "logps/rejected": -0.2824075222015381, + "loss": 0.8668164014816284, + "loss/core": 0.8668164014816284, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.9470839500427246, + "rewards/margins": 1.8550535440444946, + "rewards/rejected": 1.0920301675796509, + "step": 695 + }, + { + "epoch": 0.6680983058935814, + "grad_norm": 756.0, + "ht_mnpo/logit": 0.22229436039924622, + "ht_mnpo/residual": 0.21479432284832, + "ht_mnpo/residual_abs": 0.28674596548080444, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 7.219785699746572e-08, + "logits/chosen": -2.061213731765747, + "logits/rejected": -2.021340847015381, + "logps/chosen": -0.30653488636016846, + "logps/rejected": -0.3151229918003082, + "loss": 0.881302535533905, + "loss/core": 0.881302535533905, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 4.028003692626953, + "rewards/margins": 2.2229433059692383, + "rewards/rejected": 1.8050603866577148, + "step": 700 + }, + { + "epoch": 0.6728704366499642, + "grad_norm": 1.5390625, + "ht_mnpo/logit": 0.0755210891366005, + "ht_mnpo/residual": 0.06802109628915787, + "ht_mnpo/residual_abs": 0.15567728877067566, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.882335432620779e-08, + "logits/chosen": -2.249558687210083, + "logits/rejected": -2.192272663116455, + "logps/chosen": -0.2608739733695984, + "logps/rejected": -0.27803075313568115, + "loss": 0.30238884687423706, + "loss/core": 0.30238884687423706, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.7999625205993652, + "rewards/margins": 0.7552109956741333, + "rewards/rejected": 2.0447516441345215, + "step": 705 + }, + { + "epoch": 0.677642567406347, + "grad_norm": 414.0, + "ht_mnpo/logit": 0.11202869564294815, + "ht_mnpo/residual": 0.10452868789434433, + "ht_mnpo/residual_abs": 0.25466471910476685, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.551698478180589e-08, + "logits/chosen": -2.04231333732605, + "logits/rejected": -2.1298766136169434, + "logps/chosen": -0.3229544162750244, + "logps/rejected": -0.2944870591163635, + "loss": 0.6831266283988953, + "loss/core": 0.6831266283988953, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.0203940868377686, + "rewards/margins": 1.1202871799468994, + "rewards/rejected": 1.9001071453094482, + "step": 710 + }, + { + "epoch": 0.6824146981627297, + "grad_norm": 11.75, + "ht_mnpo/logit": 0.16674119234085083, + "ht_mnpo/residual": 0.15924116969108582, + "ht_mnpo/residual_abs": 0.1635814756155014, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.22799917546252e-08, + "logits/chosen": -1.7993494272232056, + "logits/rejected": -1.8742660284042358, + "logps/chosen": -0.2802506387233734, + "logps/rejected": -0.30492326617240906, + "loss": 0.26578038930892944, + "loss/core": 0.26578038930892944, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.977574110031128, + "rewards/margins": 1.6674118041992188, + "rewards/rejected": 1.3101623058319092, + "step": 715 + }, + { + "epoch": 0.6871868289191124, + "grad_norm": 2.625, + "ht_mnpo/logit": 0.21696901321411133, + "ht_mnpo/residual": 0.2094690054655075, + "ht_mnpo/residual_abs": 0.21472272276878357, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.9113592545359944e-08, + "logits/chosen": -2.0282702445983887, + "logits/rejected": -2.018631935119629, + "logps/chosen": -0.29306745529174805, + "logps/rejected": -0.2762381434440613, + "loss": 0.5803450345993042, + "loss/core": 0.5803450345993042, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.3107528686523438, + "rewards/margins": 2.169689893722534, + "rewards/rejected": 1.1410627365112305, + "step": 720 + }, + { + "epoch": 0.6919589596754951, + "grad_norm": 1.5078125, + "ht_mnpo/logit": 0.09387997537851334, + "ht_mnpo/residual": 0.08637996762990952, + "ht_mnpo/residual_abs": 0.19136810302734375, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.601897790725643e-08, + "logits/chosen": -2.2025420665740967, + "logits/rejected": -2.1641955375671387, + "logps/chosen": -0.2895504832267761, + "logps/rejected": -0.29813945293426514, + "loss": 0.2689235210418701, + "loss/core": 0.2689235210418701, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.5900564193725586, + "rewards/margins": 0.9387996792793274, + "rewards/rejected": 1.6512569189071655, + "step": 725 + }, + { + "epoch": 0.6967310904318779, + "grad_norm": 334.0, + "ht_mnpo/logit": 0.10368786752223969, + "ht_mnpo/residual": 0.09618787467479706, + "ht_mnpo/residual_abs": 0.11059357970952988, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.299731159831952e-08, + "logits/chosen": -2.100865364074707, + "logits/rejected": -2.1864678859710693, + "logps/chosen": -0.3016471862792969, + "logps/rejected": -0.28621745109558105, + "loss": 0.18038377165794373, + "loss/core": 0.18038377165794373, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.1753203868865967, + "rewards/margins": 1.0368787050247192, + "rewards/rejected": 1.138441801071167, + "step": 730 + }, + { + "epoch": 0.7015032211882606, + "grad_norm": 153.0, + "ht_mnpo/logit": -0.06999675184488297, + "ht_mnpo/residual": -0.07749674469232559, + "ht_mnpo/residual_abs": 0.15613941848278046, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.0049729943671013e-08, + "logits/chosen": -2.235835552215576, + "logits/rejected": -2.1666109561920166, + "logps/chosen": -0.3065755069255829, + "logps/rejected": -0.323150098323822, + "loss": 0.2582674026489258, + "loss/core": 0.2582674026489258, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.2052594423294067, + "rewards/margins": -0.6999675035476685, + "rewards/rejected": 1.9052270650863647, + "step": 735 + }, + { + "epoch": 0.7062753519446433, + "grad_norm": 9.5, + "ht_mnpo/logit": 0.08259505033493042, + "ht_mnpo/residual": 0.07509505748748779, + "ht_mnpo/residual_abs": 0.21390943229198456, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.7177341408223994e-08, + "logits/chosen": -2.0588953495025635, + "logits/rejected": -1.9913275241851807, + "logps/chosen": -0.266545832157135, + "logps/rejected": -0.2904679477214813, + "loss": 0.3255937695503235, + "loss/core": 0.3255937695503235, + "rewards/accuracies": 0.824999988079071, + "rewards/chosen": 2.877443790435791, + "rewards/margins": 0.8259506225585938, + "rewards/rejected": 2.0514931678771973, + "step": 740 + }, + { + "epoch": 0.711047482701026, + "grad_norm": 23.0, + "ht_mnpo/logit": 0.15350745618343353, + "ht_mnpo/residual": 0.1460074782371521, + "ht_mnpo/residual_abs": 0.15337680280208588, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.438122617983442e-08, + "logits/chosen": -2.0869879722595215, + "logits/rejected": -2.1158251762390137, + "logps/chosen": -0.27177831530570984, + "logps/rejected": -0.27791404724121094, + "loss": 0.2072025090456009, + "loss/core": 0.2072025090456009, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.250014543533325, + "rewards/margins": 1.5350747108459473, + "rewards/rejected": 0.7149398922920227, + "step": 745 + }, + { + "epoch": 0.7158196134574087, + "grad_norm": 14.1875, + "ht_mnpo/logit": 0.17855484783649445, + "ht_mnpo/residual": 0.17105485498905182, + "ht_mnpo/residual_abs": 0.18604342639446259, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.1662435763087114e-08, + "logits/chosen": -2.2527213096618652, + "logits/rejected": -2.2769358158111572, + "logps/chosen": -0.3151988387107849, + "logps/rejected": -0.2924613654613495, + "loss": 0.877219557762146, + "loss/core": 0.877219557762146, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.44504451751709, + "rewards/margins": 1.785548448562622, + "rewards/rejected": 0.6594961881637573, + "step": 750 + }, + { + "epoch": 0.7205917442137915, + "grad_norm": 56.0, + "ht_mnpo/logit": 0.170209139585495, + "ht_mnpo/residual": 0.16270914673805237, + "ht_mnpo/residual_abs": 0.17232632637023926, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.902199258386732e-08, + "logits/chosen": -2.1373720169067383, + "logits/rejected": -2.106365203857422, + "logps/chosen": -0.28223562240600586, + "logps/rejected": -0.27106139063835144, + "loss": 0.3418263792991638, + "loss/core": 0.3418263792991638, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.8122715950012207, + "rewards/margins": 1.7020914554595947, + "rewards/rejected": 1.1101800203323364, + "step": 755 + }, + { + "epoch": 0.7253638749701742, + "grad_norm": 7.03125, + "ht_mnpo/logit": 0.17837955057621002, + "ht_mnpo/residual": 0.1708795577287674, + "ht_mnpo/residual_abs": 0.22366230189800262, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.646088960486862e-08, + "logits/chosen": -2.1923656463623047, + "logits/rejected": -2.134833812713623, + "logps/chosen": -0.2628360092639923, + "logps/rejected": -0.2624150216579437, + "loss": 0.4018256664276123, + "loss/core": 0.4018256664276123, + "rewards/accuracies": 0.8125, + "rewards/chosen": 3.6742782592773438, + "rewards/margins": 1.7837955951690674, + "rewards/rejected": 1.8904825448989868, + "step": 760 + }, + { + "epoch": 0.7301360057265569, + "grad_norm": 19.5, + "ht_mnpo/logit": 0.07248622924089432, + "ht_mnpo/residual": 0.0649862214922905, + "ht_mnpo/residual_abs": 0.1020934134721756, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.398008995217988e-08, + "logits/chosen": -2.1482584476470947, + "logits/rejected": -2.178565263748169, + "logps/chosen": -0.28691160678863525, + "logps/rejected": -0.29412776231765747, + "loss": 0.08699820935726166, + "loss/core": 0.08699820935726166, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 1.3631985187530518, + "rewards/margins": 0.7248622179031372, + "rewards/rejected": 0.6383363604545593, + "step": 765 + }, + { + "epoch": 0.7349081364829396, + "grad_norm": 74.0, + "ht_mnpo/logit": 0.04947735741734505, + "ht_mnpo/residual": 0.04197736084461212, + "ht_mnpo/residual_abs": 0.23659929633140564, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.1580526553093315e-08, + "logits/chosen": -2.1913866996765137, + "logits/rejected": -2.0948822498321533, + "logps/chosen": -0.27521374821662903, + "logps/rejected": -0.2676638662815094, + "loss": 0.31541621685028076, + "loss/core": 0.31541621685028076, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.8486695289611816, + "rewards/margins": 0.4947734475135803, + "rewards/rejected": 2.353895902633667, + "step": 770 + }, + { + "epoch": 0.7396802672393223, + "grad_norm": 2.734375, + "ht_mnpo/logit": 0.1934659779071808, + "ht_mnpo/residual": 0.18596597015857697, + "ht_mnpo/residual_abs": 0.21423058211803436, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.9263101785268252e-08, + "logits/chosen": -2.0273308753967285, + "logits/rejected": -2.0538253784179688, + "logps/chosen": -0.3183925747871399, + "logps/rejected": -0.29416340589523315, + "loss": 0.6891728639602661, + "loss/core": 0.6891728639602661, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.740612030029297, + "rewards/margins": 1.934659719467163, + "rewards/rejected": 0.8059521913528442, + "step": 775 + }, + { + "epoch": 0.7444523979957051, + "grad_norm": 1.25, + "ht_mnpo/logit": 0.08660490810871124, + "ht_mnpo/residual": 0.07910490036010742, + "ht_mnpo/residual_abs": 0.08363465964794159, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.7028687137384264e-08, + "logits/chosen": -2.064676284790039, + "logits/rejected": -2.143580198287964, + "logps/chosen": -0.2868286669254303, + "logps/rejected": -0.26787668466567993, + "loss": 0.03424781560897827, + "loss/core": 0.03424781560897827, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.0800483226776123, + "rewards/margins": 0.8660489916801453, + "rewards/rejected": 1.2139991521835327, + "step": 780 + }, + { + "epoch": 0.7492245287520878, + "grad_norm": 436.0, + "ht_mnpo/logit": 0.14060871303081512, + "ht_mnpo/residual": 0.1331087350845337, + "ht_mnpo/residual_abs": 0.1878836750984192, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4878122881409447e-08, + "logits/chosen": -2.108158588409424, + "logits/rejected": -2.0105137825012207, + "logps/chosen": -0.28803449869155884, + "logps/rejected": -0.3160260319709778, + "loss": 0.2309446781873703, + "loss/core": 0.2309446781873703, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.6357979774475098, + "rewards/margins": 1.4060871601104736, + "rewards/rejected": 1.2297108173370361, + "step": 785 + }, + { + "epoch": 0.7539966595084705, + "grad_norm": 14.375, + "ht_mnpo/logit": 0.12588101625442505, + "ht_mnpo/residual": 0.11838103830814362, + "ht_mnpo/residual_abs": 0.13901808857917786, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.2812217756608937e-08, + "logits/chosen": -1.92262864112854, + "logits/rejected": -1.9975627660751343, + "logps/chosen": -0.30886396765708923, + "logps/rejected": -0.2946474552154541, + "loss": 0.2501571774482727, + "loss/core": 0.2501571774482727, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 2.383531093597412, + "rewards/margins": 1.25881028175354, + "rewards/rejected": 1.1247209310531616, + "step": 790 + }, + { + "epoch": 0.7587687902648532, + "grad_norm": 153.0, + "ht_mnpo/logit": 0.20453031361103058, + "ht_mnpo/residual": 0.19703029096126556, + "ht_mnpo/residual_abs": 0.2428302764892578, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.0831748665410763e-08, + "logits/chosen": -1.9690449237823486, + "logits/rejected": -1.9464256763458252, + "logps/chosen": -0.3275276720523834, + "logps/rejected": -0.32991674542427063, + "loss": 0.47860488295555115, + "loss/core": 0.47860488295555115, + "rewards/accuracies": 0.8125, + "rewards/chosen": 4.1561384201049805, + "rewards/margins": 2.0453028678894043, + "rewards/rejected": 2.110835313796997, + "step": 795 + }, + { + "epoch": 0.7635409210212359, + "grad_norm": 62.5, + "ht_mnpo/logit": 0.048203855752944946, + "ht_mnpo/residual": 0.040703851729631424, + "ht_mnpo/residual_abs": 0.19021469354629517, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8937460381244967e-08, + "logits/chosen": -2.151543617248535, + "logits/rejected": -2.2043967247009277, + "logps/chosen": -0.2940928339958191, + "logps/rejected": -0.2848767042160034, + "loss": 0.5195012092590332, + "loss/core": 0.5195012092590332, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 2.649043560028076, + "rewards/margins": 0.4820384383201599, + "rewards/rejected": 2.1670050621032715, + "step": 800 + }, + { + "epoch": 0.7683130517776187, + "grad_norm": 66.0, + "ht_mnpo/logit": 0.1901978701353073, + "ht_mnpo/residual": 0.18269787728786469, + "ht_mnpo/residual_abs": 0.213016077876091, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.713006526846439e-08, + "logits/chosen": -2.0276436805725098, + "logits/rejected": -2.1004996299743652, + "logps/chosen": -0.25963181257247925, + "logps/rejected": -0.2606484591960907, + "loss": 0.2585400938987732, + "loss/core": 0.2585400938987732, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.2736213207244873, + "rewards/margins": 1.9019787311553955, + "rewards/rejected": 1.3716427087783813, + "step": 805 + }, + { + "epoch": 0.7730851825340014, + "grad_norm": 7.46875, + "ht_mnpo/logit": 0.10098608583211899, + "ht_mnpo/residual": 0.09348608553409576, + "ht_mnpo/residual_abs": 0.09852565079927444, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.541024301445404e-08, + "logits/chosen": -2.122680187225342, + "logits/rejected": -2.235227108001709, + "logps/chosen": -0.2972281575202942, + "logps/rejected": -0.2812325060367584, + "loss": 0.060658883303403854, + "loss/core": 0.060658883303403854, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.0125930309295654, + "rewards/margins": 1.0098607540130615, + "rewards/rejected": 1.0027321577072144, + "step": 810 + }, + { + "epoch": 0.7778573132903842, + "grad_norm": 12.0, + "ht_mnpo/logit": 0.1887650042772293, + "ht_mnpo/residual": 0.1812649965286255, + "ht_mnpo/residual_abs": 0.18580570816993713, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.3778640374027983e-08, + "logits/chosen": -2.242436170578003, + "logits/rejected": -2.2910566329956055, + "logps/chosen": -0.2807729244232178, + "logps/rejected": -0.2695391774177551, + "loss": 0.46180328726768494, + "loss/core": 0.46180328726768494, + "rewards/accuracies": 0.9375, + "rewards/chosen": 2.7984201908111572, + "rewards/margins": 1.8876501321792603, + "rewards/rejected": 0.9107701182365417, + "step": 815 + }, + { + "epoch": 0.7826294440467669, + "grad_norm": 0.73828125, + "ht_mnpo/logit": 0.25789007544517517, + "ht_mnpo/residual": 0.25039008259773254, + "ht_mnpo/residual_abs": 0.25934451818466187, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.2235870926211616e-08, + "logits/chosen": -2.1225481033325195, + "logits/rejected": -2.116036891937256, + "logps/chosen": -0.29405006766319275, + "logps/rejected": -0.3059104084968567, + "loss": 0.9970185160636902, + "loss/core": 0.9970185160636902, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 3.760523557662964, + "rewards/margins": 2.5789003372192383, + "rewards/rejected": 1.1816229820251465, + "step": 820 + }, + { + "epoch": 0.7874015748031497, + "grad_norm": 30.25, + "ht_mnpo/logit": 0.12419946491718292, + "ht_mnpo/residual": 0.1166994571685791, + "ht_mnpo/residual_abs": 0.17805686593055725, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.0782514843499652e-08, + "logits/chosen": -2.0180411338806152, + "logits/rejected": -2.0512144565582275, + "logps/chosen": -0.2761440873146057, + "logps/rejected": -0.27113255858421326, + "loss": 0.39623942971229553, + "loss/core": 0.39623942971229553, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.456834554672241, + "rewards/margins": 1.2419946193695068, + "rewards/rejected": 1.2148399353027344, + "step": 825 + }, + { + "epoch": 0.7921737055595324, + "grad_norm": 280.0, + "ht_mnpo/logit": 0.1404004991054535, + "ht_mnpo/residual": 0.13290049135684967, + "ht_mnpo/residual_abs": 0.209401935338974, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 9.419118673676924e-09, + "logits/chosen": -2.013359785079956, + "logits/rejected": -2.045358657836914, + "logps/chosen": -0.2946194112300873, + "logps/rejected": -0.2898050844669342, + "loss": 0.4411865770816803, + "loss/core": 0.4411865770816803, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 2.967160940170288, + "rewards/margins": 1.4040049314498901, + "rewards/rejected": 1.5631558895111084, + "step": 830 + }, + { + "epoch": 0.7969458363159151, + "grad_norm": 7.65625, + "ht_mnpo/logit": 0.1146935448050499, + "ht_mnpo/residual": 0.10719354450702667, + "ht_mnpo/residual_abs": 0.1680370271205902, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.14619513428405e-09, + "logits/chosen": -1.9626811742782593, + "logits/rejected": -1.9111385345458984, + "logps/chosen": -0.3240414261817932, + "logps/rejected": -0.27530187368392944, + "loss": 0.43495216965675354, + "loss/core": 0.43495216965675354, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.719301700592041, + "rewards/margins": 1.1469354629516602, + "rewards/rejected": 1.5723663568496704, + "step": 835 + }, + { + "epoch": 0.8017179670722978, + "grad_norm": 720.0, + "ht_mnpo/logit": -0.0722452700138092, + "ht_mnpo/residual": -0.07974526286125183, + "ht_mnpo/residual_abs": 0.19856789708137512, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.96422291980539e-09, + "logits/chosen": -2.2160840034484863, + "logits/rejected": -2.060520648956299, + "logps/chosen": -0.28456607460975647, + "logps/rejected": -0.29475289583206177, + "loss": 0.600700318813324, + "loss/core": 0.600700318813324, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.7836272716522217, + "rewards/margins": -0.7224526405334473, + "rewards/rejected": 2.506080150604248, + "step": 840 + }, + { + "epoch": 0.8064900978286805, + "grad_norm": 3.734375, + "ht_mnpo/logit": 0.057844363152980804, + "ht_mnpo/residual": 0.05034436658024788, + "ht_mnpo/residual_abs": 0.18020616471767426, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 5.8736465216517594e-09, + "logits/chosen": -2.0028698444366455, + "logits/rejected": -2.0154130458831787, + "logps/chosen": -0.3073137104511261, + "logps/rejected": -0.3115970492362976, + "loss": 0.3085273802280426, + "loss/core": 0.3085273802280426, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 2.327054500579834, + "rewards/margins": 0.5784436464309692, + "rewards/rejected": 1.7486108541488647, + "step": 845 + }, + { + "epoch": 0.8112622285850632, + "grad_norm": 1.3671875, + "ht_mnpo/logit": 0.1011844128370285, + "ht_mnpo/residual": 0.09368441998958588, + "ht_mnpo/residual_abs": 0.09876108914613724, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.874876061005173e-09, + "logits/chosen": -2.039194345474243, + "logits/rejected": -2.047559976577759, + "logps/chosen": -0.28829044103622437, + "logps/rejected": -0.2785061001777649, + "loss": 0.08063577860593796, + "loss/core": 0.08063577860593796, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 2.0972280502319336, + "rewards/margins": 1.0118441581726074, + "rewards/rejected": 1.0853837728500366, + "step": 850 + }, + { + "epoch": 0.816034359341446, + "grad_norm": 1.0234375, + "ht_mnpo/logit": 0.057864684611558914, + "ht_mnpo/residual": 0.050364673137664795, + "ht_mnpo/residual_abs": 0.07226506620645523, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.968287134589188e-09, + "logits/chosen": -2.10422945022583, + "logits/rejected": -2.046278476715088, + "logps/chosen": -0.2990795373916626, + "logps/rejected": -0.28812864422798157, + "loss": 0.02833705022931099, + "loss/core": 0.02833705022931099, + "rewards/accuracies": 0.8374999761581421, + "rewards/chosen": 1.6183888912200928, + "rewards/margins": 0.5786467790603638, + "rewards/rejected": 1.039742112159729, + "step": 855 + }, + { + "epoch": 0.8208064900978287, + "grad_norm": 16.5, + "ht_mnpo/logit": 0.15453629195690155, + "ht_mnpo/residual": 0.14703628420829773, + "ht_mnpo/residual_abs": 0.15095371007919312, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 3.154220673422192e-09, + "logits/chosen": -2.239527702331543, + "logits/rejected": -2.4272561073303223, + "logps/chosen": -0.2691555619239807, + "logps/rejected": -0.25069355964660645, + "loss": 0.25770825147628784, + "loss/core": 0.25770825147628784, + "rewards/accuracies": 0.9125000238418579, + "rewards/chosen": 3.0894973278045654, + "rewards/margins": 1.5453628301620483, + "rewards/rejected": 1.5441343784332275, + "step": 860 + }, + { + "epoch": 0.8255786208542114, + "grad_norm": 1.9140625, + "ht_mnpo/logit": 0.08422042429447174, + "ht_mnpo/residual": 0.07672043144702911, + "ht_mnpo/residual_abs": 0.10378699004650116, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.4329828146074096e-09, + "logits/chosen": -2.023658275604248, + "logits/rejected": -2.0356650352478027, + "logps/chosen": -0.3199964165687561, + "logps/rejected": -0.2985342741012573, + "loss": 0.25588542222976685, + "loss/core": 0.25588542222976685, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": 1.785983681678772, + "rewards/margins": 0.8422042727470398, + "rewards/rejected": 0.9437793493270874, + "step": 865 + }, + { + "epoch": 0.8303507516105941, + "grad_norm": 932.0, + "ht_mnpo/logit": 0.014267271384596825, + "ht_mnpo/residual": 0.006767272017896175, + "ht_mnpo/residual_abs": 0.1964888572692871, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8048447862070714e-09, + "logits/chosen": -2.158407688140869, + "logits/rejected": -2.176384449005127, + "logps/chosen": -0.24673886597156525, + "logps/rejected": -0.25381556153297424, + "loss": 0.5949617624282837, + "loss/core": 0.5949617624282837, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 2.17032790184021, + "rewards/margins": 0.14267274737358093, + "rewards/rejected": 2.0276553630828857, + "step": 870 + }, + { + "epoch": 0.8351228823669768, + "grad_norm": 1672.0, + "ht_mnpo/logit": 0.18652717769145966, + "ht_mnpo/residual": 0.17902716994285583, + "ht_mnpo/residual_abs": 0.22813236713409424, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.2700428052447033e-09, + "logits/chosen": -2.046302080154419, + "logits/rejected": -2.0166726112365723, + "logps/chosen": -0.2716887295246124, + "logps/rejected": -0.29062873125076294, + "loss": 0.7612979412078857, + "loss/core": 0.7612979412078857, + "rewards/accuracies": 0.875, + "rewards/chosen": 2.7709009647369385, + "rewards/margins": 1.8652719259262085, + "rewards/rejected": 0.9056289792060852, + "step": 875 + }, + { + "epoch": 0.8398950131233596, + "grad_norm": 2.5, + "ht_mnpo/logit": 0.04809325188398361, + "ht_mnpo/residual": 0.04059325158596039, + "ht_mnpo/residual_abs": 0.04977937787771225, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 8.287779888734858e-10, + "logits/chosen": -2.058687686920166, + "logits/rejected": -2.1479878425598145, + "logps/chosen": -0.2778651714324951, + "logps/rejected": -0.2754516899585724, + "loss": 0.008843088522553444, + "loss/core": 0.008843088522553444, + "rewards/accuracies": 0.862500011920929, + "rewards/chosen": 1.213937520980835, + "rewards/margins": 0.4809325337409973, + "rewards/rejected": 0.7330051064491272, + "step": 880 + }, + { + "epoch": 0.8446671438797423, + "grad_norm": 1.9765625, + "ht_mnpo/logit": 0.07707710564136505, + "ht_mnpo/residual": 0.06957711279392242, + "ht_mnpo/residual_abs": 0.08699636161327362, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 4.812162787445062e-10, + "logits/chosen": -2.0416557788848877, + "logits/rejected": -2.0932555198669434, + "logps/chosen": -0.3003470003604889, + "logps/rejected": -0.28412142395973206, + "loss": 0.04094807058572769, + "loss/core": 0.04094807058572769, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 1.4210716485977173, + "rewards/margins": 0.7707711458206177, + "rewards/rejected": 0.6503006219863892, + "step": 885 + }, + { + "epoch": 0.849439274636125, + "grad_norm": 648.0, + "ht_mnpo/logit": 0.17258305847644806, + "ht_mnpo/residual": 0.16508305072784424, + "ht_mnpo/residual_abs": 0.2527818977832794, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 2.2748837860270265e-10, + "logits/chosen": -2.100677967071533, + "logits/rejected": -2.1688926219940186, + "logps/chosen": -0.29442304372787476, + "logps/rejected": -0.33285215497016907, + "loss": 0.8560763597488403, + "loss/core": 0.8560763597488403, + "rewards/accuracies": 0.875, + "rewards/chosen": 3.4236044883728027, + "rewards/margins": 1.7258304357528687, + "rewards/rejected": 1.6977739334106445, + "step": 890 + }, + { + "epoch": 0.8542114053925077, + "grad_norm": 16.625, + "ht_mnpo/logit": 0.14334656298160553, + "ht_mnpo/residual": 0.1358465701341629, + "ht_mnpo/residual_abs": 0.14593496918678284, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 6.768970513457151e-11, + "logits/chosen": -2.189239501953125, + "logits/rejected": -2.1437931060791016, + "logps/chosen": -0.30115875601768494, + "logps/rejected": -0.27261587977409363, + "loss": 0.3960161805152893, + "loss/core": 0.3960161805152893, + "rewards/accuracies": 0.7875000238418579, + "rewards/chosen": 2.458888530731201, + "rewards/margins": 1.4334657192230225, + "rewards/rejected": 1.0254226922988892, + "step": 895 + }, + { + "epoch": 0.8589835361488904, + "grad_norm": 0.265625, + "ht_mnpo/logit": 0.10469138622283936, + "ht_mnpo/residual": 0.09719139337539673, + "ht_mnpo/residual_abs": 0.10287211835384369, + "ht_mnpo/target": 0.007499999366700649, + "ht_mnpo/target_abs": 0.007499999366700649, + "learning_rate": 1.8803520859811406e-12, + "logits/chosen": -2.1673264503479004, + "logits/rejected": -2.2037110328674316, + "logps/chosen": -0.3311048448085785, + "logps/rejected": -0.31003594398498535, + "loss": 0.20032985508441925, + "loss/core": 0.20032985508441925, + "rewards/accuracies": 0.887499988079071, + "rewards/chosen": 2.4984869956970215, + "rewards/margins": 1.0469138622283936, + "rewards/rejected": 1.451573133468628, + "step": 900 + }, + { + "epoch": 0.8589835361488904, + "step": 900, + "total_flos": 0.0, + "train_loss": 0.3534543008564247, + "train_runtime": 7070.5487, + "train_samples_per_second": 2.037, + "train_steps_per_second": 0.127 + } + ], + "logging_steps": 5, + "max_steps": 900, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 900, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..00e546d --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3fd21d2f31c23ff96b347ac1c7844aab05bbe95a728241754c2ca301b1288dd +size 6993