70 lines
2.2 KiB
JSON
70 lines
2.2 KiB
JSON
|
{
|
||
|
|
"schema_version": 1,
|
||
|
|
"frozen_at_kst": "2026-07-12T21:40:00+09:00",
|
||
|
|
"base_model": "Qwen/Qwen3-8B",
|
||
|
|
"primary_triple": {
|
||
|
|
"helpfulness": {
|
||
|
|
"model": "RLHFlow/ArmoRM-Llama3-8B-v0.1",
|
||
|
|
"head": "ultrafeedback-helpfulness",
|
||
|
|
"transform": "identity"
|
||
|
|
},
|
||
|
|
"safety": {
|
||
|
|
"model": "RLHFlow/ArmoRM-Llama3-8B-v0.1",
|
||
|
|
"head": "beavertails-is_safe",
|
||
|
|
"transform": "identity"
|
||
|
|
},
|
||
|
|
"conciseness": {
|
||
|
|
"model": "RLHFlow/ArmoRM-Llama3-8B-v0.1",
|
||
|
|
"head": "helpsteer-verbosity",
|
||
|
|
"transform": "negate"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"gate_thresholds": {
|
||
|
|
"raw_score_std_strictly_greater_than": 0.01,
|
||
|
|
"aggregate_prompt_pair_spearman_median_strictly_less_than": 0.0,
|
||
|
|
"aggregate_pairwise_top1_mismatch_at_least": 0.5
|
||
|
|
},
|
||
|
|
"gate_pool": "source-train Qwen3-8B response pool split into deterministic train/validation; sealed source-test excluded",
|
||
|
|
"training_seeds": [42, 43, 44],
|
||
|
|
"optimizer_steps": 900,
|
||
|
|
"effective_batch_size": 16,
|
||
|
|
"decode": {
|
||
|
|
"enable_thinking": false,
|
||
|
|
"temperature": 0.7,
|
||
|
|
"top_p": 0.9,
|
||
|
|
"max_new_tokens": 2048,
|
||
|
|
"decode_seed": 42
|
||
|
|
},
|
||
|
|
"methods": [
|
||
|
|
"ronpo_full_expect",
|
||
|
|
"ronpo_k_only",
|
||
|
|
"dpo",
|
||
|
|
"ipo",
|
||
|
|
"simpo",
|
||
|
|
"kto",
|
||
|
|
"sppo_avg",
|
||
|
|
"inpo_avg",
|
||
|
|
"ht_mnpo_helpfulness",
|
||
|
|
"ht_mnpo_safety",
|
||
|
|
"ht_mnpo_conciseness"
|
||
|
|
],
|
||
|
|
"selection_policy": "No sealed-test selection. One preregistered configuration per method; failures use only the declared stability retry recipe, at most three attempts.",
|
||
|
|
"stability_gate": {
|
||
|
|
"validation_prompts": 128,
|
||
|
|
"min_candidate_base_mean_word_ratio": 0.33,
|
||
|
|
"max_candidate_base_mean_word_ratio": 2.0,
|
||
|
|
"max_consecutive_identical_word_run": 20,
|
||
|
|
"think_tag_count": 0,
|
||
|
|
"empty_response_count": 0,
|
||
|
|
"retry_recipe": "attempt 2 halves learning rate, enables gradient checkpointing, and adds reference=0.02/SFT=0.002 anchoring to unanchored unified losses; attempt 3 quarters learning rate and uses reference=0.05/SFT=0.005; RONPO uses reference=0.05/SFT=0.005 from attempt 1"
|
||
|
|
},
|
||
|
|
"primary_metric": "mean prompt-level minimum of per-prompt min-max-normalized helpfulness, safety, conciseness",
|
||
|
|
"bootstrap": {
|
||
|
|
"unit": "prompt",
|
||
|
|
"paired": true,
|
||
|
|
"resamples": 2000,
|
||
|
|
"interval": "percentile_95",
|
||
|
|
"seed": 42
|
||
|
|
}
|
||
|
|
}
|