初始化项目，由ModelHub XC社区提供模型
Model: promotion/ronpo-qwen3-8b-fair-ronpo-full-expect-s42 Source: Original Platform
This commit is contained in:
ModelHub XC
85
experiment/evaluator_lock.json
Normal file
85
experiment/evaluator_lock.json
Normal file
@@ -0,0 +1,85 @@
|
||||
{
|
||||
"status": "LOCKED_BEFORE_ANY_NEW_METHOD_RANKING",
|
||||
"locked_at": "2026-07-15T20:20:29+09:00",
|
||||
"objective_signals": [
|
||||
"skywork",
|
||||
"armo_safety",
|
||||
"length_conciseness"
|
||||
],
|
||||
"objective_semantics": [
|
||||
"helpfulness",
|
||||
"safety",
|
||||
"conciseness"
|
||||
],
|
||||
"primary": {
|
||||
"name": "open_weight_panel_mean_prompt_worst_vs_base",
|
||||
"definition": "For each prompt and objective, average win=1/tie=0.5/loss=0 over two A/B positions and two judges; take the minimum objective, then mean over prompts.",
|
||||
"judges": [
|
||||
{
|
||||
"model": "openai/gpt-oss-120b",
|
||||
"revision": "b5c939de8f754692c1647ca79fbf85e8c1e70f8a"
|
||||
},
|
||||
{
|
||||
"model": "Qwen/Qwen3-32B",
|
||||
"revision": "9216db5781bf21249d130ec9da846c4624c16137"
|
||||
}
|
||||
],
|
||||
"decode": {
|
||||
"temperature": 0.0,
|
||||
"top_p": 1.0,
|
||||
"max_new_tokens": 512,
|
||||
"seed": 42
|
||||
},
|
||||
"position_swap": true,
|
||||
"validation_selection_rule": "Highest eligible mean prompt-level worst panel score within each method; an exact numeric tie is broken by candidate_id lexical order."
|
||||
},
|
||||
"secondary": {
|
||||
"normalization": "For each locked signal, divide each evaluation prompt's candidate-minus-base raw delta by the population SD of the diagnostic base and matched-control scores; take the prompt-level minimum across objectives, then the mean. No per-prompt min-max.",
|
||||
"diagnostic_control_scale": {
|
||||
"skywork": 22.807301785782975,
|
||||
"armo_safety": 0.3098083353203591,
|
||||
"length_conciseness": 1.5891969646897472
|
||||
},
|
||||
"reward_signal_provenance": {
|
||||
"skywork": {
|
||||
"model": "Skywork/Skywork-Reward-V2-Llama-3.1-8B",
|
||||
"revision": "cba2f842f3f1af2f1b2f0d35e794d789976390c5",
|
||||
"semantics": "helpfulness"
|
||||
},
|
||||
"armo_safety": {
|
||||
"model": "RLHFlow/ArmoRM-Llama3-8B-v0.1",
|
||||
"revision": "eb2676d20da2f2d41082289d23c59b9f7427f955",
|
||||
"head": "beavertails-is_safe",
|
||||
"transform": "identity"
|
||||
},
|
||||
"length_conciseness": {
|
||||
"deterministic": "-log1p(response_word_count)",
|
||||
"tokenization": "Python str.split whitespace words"
|
||||
}
|
||||
},
|
||||
"report_raw_paired_deltas": true
|
||||
},
|
||||
"bootstrap": {
|
||||
"paired_prompt_resamples": 2000,
|
||||
"seed": 42,
|
||||
"interval": "percentile_95"
|
||||
},
|
||||
"power": {
|
||||
"target_absolute_effect": 0.05,
|
||||
"paired_sd": 0.13831629563357775,
|
||||
"required_prompts_80pct_power": 61,
|
||||
"planned_fresh_test_prompts": 1024,
|
||||
"alpha_two_sided": 0.05,
|
||||
"power": 0.8
|
||||
},
|
||||
"fresh_test_source": {
|
||||
"dataset": "HuggingFaceH4/ultrachat_200k",
|
||||
"revision": "8049631c405ae6576f93f445c6b8166f76f5505a",
|
||||
"split": "test_sft"
|
||||
},
|
||||
"diagnostic_summary_sha256": "e365ed814926ac5c17777e0fc65d0299a45b81bd390542889bbd55b5453d4ffe",
|
||||
"judge_diagnostic_lock_sha256": "fce2d97806053cabdf3d358806f4baa80df7f89d71d8c1c298883327d4e02635",
|
||||
"prereg_sha256": "71cbd1ac17c3e807f2ece1cb0a82374bbe74edb654c2845b97c3222ed680f0d0",
|
||||
"method_ranking_computed": false,
|
||||
"spent_sealed_split_touched": false
|
||||
}
|
||||
99
experiment/selection_lock.json
Normal file
99
experiment/selection_lock.json
Normal file
@@ -0,0 +1,99 @@
|
||||
{
|
||||
"status": "VALIDATION_SELECTION_LOCKED_BEFORE_FRESH_TEST",
|
||||
"locked_at": "2026-07-15T20:37:11+09:00",
|
||||
"selection_metric": "open_weight_panel_mean_prompt_worst_vs_base",
|
||||
"tie_break": "candidate_id lexical order",
|
||||
"selected_by_method": {
|
||||
"dpo": {
|
||||
"candidate_id": "dpo_b",
|
||||
"validation_primary": 0.40625,
|
||||
"validation_primary_ci95": [
|
||||
0.3837890625,
|
||||
0.427734375
|
||||
],
|
||||
"eligible_candidates": [
|
||||
"dpo_a",
|
||||
"dpo_b"
|
||||
]
|
||||
},
|
||||
"ht_mnpo_helpfulness": {
|
||||
"candidate_id": "ht_help_b",
|
||||
"validation_primary": 0.4072265625,
|
||||
"validation_primary_ci95": [
|
||||
0.3828125,
|
||||
0.4296875
|
||||
],
|
||||
"eligible_candidates": [
|
||||
"ht_help_b"
|
||||
]
|
||||
},
|
||||
"ht_mnpo_safety": {
|
||||
"candidate_id": "ht_safety_a",
|
||||
"validation_primary": 0.4150390625,
|
||||
"validation_primary_ci95": [
|
||||
0.3925537109375,
|
||||
0.4384765625
|
||||
],
|
||||
"eligible_candidates": [
|
||||
"ht_safety_a",
|
||||
"ht_safety_b"
|
||||
]
|
||||
},
|
||||
"inpo_avg": {
|
||||
"candidate_id": "inpo_b",
|
||||
"validation_primary": 0.40625,
|
||||
"validation_primary_ci95": [
|
||||
0.3818359375,
|
||||
0.427734375
|
||||
],
|
||||
"eligible_candidates": [
|
||||
"inpo_b"
|
||||
]
|
||||
},
|
||||
"ronpo_full_expect": {
|
||||
"candidate_id": "ronpo_full_a",
|
||||
"validation_primary": 0.4033203125,
|
||||
"validation_primary_ci95": [
|
||||
0.37890625,
|
||||
0.42580566406249987
|
||||
],
|
||||
"eligible_candidates": [
|
||||
"ronpo_full_a",
|
||||
"ronpo_full_b"
|
||||
]
|
||||
},
|
||||
"ronpo_k_only": {
|
||||
"candidate_id": "ronpo_top_a",
|
||||
"validation_primary": 0.412109375,
|
||||
"validation_primary_ci95": [
|
||||
0.3876953125,
|
||||
0.43557128906249987
|
||||
],
|
||||
"eligible_candidates": [
|
||||
"ronpo_top_a"
|
||||
]
|
||||
},
|
||||
"sppo_avg": {
|
||||
"candidate_id": "sppo_b",
|
||||
"validation_primary": 0.4111328125,
|
||||
"validation_primary_ci95": [
|
||||
0.3866943359375,
|
||||
0.4345703125
|
||||
],
|
||||
"eligible_candidates": [
|
||||
"sppo_b"
|
||||
]
|
||||
}
|
||||
},
|
||||
"selected_ronpo_overall": "ronpo_top_a",
|
||||
"failed_methods": [
|
||||
"ht_mnpo_conciseness",
|
||||
"ipo",
|
||||
"simpo"
|
||||
],
|
||||
"panel_summary_sha256": "8850f23e16cadf56a99e3d562d224ce5337ab36fa588d8a843382853e6ad17bd",
|
||||
"evaluator_lock_sha256": "ab7b12c84932ed57acfec7edbcbc1ff1df5e9acd203869b1458343adf6e5b2c8",
|
||||
"grid_sha256": "abfd982ef10c3bc71364752241150286ebf06be5e97f087596adf80e55d459e0",
|
||||
"fresh_test_opened": false,
|
||||
"spent_sealed_split_touched": false
|
||||
}
|
||||
46
experiment/sweep_grid.json
Normal file
46
experiment/sweep_grid.json
Normal file
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"status": "frozen_before_training_and_validation_ranking",
|
||||
"frozen_at_kst": "2026-07-15T09:01:45+09:00",
|
||||
"seed": 42,
|
||||
"common": {
|
||||
"optimizer_steps": 900,
|
||||
"effective_batch_size": 16,
|
||||
"per_device_train_batch_size": 1,
|
||||
"gradient_accumulation_steps": 16,
|
||||
"gradient_checkpointing": true,
|
||||
"bf16": true,
|
||||
"attn_implementation": "sdpa",
|
||||
"cudnn_sdpa": false,
|
||||
"lr_scheduler_type": "cosine",
|
||||
"max_length": 2048,
|
||||
"max_prompt_length": 1800,
|
||||
"save_total_limit": 1,
|
||||
"wandb_entity": "promotion-kim",
|
||||
"wandb_project": "mnpo",
|
||||
"wandb_group": "qwen3-8b-fair-demo-20260715"
|
||||
},
|
||||
"budget_rule": "Exactly two configs per reported method; no extension after validation rankings are visible.",
|
||||
"candidates": [
|
||||
{"id":"ronpo_full_a","method":"ronpo_full_expect","dataset":"ronpo_full_expect_kall","learning_rate":1e-7,"warmup_ratio":0.15,"ronpo_alpha":0.25,"ronpo_tau":0.10,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075,"expected_support_k":0},
|
||||
{"id":"ronpo_full_b","method":"ronpo_full_expect","dataset":"ronpo_full_expect_k6","learning_rate":2.5e-7,"warmup_ratio":0.20,"ronpo_alpha":0.35,"ronpo_tau":0.05,"reference_anchor_weight":0.10,"preference_sft_weight":0.01,"expected_support_k":6},
|
||||
{"id":"ronpo_top_a","method":"ronpo_k_only","dataset":"ronpo_k_only_k1","learning_rate":5e-8,"warmup_ratio":0.20,"ronpo_alpha":0.15,"ronpo_tau":0.10,"reference_anchor_weight":0.10,"preference_sft_weight":0.01,"expected_support_k":1},
|
||||
{"id":"ronpo_top_b","method":"ronpo_k_only","dataset":"ronpo_k_only_k2","learning_rate":1e-7,"warmup_ratio":0.15,"ronpo_alpha":0.25,"ronpo_tau":0.10,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075,"expected_support_k":2},
|
||||
{"id":"dpo_a","method":"dpo","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.10,"dpo_beta":0.05,"reference_anchor_weight":0.02,"preference_sft_weight":0.002},
|
||||
{"id":"dpo_b","method":"dpo","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.15,"dpo_beta":0.10,"reference_anchor_weight":0.05,"preference_sft_weight":0.005},
|
||||
{"id":"ipo_a","method":"ipo","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.15,"dpo_beta":0.20,"reference_anchor_weight":0.05,"preference_sft_weight":0.005},
|
||||
{"id":"ipo_b","method":"ipo","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.20,"dpo_beta":0.50,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075},
|
||||
{"id":"simpo_a","method":"simpo","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.10,"simpo_beta":1.0,"simpo_gamma":0.3,"reference_anchor_weight":0.02,"preference_sft_weight":0.002},
|
||||
{"id":"simpo_b","method":"simpo","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.15,"simpo_beta":2.0,"simpo_gamma":0.6,"reference_anchor_weight":0.05,"preference_sft_weight":0.005},
|
||||
{"id":"sppo_a","method":"sppo_avg","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.15,"eta":0.005,"reference_anchor_weight":0.05,"preference_sft_weight":0.005},
|
||||
{"id":"sppo_b","method":"sppo_avg","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.20,"eta":0.010,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075},
|
||||
{"id":"inpo_a","method":"inpo_avg","dataset":"avg","learning_rate":2.5e-7,"warmup_ratio":0.15,"eta":0.050,"ratio":0.3333,"reference_anchor_weight":0.05,"preference_sft_weight":0.005},
|
||||
{"id":"inpo_b","method":"inpo_avg","dataset":"avg","learning_rate":5e-7,"warmup_ratio":0.20,"eta":0.100,"ratio":0.3333,"reference_anchor_weight":0.075,"preference_sft_weight":0.0075},
|
||||
{"id":"ht_help_a","method":"ht_mnpo_helpfulness","dataset":"ht_mnpo_helpfulness","learning_rate":2.5e-7,"warmup_ratio":0.10,"eta":0.005,"reference_anchor_weight":0.02,"preference_sft_weight":0.002},
|
||||
{"id":"ht_help_b","method":"ht_mnpo_helpfulness","dataset":"ht_mnpo_helpfulness","learning_rate":5e-7,"warmup_ratio":0.15,"eta":0.010,"reference_anchor_weight":0.05,"preference_sft_weight":0.005},
|
||||
{"id":"ht_safety_a","method":"ht_mnpo_safety","dataset":"ht_mnpo_safety","learning_rate":2.5e-7,"warmup_ratio":0.10,"eta":0.005,"reference_anchor_weight":0.02,"preference_sft_weight":0.002},
|
||||
{"id":"ht_safety_b","method":"ht_mnpo_safety","dataset":"ht_mnpo_safety","learning_rate":5e-7,"warmup_ratio":0.15,"eta":0.010,"reference_anchor_weight":0.05,"preference_sft_weight":0.005},
|
||||
{"id":"ht_concise_a","method":"ht_mnpo_conciseness","dataset":"ht_mnpo_conciseness","learning_rate":2.5e-7,"warmup_ratio":0.10,"eta":0.005,"reference_anchor_weight":0.02,"preference_sft_weight":0.002},
|
||||
{"id":"ht_concise_b","method":"ht_mnpo_conciseness","dataset":"ht_mnpo_conciseness","learning_rate":5e-7,"warmup_ratio":0.15,"eta":0.010,"reference_anchor_weight":0.05,"preference_sft_weight":0.005}
|
||||
]
|
||||
}
|
||||
19
experiment/training_status.json
Normal file
19
experiment/training_status.json
Normal file
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"status": "completed",
|
||||
"candidate_id": "ronpo_full_a",
|
||||
"method": "ronpo_full_expect",
|
||||
"seed": 42,
|
||||
"gpu": 1,
|
||||
"config": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/fair_demo_20260715/sweep/candidates/ronpo_full_a/config.yaml",
|
||||
"dataset": "/NHNHOME/26msit001_A/BASE/AIPR/sjkim/revision_qwen3_8b/full_iter1/flagship_20260712/precomputed/ronpo_full_expect",
|
||||
"optimizer_steps": 900,
|
||||
"effective_batch_size": 16,
|
||||
"wandb_run_id": "234ea82cbcb3",
|
||||
"wandb_url": "https://wandb.ai/promotion-kim/mnpo/runs/234ea82cbcb3",
|
||||
"launcher": "supplemental_one_candidate",
|
||||
"started_at": "2026-07-15T18:02:38+09:00",
|
||||
"returncode": 0,
|
||||
"model_complete": true,
|
||||
"finite_results": true,
|
||||
"completed_at": "2026-07-15T20:05:18+09:00"
|
||||
}
|
||||
Reference in New Issue
Block a user