85c838abd0
Model: mntunur/Qwen2.5-0.5B-Instruct-Gensyn-Swarm-reclusive_bristly_horse Source: Original Platform
1944 lines
74 KiB
JSON
1944 lines
74 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 5.714285714285714,
|
|
"eval_steps": 500,
|
|
"global_step": 200,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"completion_length": 259.0625,
|
|
"epoch": 0.05714285714285714,
|
|
"grad_norm": 40.40498733520508,
|
|
"kl": 0.0,
|
|
"learning_rate": 1.6666666666666665e-07,
|
|
"loss": -0.0001,
|
|
"reward": 4.278493709862232,
|
|
"reward_std": 1.2902945003088462,
|
|
"rewards/concensus_correctness_reward_func": 1.7047499865293503,
|
|
"rewards/consensus_reward_func": 1.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0625,
|
|
"rewards/question_recreation_reward_func": 0.675962442997843,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.1875,
|
|
"rewards/xmlcount_reward_func": 0.6477812505327165,
|
|
"step": 2
|
|
},
|
|
{
|
|
"completion_length": 131.8125,
|
|
"epoch": 0.11428571428571428,
|
|
"grad_norm": 11.698363304138184,
|
|
"kl": 0.05078949875195282,
|
|
"learning_rate": 5e-07,
|
|
"loss": 0.0,
|
|
"reward": 6.484069406986237,
|
|
"reward_std": 0.30921973334625363,
|
|
"rewards/concensus_correctness_reward_func": 1.9382499903440475,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.1875,
|
|
"rewards/question_recreation_reward_func": 0.866131940856576,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.40625,
|
|
"rewards/xmlcount_reward_func": 1.0859375,
|
|
"step": 4
|
|
},
|
|
{
|
|
"completion_length": 167.59375,
|
|
"epoch": 0.17142857142857143,
|
|
"grad_norm": 113298272.0,
|
|
"kl": 3785875.3213646472,
|
|
"learning_rate": 8.333333333333333e-07,
|
|
"loss": 3785.876,
|
|
"reward": 6.581158250570297,
|
|
"reward_std": 0.3191240727901459,
|
|
"rewards/concensus_correctness_reward_func": 2.0427499786019325,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0625,
|
|
"rewards/question_recreation_reward_func": 0.8977832514792681,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.421875,
|
|
"rewards/xmlcount_reward_func": 1.15625,
|
|
"step": 6
|
|
},
|
|
{
|
|
"completion_length": 139.6875,
|
|
"epoch": 0.22857142857142856,
|
|
"grad_norm": 898.8678588867188,
|
|
"kl": 377.1116261743221,
|
|
"learning_rate": 9.99934441832816e-07,
|
|
"loss": 0.3771,
|
|
"reward": 6.764802664518356,
|
|
"reward_std": 0.35591571987606585,
|
|
"rewards/concensus_correctness_reward_func": 2.1679999828338623,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.1875,
|
|
"rewards/question_recreation_reward_func": 0.8858651723712683,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.375,
|
|
"rewards/xmlcount_reward_func": 1.1484375,
|
|
"step": 8
|
|
},
|
|
{
|
|
"completion_length": 150.0625,
|
|
"epoch": 0.2857142857142857,
|
|
"grad_norm": 2745.2119140625,
|
|
"kl": 198.94737304830204,
|
|
"learning_rate": 9.994100796397953e-07,
|
|
"loss": 0.1989,
|
|
"reward": 6.304591566324234,
|
|
"reward_std": 0.26397108286619186,
|
|
"rewards/concensus_correctness_reward_func": 1.9194999784231186,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.8811853062361479,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.375,
|
|
"rewards/xmlcount_reward_func": 1.12890625,
|
|
"step": 10
|
|
},
|
|
{
|
|
"completion_length": 157.65625,
|
|
"epoch": 0.34285714285714286,
|
|
"grad_norm": 47.17015075683594,
|
|
"kl": 10.171083455644862,
|
|
"learning_rate": 9.983619052372847e-07,
|
|
"loss": 0.0102,
|
|
"reward": 6.150758773088455,
|
|
"reward_std": 0.26591456681489944,
|
|
"rewards/concensus_correctness_reward_func": 1.9152499809861183,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.8178525845287368,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.34375,
|
|
"rewards/xmlcount_reward_func": 1.0739062502980232,
|
|
"step": 12
|
|
},
|
|
{
|
|
"completion_length": 127.9375,
|
|
"epoch": 0.4,
|
|
"grad_norm": 30.31770896911621,
|
|
"kl": 1.7561549306603865,
|
|
"learning_rate": 9.967910180154888e-07,
|
|
"loss": 0.0018,
|
|
"reward": 6.409551799297333,
|
|
"reward_std": 0.0954170327167958,
|
|
"rewards/concensus_correctness_reward_func": 1.9182499796152115,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.8545830636285245,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.4375,
|
|
"rewards/xmlcount_reward_func": 1.19921875,
|
|
"step": 14
|
|
},
|
|
{
|
|
"completion_length": 125.1875,
|
|
"epoch": 0.45714285714285713,
|
|
"grad_norm": 21.316240310668945,
|
|
"kl": 61.69702582350237,
|
|
"learning_rate": 9.946990656181779e-07,
|
|
"loss": 0.0617,
|
|
"reward": 7.006136029958725,
|
|
"reward_std": 0.3882664368939004,
|
|
"rewards/concensus_correctness_reward_func": 2.293499991297722,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.3125,
|
|
"rewards/question_recreation_reward_func": 0.8649797681719065,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.40625,
|
|
"rewards/xmlcount_reward_func": 1.12890625,
|
|
"step": 16
|
|
},
|
|
{
|
|
"completion_length": 117.84375,
|
|
"epoch": 0.5142857142857142,
|
|
"grad_norm": 0.30895093083381653,
|
|
"kl": 3.9632333860547533,
|
|
"learning_rate": 9.92088242214537e-07,
|
|
"loss": 0.004,
|
|
"reward": 7.0876125395298,
|
|
"reward_std": 0.02687215618789196,
|
|
"rewards/concensus_correctness_reward_func": 2.1668749675154686,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.25,
|
|
"rewards/question_recreation_reward_func": 0.9754250645637512,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.46875,
|
|
"rewards/xmlcount_reward_func": 1.2265625,
|
|
"step": 18
|
|
},
|
|
{
|
|
"completion_length": 121.875,
|
|
"epoch": 0.5714285714285714,
|
|
"grad_norm": 27.664838790893555,
|
|
"kl": 20.31627042979403,
|
|
"learning_rate": 9.889612861977853e-07,
|
|
"loss": 0.0203,
|
|
"reward": 6.253143697977066,
|
|
"reward_std": 0.15424543642438948,
|
|
"rewards/concensus_correctness_reward_func": 1.9189999774098396,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.9044562540948391,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.328125,
|
|
"rewards/xmlcount_reward_func": 1.1015625,
|
|
"step": 20
|
|
},
|
|
{
|
|
"completion_length": 116.15625,
|
|
"epoch": 0.6285714285714286,
|
|
"grad_norm": 43.6223030090332,
|
|
"kl": 7.602870227153744,
|
|
"learning_rate": 9.853214773129795e-07,
|
|
"loss": 0.0076,
|
|
"reward": 6.478605628013611,
|
|
"reward_std": 0.09073211625218391,
|
|
"rewards/concensus_correctness_reward_func": 1.918249987065792,
|
|
"rewards/consensus_reward_func": 1.875,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.9867619276046753,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.46875,
|
|
"rewards/xmlcount_reward_func": 1.2298437505960464,
|
|
"step": 22
|
|
},
|
|
{
|
|
"completion_length": 144.28125,
|
|
"epoch": 0.6857142857142857,
|
|
"grad_norm": 208.64044189453125,
|
|
"kl": 1171.1961524431879,
|
|
"learning_rate": 9.81172633217015e-07,
|
|
"loss": 1.1712,
|
|
"reward": 6.415135711431503,
|
|
"reward_std": 0.1869081521872431,
|
|
"rewards/concensus_correctness_reward_func": 1.9849999640136957,
|
|
"rewards/consensus_reward_func": 1.875,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.3125,
|
|
"rewards/question_recreation_reward_func": 0.8572919256985188,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.328125,
|
|
"rewards/xmlcount_reward_func": 1.057218749076128,
|
|
"step": 24
|
|
},
|
|
{
|
|
"completion_length": 125.46875,
|
|
"epoch": 0.7428571428571429,
|
|
"grad_norm": 17.30021858215332,
|
|
"kl": 22246.188865144257,
|
|
"learning_rate": 9.765191054744304e-07,
|
|
"loss": 22.2462,
|
|
"reward": 6.495312124490738,
|
|
"reward_std": 0.2721763923764229,
|
|
"rewards/concensus_correctness_reward_func": 1.8801249898970127,
|
|
"rewards/consensus_reward_func": 1.875,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.9621558953076601,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.453125,
|
|
"rewards/xmlcount_reward_func": 1.1999062523245811,
|
|
"step": 26
|
|
},
|
|
{
|
|
"completion_length": 110.0625,
|
|
"epoch": 0.8,
|
|
"grad_norm": 0.00991878192871809,
|
|
"kl": 0.025511330234024854,
|
|
"learning_rate": 9.713657749932171e-07,
|
|
"loss": 0.0,
|
|
"reward": 6.9107471108436584,
|
|
"reward_std": 0.0035395766608417034,
|
|
"rewards/concensus_correctness_reward_func": 2.038249984383583,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.9974971413612366,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.5,
|
|
"rewards/xmlcount_reward_func": 1.25,
|
|
"step": 28
|
|
},
|
|
{
|
|
"completion_length": 110.5625,
|
|
"epoch": 0.8571428571428571,
|
|
"grad_norm": 3189.01806640625,
|
|
"kl": 254.6465662113756,
|
|
"learning_rate": 9.657180469054212e-07,
|
|
"loss": 0.2546,
|
|
"reward": 7.057701647281647,
|
|
"reward_std": 0.15232454985380173,
|
|
"rewards/concensus_correctness_reward_func": 2.16849996894598,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.25,
|
|
"rewards/question_recreation_reward_func": 0.9595141224563122,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.46875,
|
|
"rewards/xmlcount_reward_func": 1.2109375,
|
|
"step": 30
|
|
},
|
|
{
|
|
"completion_length": 144.03125,
|
|
"epoch": 0.9142857142857143,
|
|
"grad_norm": 1612504.375,
|
|
"kl": 182371.5907895875,
|
|
"learning_rate": 9.59581844897906e-07,
|
|
"loss": 182.3716,
|
|
"reward": 6.731081739068031,
|
|
"reward_std": 0.1562379876850173,
|
|
"rewards/concensus_correctness_reward_func": 2.1022499799728394,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.25,
|
|
"rewards/question_recreation_reward_func": 0.8358630024595186,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.40625,
|
|
"rewards/xmlcount_reward_func": 1.13671875,
|
|
"step": 32
|
|
},
|
|
{
|
|
"completion_length": 120.0,
|
|
"epoch": 0.9714285714285714,
|
|
"grad_norm": 0.11929623037576675,
|
|
"kl": 11.570230805784433,
|
|
"learning_rate": 9.529636049992233e-07,
|
|
"loss": 0.0116,
|
|
"reward": 6.652986168861389,
|
|
"reward_std": 0.16899008117616177,
|
|
"rewards/concensus_correctness_reward_func": 2.043749988079071,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0625,
|
|
"rewards/question_recreation_reward_func": 0.9419549033045769,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.421875,
|
|
"rewards/xmlcount_reward_func": 1.1829062476754189,
|
|
"step": 34
|
|
},
|
|
{
|
|
"completion_length": 155.0625,
|
|
"epoch": 1.0285714285714285,
|
|
"grad_norm": 26.967811584472656,
|
|
"kl": 41.41832308189555,
|
|
"learning_rate": 9.458702688291071e-07,
|
|
"loss": 0.0414,
|
|
"reward": 6.463270157575607,
|
|
"reward_std": 0.0986387450248003,
|
|
"rewards/concensus_correctness_reward_func": 1.978499986231327,
|
|
"rewards/consensus_reward_func": 1.875,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.9144576657563448,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.421875,
|
|
"rewards/xmlcount_reward_func": 1.1484375,
|
|
"step": 36
|
|
},
|
|
{
|
|
"completion_length": 172.125,
|
|
"epoch": 1.0857142857142856,
|
|
"grad_norm": 9184.3974609375,
|
|
"kl": 382.0947194062253,
|
|
"learning_rate": 9.383092763176738e-07,
|
|
"loss": 0.3821,
|
|
"reward": 6.269357413053513,
|
|
"reward_std": 0.37034115626011044,
|
|
"rewards/concensus_correctness_reward_func": 1.8153749704360962,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0625,
|
|
"rewards/question_recreation_reward_func": 0.8684198576956987,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.40625,
|
|
"rewards/xmlcount_reward_func": 1.1168124992400408,
|
|
"step": 38
|
|
},
|
|
{
|
|
"completion_length": 162.34375,
|
|
"epoch": 1.1428571428571428,
|
|
"grad_norm": 85.47990417480469,
|
|
"kl": 20.64761355567407,
|
|
"learning_rate": 9.302885579019626e-07,
|
|
"loss": 0.0206,
|
|
"reward": 6.838260769844055,
|
|
"reward_std": 0.28286098898388445,
|
|
"rewards/concensus_correctness_reward_func": 2.168374963104725,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.25,
|
|
"rewards/question_recreation_reward_func": 0.8730107028968632,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.40625,
|
|
"rewards/xmlcount_reward_func": 1.140625,
|
|
"step": 40
|
|
},
|
|
{
|
|
"completion_length": 238.5625,
|
|
"epoch": 1.2,
|
|
"grad_norm": 24413.638671875,
|
|
"kl": 7399.846957861285,
|
|
"learning_rate": 9.218165262080022e-07,
|
|
"loss": 7.3998,
|
|
"reward": 6.1828261613845825,
|
|
"reward_std": 0.09892372484318912,
|
|
"rewards/concensus_correctness_reward_func": 1.9538749903440475,
|
|
"rewards/consensus_reward_func": 1.875,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.7445762166171335,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.375,
|
|
"rewards/xmlcount_reward_func": 1.109375,
|
|
"step": 42
|
|
},
|
|
{
|
|
"completion_length": 121.21875,
|
|
"epoch": 1.2571428571428571,
|
|
"grad_norm": 628.685302734375,
|
|
"kl": 196.11569420348624,
|
|
"learning_rate": 9.129020672271281e-07,
|
|
"loss": 0.1961,
|
|
"reward": 6.581244736909866,
|
|
"reward_std": 0.20140560899744742,
|
|
"rewards/concensus_correctness_reward_func": 2.038249984383583,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.875025988323614,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.390625,
|
|
"rewards/xmlcount_reward_func": 1.15234375,
|
|
"step": 44
|
|
},
|
|
{
|
|
"completion_length": 144.90625,
|
|
"epoch": 1.3142857142857143,
|
|
"grad_norm": 38.4938850402832,
|
|
"kl": 14.245073152454609,
|
|
"learning_rate": 9.035545309958046e-07,
|
|
"loss": 0.0142,
|
|
"reward": 6.480484366416931,
|
|
"reward_std": 0.2079393861349672,
|
|
"rewards/concensus_correctness_reward_func": 1.920499972999096,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.8879531715065241,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.40625,
|
|
"rewards/xmlcount_reward_func": 1.1407812498509884,
|
|
"step": 46
|
|
},
|
|
{
|
|
"completion_length": 169.46875,
|
|
"epoch": 1.3714285714285714,
|
|
"grad_norm": 40.683929443359375,
|
|
"kl": 40.01735451040594,
|
|
"learning_rate": 8.937837217887272e-07,
|
|
"loss": 0.04,
|
|
"reward": 6.485899716615677,
|
|
"reward_std": 0.30478080292232335,
|
|
"rewards/concensus_correctness_reward_func": 2.0423749834299088,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.8660247628577054,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.40625,
|
|
"rewards/xmlcount_reward_func": 1.0462500005960464,
|
|
"step": 48
|
|
},
|
|
{
|
|
"completion_length": 323.28125,
|
|
"epoch": 1.4285714285714286,
|
|
"grad_norm": 136.50091552734375,
|
|
"kl": 208.73060331924557,
|
|
"learning_rate": 8.83599887835493e-07,
|
|
"loss": 0.2087,
|
|
"reward": 5.868565648794174,
|
|
"reward_std": 0.40139913094753865,
|
|
"rewards/concensus_correctness_reward_func": 1.9188749939203262,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.6675031447084621,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.28125,
|
|
"rewards/xmlcount_reward_func": 1.0009374991059303,
|
|
"step": 50
|
|
},
|
|
{
|
|
"completion_length": 223.90625,
|
|
"epoch": 1.4857142857142858,
|
|
"grad_norm": 3347453.5,
|
|
"kl": 294342.96547964675,
|
|
"learning_rate": 8.73013710571623e-07,
|
|
"loss": 294.343,
|
|
"reward": 6.5324656665325165,
|
|
"reward_std": 0.18765212316066027,
|
|
"rewards/concensus_correctness_reward_func": 2.0436249747872353,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.836496805306524,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.40625,
|
|
"rewards/xmlcount_reward_func": 1.12109375,
|
|
"step": 52
|
|
},
|
|
{
|
|
"completion_length": 108.03125,
|
|
"epoch": 1.5428571428571427,
|
|
"grad_norm": 5130.6298828125,
|
|
"kl": 589.7922593226319,
|
|
"learning_rate": 8.620362934352108e-07,
|
|
"loss": 0.5898,
|
|
"reward": 6.766858085989952,
|
|
"reward_std": 0.38228555023670197,
|
|
"rewards/concensus_correctness_reward_func": 2.107999976724386,
|
|
"rewards/consensus_reward_func": 1.9375,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.25,
|
|
"rewards/question_recreation_reward_func": 0.8854205666575581,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.4375,
|
|
"rewards/xmlcount_reward_func": 1.1484375,
|
|
"step": 54
|
|
},
|
|
{
|
|
"completion_length": 197.375,
|
|
"epoch": 1.6,
|
|
"grad_norm": 57.51034164428711,
|
|
"kl": 17.006517063246605,
|
|
"learning_rate": 8.506791502209496e-07,
|
|
"loss": 0.017,
|
|
"reward": 7.339930027723312,
|
|
"reward_std": 0.07264970572759921,
|
|
"rewards/concensus_correctness_reward_func": 2.541249990463257,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.5,
|
|
"rewards/question_recreation_reward_func": 0.8182113186921924,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.375,
|
|
"rewards/xmlcount_reward_func": 1.10546875,
|
|
"step": 56
|
|
},
|
|
{
|
|
"completion_length": 121.5625,
|
|
"epoch": 1.657142857142857,
|
|
"grad_norm": 64.63337707519531,
|
|
"kl": 37.444439904923236,
|
|
"learning_rate": 8.389541930037516e-07,
|
|
"loss": 0.0374,
|
|
"reward": 6.30495747923851,
|
|
"reward_std": 0.06559963073232211,
|
|
"rewards/concensus_correctness_reward_func": 1.9192499816417694,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.8310199771076441,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.40625,
|
|
"rewards/xmlcount_reward_func": 1.1484375,
|
|
"step": 58
|
|
},
|
|
{
|
|
"completion_length": 266.6875,
|
|
"epoch": 1.7142857142857144,
|
|
"grad_norm": 81.60047912597656,
|
|
"kl": 40.02249676991286,
|
|
"learning_rate": 8.268737196446263e-07,
|
|
"loss": 0.04,
|
|
"reward": 6.112664610147476,
|
|
"reward_std": 0.12300743258674629,
|
|
"rewards/concensus_correctness_reward_func": 1.9388749822974205,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.6816021917038597,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.328125,
|
|
"rewards/xmlcount_reward_func": 1.0390625,
|
|
"step": 60
|
|
},
|
|
{
|
|
"completion_length": 168.25,
|
|
"epoch": 1.7714285714285714,
|
|
"grad_norm": 7163.0244140625,
|
|
"kl": 595.5683929037647,
|
|
"learning_rate": 8.144504008919222e-07,
|
|
"loss": 0.5956,
|
|
"reward": 6.3996052742004395,
|
|
"reward_std": 0.0010765780170913786,
|
|
"rewards/concensus_correctness_reward_func": 1.917749971151352,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.8724802895449102,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.4375,
|
|
"rewards/xmlcount_reward_func": 1.171875,
|
|
"step": 62
|
|
},
|
|
{
|
|
"completion_length": 110.59375,
|
|
"epoch": 1.8285714285714287,
|
|
"grad_norm": 7897.408203125,
|
|
"kl": 480.5656087023023,
|
|
"learning_rate": 8.016972670914623e-07,
|
|
"loss": 0.4806,
|
|
"reward": 7.056159615516663,
|
|
"reward_std": 0.06930830026976764,
|
|
"rewards/concensus_correctness_reward_func": 2.169124998152256,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.25,
|
|
"rewards/question_recreation_reward_func": 0.953440960496664,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.453125,
|
|
"rewards/xmlcount_reward_func": 1.23046875,
|
|
"step": 64
|
|
},
|
|
{
|
|
"completion_length": 180.96875,
|
|
"epoch": 1.8857142857142857,
|
|
"grad_norm": 168.32589721679688,
|
|
"kl": 21.501576103312345,
|
|
"learning_rate": 7.886276945195097e-07,
|
|
"loss": 0.0215,
|
|
"reward": 6.382116913795471,
|
|
"reward_std": 0.37760969676310197,
|
|
"rewards/concensus_correctness_reward_func": 2.043124981224537,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0625,
|
|
"rewards/question_recreation_reward_func": 0.7921168450266123,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.375,
|
|
"rewards/xmlcount_reward_func": 1.109375,
|
|
"step": 66
|
|
},
|
|
{
|
|
"completion_length": 232.65625,
|
|
"epoch": 1.9428571428571428,
|
|
"grad_norm": 129.97056579589844,
|
|
"kl": 17.854988246223172,
|
|
"learning_rate": 7.752553913529018e-07,
|
|
"loss": 0.0179,
|
|
"reward": 6.253960728645325,
|
|
"reward_std": 0.4535234868526459,
|
|
"rewards/concensus_correctness_reward_func": 2.0708749908953905,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.1875,
|
|
"rewards/question_recreation_reward_func": 0.6741795390844345,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.28125,
|
|
"rewards/xmlcount_reward_func": 1.0401562489569187,
|
|
"step": 68
|
|
},
|
|
{
|
|
"completion_length": 143.1875,
|
|
"epoch": 2.0,
|
|
"grad_norm": 72.03203582763672,
|
|
"kl": 3.0041830078563976,
|
|
"learning_rate": 7.61594383291065e-07,
|
|
"loss": 0.003,
|
|
"reward": 6.1214989721775055,
|
|
"reward_std": 0.33512147367582656,
|
|
"rewards/concensus_correctness_reward_func": 2.042374961078167,
|
|
"rewards/consensus_reward_func": 1.75,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0625,
|
|
"rewards/question_recreation_reward_func": 0.8236239920370281,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.359375,
|
|
"rewards/xmlcount_reward_func": 1.0836249999701977,
|
|
"step": 70
|
|
},
|
|
{
|
|
"completion_length": 152.0,
|
|
"epoch": 2.057142857142857,
|
|
"grad_norm": 43.813533782958984,
|
|
"kl": 6.536038895829506,
|
|
"learning_rate": 7.476589988449938e-07,
|
|
"loss": 0.0065,
|
|
"reward": 6.629714250564575,
|
|
"reward_std": 0.3594768207985908,
|
|
"rewards/concensus_correctness_reward_func": 2.107249964028597,
|
|
"rewards/consensus_reward_func": 1.9375,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.25,
|
|
"rewards/question_recreation_reward_func": 0.8115267266985029,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.390625,
|
|
"rewards/xmlcount_reward_func": 1.1328125,
|
|
"step": 72
|
|
},
|
|
{
|
|
"completion_length": 178.46875,
|
|
"epoch": 2.1142857142857143,
|
|
"grad_norm": 233.30178833007812,
|
|
"kl": 18.578114512876056,
|
|
"learning_rate": 7.334638543086203e-07,
|
|
"loss": 0.0186,
|
|
"reward": 6.874465793371201,
|
|
"reward_std": 0.40307298954576254,
|
|
"rewards/concensus_correctness_reward_func": 2.2929999753832817,
|
|
"rewards/consensus_reward_func": 1.875,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.3125,
|
|
"rewards/question_recreation_reward_func": 0.8509970791637897,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.390625,
|
|
"rewards/xmlcount_reward_func": 1.15234375,
|
|
"step": 74
|
|
},
|
|
{
|
|
"completion_length": 132.71875,
|
|
"epoch": 2.1714285714285713,
|
|
"grad_norm": 33.67573928833008,
|
|
"kl": 12.638730006660069,
|
|
"learning_rate": 7.190238384283412e-07,
|
|
"loss": 0.0126,
|
|
"reward": 6.277231350541115,
|
|
"reward_std": 0.0966361474711448,
|
|
"rewards/concensus_correctness_reward_func": 1.8519999831914902,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.8510125828906894,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.421875,
|
|
"rewards/xmlcount_reward_func": 1.15234375,
|
|
"step": 76
|
|
},
|
|
{
|
|
"completion_length": 119.125,
|
|
"epoch": 2.2285714285714286,
|
|
"grad_norm": 71.8940658569336,
|
|
"kl": 13.95311574424943,
|
|
"learning_rate": 7.043540967867781e-07,
|
|
"loss": 0.014,
|
|
"reward": 6.613084226846695,
|
|
"reward_std": 0.4508566010626964,
|
|
"rewards/concensus_correctness_reward_func": 2.167749986052513,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.1875,
|
|
"rewards/question_recreation_reward_func": 0.7734592199558392,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.375,
|
|
"rewards/xmlcount_reward_func": 1.109375,
|
|
"step": 78
|
|
},
|
|
{
|
|
"completion_length": 175.90625,
|
|
"epoch": 2.2857142857142856,
|
|
"grad_norm": 77835.71875,
|
|
"kl": 3964.1012081073877,
|
|
"learning_rate": 6.894700159171534e-07,
|
|
"loss": 3.9641,
|
|
"reward": 5.97839692234993,
|
|
"reward_std": 0.07155295684788143,
|
|
"rewards/concensus_correctness_reward_func": 1.918999969959259,
|
|
"rewards/consensus_reward_func": 1.875,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.7429906615288928,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.328125,
|
|
"rewards/xmlcount_reward_func": 1.11328125,
|
|
"step": 80
|
|
},
|
|
{
|
|
"completion_length": 179.65625,
|
|
"epoch": 2.342857142857143,
|
|
"grad_norm": 125543.6640625,
|
|
"kl": 9835.70870141859,
|
|
"learning_rate": 6.743872071649411e-07,
|
|
"loss": 9.8357,
|
|
"reward": 6.020951867103577,
|
|
"reward_std": 0.06402797414375527,
|
|
"rewards/concensus_correctness_reward_func": 1.9197499826550484,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.7222956451587379,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.328125,
|
|
"rewards/xmlcount_reward_func": 1.05078125,
|
|
"step": 82
|
|
},
|
|
{
|
|
"completion_length": 232.0,
|
|
"epoch": 2.4,
|
|
"grad_norm": 1412.854736328125,
|
|
"kl": 446.583844974978,
|
|
"learning_rate": 6.59121490313722e-07,
|
|
"loss": 0.4466,
|
|
"reward": 6.281670957803726,
|
|
"reward_std": 0.18979688500985503,
|
|
"rewards/concensus_correctness_reward_func": 2.004124980419874,
|
|
"rewards/consensus_reward_func": 1.875,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.25,
|
|
"rewards/question_recreation_reward_func": 0.749139740364626,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.34375,
|
|
"rewards/xmlcount_reward_func": 1.0596562512218952,
|
|
"step": 84
|
|
},
|
|
{
|
|
"completion_length": 118.21875,
|
|
"epoch": 2.4571428571428573,
|
|
"grad_norm": 61.02229309082031,
|
|
"kl": 31.27182410069142,
|
|
"learning_rate": 6.436888769924141e-07,
|
|
"loss": 0.0313,
|
|
"reward": 6.057249546051025,
|
|
"reward_std": 0.34927656571380794,
|
|
"rewards/concensus_correctness_reward_func": 1.95499999076128,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.7006870835321024,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.34375,
|
|
"rewards/xmlcount_reward_func": 1.0578125044703484,
|
|
"step": 86
|
|
},
|
|
{
|
|
"completion_length": 118.75,
|
|
"epoch": 2.5142857142857142,
|
|
"grad_norm": 14.912981986999512,
|
|
"kl": 3.7455607797385255,
|
|
"learning_rate": 6.281055538812861e-07,
|
|
"loss": 0.0037,
|
|
"reward": 6.8723739087581635,
|
|
"reward_std": 0.13205374777317047,
|
|
"rewards/concensus_correctness_reward_func": 1.8407499808818102,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.375,
|
|
"rewards/question_recreation_reward_func": 0.9721551463007927,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.46875,
|
|
"rewards/xmlcount_reward_func": 1.215718749910593,
|
|
"step": 88
|
|
},
|
|
{
|
|
"completion_length": 153.84375,
|
|
"epoch": 2.571428571428571,
|
|
"grad_norm": 0.0025314288213849068,
|
|
"kl": 8.448847692096706,
|
|
"learning_rate": 6.123878657343647e-07,
|
|
"loss": 0.0084,
|
|
"reward": 6.656649470329285,
|
|
"reward_std": 0.08741115219891071,
|
|
"rewards/concensus_correctness_reward_func": 2.0427499935030937,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.8717119987122715,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.421875,
|
|
"rewards/xmlcount_reward_func": 1.1953125,
|
|
"step": 90
|
|
},
|
|
{
|
|
"completion_length": 251.75,
|
|
"epoch": 2.6285714285714286,
|
|
"grad_norm": 118.81327056884766,
|
|
"kl": 13.05496934192388,
|
|
"learning_rate": 5.96552298236044e-07,
|
|
"loss": 0.0131,
|
|
"reward": 6.694221943616867,
|
|
"reward_std": 0.4122789396205917,
|
|
"rewards/concensus_correctness_reward_func": 2.2352499924600124,
|
|
"rewards/consensus_reward_func": 1.9375,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.375,
|
|
"rewards/question_recreation_reward_func": 0.7167846110532992,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.34375,
|
|
"rewards/xmlcount_reward_func": 1.0859375,
|
|
"step": 92
|
|
},
|
|
{
|
|
"completion_length": 163.65625,
|
|
"epoch": 2.685714285714286,
|
|
"grad_norm": 87.89823913574219,
|
|
"kl": 2.4992747763023715,
|
|
"learning_rate": 5.806154607098799e-07,
|
|
"loss": 0.0025,
|
|
"reward": 6.533147543668747,
|
|
"reward_std": 0.003381799964699894,
|
|
"rewards/concensus_correctness_reward_func": 1.9176249727606773,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.9358350443653762,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.46875,
|
|
"rewards/xmlcount_reward_func": 1.2109375,
|
|
"step": 94
|
|
},
|
|
{
|
|
"completion_length": 153.03125,
|
|
"epoch": 2.742857142857143,
|
|
"grad_norm": 400.9477844238281,
|
|
"kl": 38.25428117752017,
|
|
"learning_rate": 5.645940686977032e-07,
|
|
"loss": 0.0383,
|
|
"reward": 6.829828351736069,
|
|
"reward_std": 0.3507880922406912,
|
|
"rewards/concensus_correctness_reward_func": 2.167749986052513,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.25,
|
|
"rewards/question_recreation_reward_func": 0.8925471268594265,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.375,
|
|
"rewards/xmlcount_reward_func": 1.14453125,
|
|
"step": 96
|
|
},
|
|
{
|
|
"completion_length": 144.6875,
|
|
"epoch": 2.8,
|
|
"grad_norm": 55.653812408447266,
|
|
"kl": 107186.28675935966,
|
|
"learning_rate": 5.485049264273241e-07,
|
|
"loss": 107.1863,
|
|
"reward": 6.462806463241577,
|
|
"reward_std": 0.14959648743388243,
|
|
"rewards/concensus_correctness_reward_func": 1.9177499786019325,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.9200564883649349,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.4375,
|
|
"rewards/xmlcount_reward_func": 1.1875,
|
|
"step": 98
|
|
},
|
|
{
|
|
"completion_length": 173.96875,
|
|
"epoch": 2.857142857142857,
|
|
"grad_norm": 77.20722198486328,
|
|
"kl": 11.116460184275184,
|
|
"learning_rate": 5.323649091872178e-07,
|
|
"loss": 0.0111,
|
|
"reward": 6.238676607608795,
|
|
"reward_std": 0.16680637560784817,
|
|
"rewards/concensus_correctness_reward_func": 1.9146249815821648,
|
|
"rewards/consensus_reward_func": 1.875,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.88452033046633,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.421875,
|
|
"rewards/xmlcount_reward_func": 1.1426562517881393,
|
|
"step": 100
|
|
},
|
|
{
|
|
"completion_length": 127.15625,
|
|
"epoch": 2.914285714285714,
|
|
"grad_norm": 18.17213249206543,
|
|
"kl": 9.415469576871885,
|
|
"learning_rate": 5.16190945626678e-07,
|
|
"loss": 0.0094,
|
|
"reward": 6.7656005918979645,
|
|
"reward_std": 0.1332016959786415,
|
|
"rewards/concensus_correctness_reward_func": 2.0424999743700027,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.9379443395882845,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.453125,
|
|
"rewards/xmlcount_reward_func": 1.20703125,
|
|
"step": 102
|
|
},
|
|
{
|
|
"completion_length": 110.46875,
|
|
"epoch": 2.9714285714285715,
|
|
"grad_norm": 22.680030822753906,
|
|
"kl": 16.711946348986203,
|
|
"learning_rate": 5e-07,
|
|
"loss": 0.0167,
|
|
"reward": 6.9859276711940765,
|
|
"reward_std": 0.1056629114318639,
|
|
"rewards/concensus_correctness_reward_func": 2.169499985873699,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.25,
|
|
"rewards/question_recreation_reward_func": 0.9140838906168938,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.453125,
|
|
"rewards/xmlcount_reward_func": 1.19921875,
|
|
"step": 104
|
|
},
|
|
{
|
|
"completion_length": 137.53125,
|
|
"epoch": 3.0285714285714285,
|
|
"grad_norm": 36.1477165222168,
|
|
"kl": 13.781904241098744,
|
|
"learning_rate": 4.838090543733221e-07,
|
|
"loss": 0.0138,
|
|
"reward": 6.522283971309662,
|
|
"reward_std": 0.15269813538179733,
|
|
"rewards/concensus_correctness_reward_func": 2.0437499657273293,
|
|
"rewards/consensus_reward_func": 1.875,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.8847840757807717,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.40625,
|
|
"rewards/xmlcount_reward_func": 1.1875,
|
|
"step": 106
|
|
},
|
|
{
|
|
"completion_length": 124.65625,
|
|
"epoch": 3.085714285714286,
|
|
"grad_norm": 0.24235893785953522,
|
|
"kl": 146.02455591049488,
|
|
"learning_rate": 4.676350908127821e-07,
|
|
"loss": 0.146,
|
|
"reward": 6.913030564785004,
|
|
"reward_std": 0.17469462309964,
|
|
"rewards/concensus_correctness_reward_func": 2.167249970138073,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.25,
|
|
"rewards/question_recreation_reward_func": 0.886405548080802,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.4375,
|
|
"rewards/xmlcount_reward_func": 1.171875,
|
|
"step": 108
|
|
},
|
|
{
|
|
"completion_length": 124.09375,
|
|
"epoch": 3.142857142857143,
|
|
"grad_norm": 14.452537536621094,
|
|
"kl": 9.019021915784492,
|
|
"learning_rate": 4.5149507357267597e-07,
|
|
"loss": 0.009,
|
|
"reward": 6.673075348138809,
|
|
"reward_std": 0.23445336427539587,
|
|
"rewards/concensus_correctness_reward_func": 2.043874979019165,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.8596691160928458,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.453125,
|
|
"rewards/xmlcount_reward_func": 1.19140625,
|
|
"step": 110
|
|
},
|
|
{
|
|
"completion_length": 142.78125,
|
|
"epoch": 3.2,
|
|
"grad_norm": 23.269039154052734,
|
|
"kl": 102600.93675793686,
|
|
"learning_rate": 4.354059313022969e-07,
|
|
"loss": 102.6009,
|
|
"reward": 6.444055080413818,
|
|
"reward_std": 0.13274624338373542,
|
|
"rewards/concensus_correctness_reward_func": 1.918249972164631,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.9086175980046391,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.4375,
|
|
"rewards/xmlcount_reward_func": 1.1796875,
|
|
"step": 112
|
|
},
|
|
{
|
|
"completion_length": 156.375,
|
|
"epoch": 3.257142857142857,
|
|
"grad_norm": 0.0024780540261417627,
|
|
"kl": 10458499.638899168,
|
|
"learning_rate": 4.193845392901201e-07,
|
|
"loss": 10458.5,
|
|
"reward": 6.469187945127487,
|
|
"reward_std": 0.13797323405742645,
|
|
"rewards/concensus_correctness_reward_func": 1.9172499775886536,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.9074067602632567,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.453125,
|
|
"rewards/xmlcount_reward_func": 1.19140625,
|
|
"step": 114
|
|
},
|
|
{
|
|
"completion_length": 120.1875,
|
|
"epoch": 3.314285714285714,
|
|
"grad_norm": 3.623257875442505,
|
|
"kl": 4.9637036175093385,
|
|
"learning_rate": 4.0344770176395606e-07,
|
|
"loss": 0.005,
|
|
"reward": 6.929087281227112,
|
|
"reward_std": 0.05713449977338314,
|
|
"rewards/concensus_correctness_reward_func": 2.166124999523163,
|
|
"rewards/consensus_reward_func": 1.875,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.25,
|
|
"rewards/question_recreation_reward_func": 0.9309310428798199,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.46875,
|
|
"rewards/xmlcount_reward_func": 1.23828125,
|
|
"step": 116
|
|
},
|
|
{
|
|
"completion_length": 164.125,
|
|
"epoch": 3.3714285714285714,
|
|
"grad_norm": 57468.05859375,
|
|
"kl": 7235.024369265923,
|
|
"learning_rate": 3.8761213426563543e-07,
|
|
"loss": 7.235,
|
|
"reward": 6.663719207048416,
|
|
"reward_std": 0.16646834812127054,
|
|
"rewards/concensus_correctness_reward_func": 2.0431249737739563,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.8979379505617544,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.421875,
|
|
"rewards/xmlcount_reward_func": 1.17578125,
|
|
"step": 118
|
|
},
|
|
{
|
|
"completion_length": 118.5,
|
|
"epoch": 3.4285714285714284,
|
|
"grad_norm": 520.0580444335938,
|
|
"kl": 67.83392252838627,
|
|
"learning_rate": 3.718944461187138e-07,
|
|
"loss": 0.0678,
|
|
"reward": 6.547982841730118,
|
|
"reward_std": 0.14045805705245584,
|
|
"rewards/concensus_correctness_reward_func": 1.9401249885559082,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.8695765999145806,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.4375,
|
|
"rewards/xmlcount_reward_func": 1.17578125,
|
|
"step": 120
|
|
},
|
|
{
|
|
"completion_length": 172.46875,
|
|
"epoch": 3.4857142857142858,
|
|
"grad_norm": 136.3828887939453,
|
|
"kl": 66.3491708847132,
|
|
"learning_rate": 3.563111230075859e-07,
|
|
"loss": 0.0664,
|
|
"reward": 6.197802782058716,
|
|
"reward_std": 0.08567016178858466,
|
|
"rewards/concensus_correctness_reward_func": 1.8532499894499779,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.8328340587904677,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.390625,
|
|
"rewards/xmlcount_reward_func": 1.12109375,
|
|
"step": 122
|
|
},
|
|
{
|
|
"completion_length": 165.90625,
|
|
"epoch": 3.5428571428571427,
|
|
"grad_norm": 0.05435093119740486,
|
|
"kl": 150.12784510711583,
|
|
"learning_rate": 3.408785096862782e-07,
|
|
"loss": 0.1501,
|
|
"reward": 6.551819205284119,
|
|
"reward_std": 0.015193770406767726,
|
|
"rewards/concensus_correctness_reward_func": 2.0423749908804893,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.8338817218318582,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.40625,
|
|
"rewards/xmlcount_reward_func": 1.1443125009536743,
|
|
"step": 124
|
|
},
|
|
{
|
|
"completion_length": 119.78125,
|
|
"epoch": 3.6,
|
|
"grad_norm": 0.8537346124649048,
|
|
"kl": 0.3934392740372914,
|
|
"learning_rate": 3.2561279283505884e-07,
|
|
"loss": 0.0004,
|
|
"reward": 7.02690127491951,
|
|
"reward_std": 0.026761331711895764,
|
|
"rewards/concensus_correctness_reward_func": 2.0481249764561653,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.25,
|
|
"rewards/question_recreation_reward_func": 0.9951512552797794,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.484375,
|
|
"rewards/xmlcount_reward_func": 1.2492499947547913,
|
|
"step": 126
|
|
},
|
|
{
|
|
"completion_length": 151.15625,
|
|
"epoch": 3.657142857142857,
|
|
"grad_norm": 0.0007187644951045513,
|
|
"kl": 35.26266118535864,
|
|
"learning_rate": 3.105299840828466e-07,
|
|
"loss": 0.0353,
|
|
"reward": 6.682261526584625,
|
|
"reward_std": 0.03159405663609505,
|
|
"rewards/concensus_correctness_reward_func": 2.044749967753887,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.8836052501574159,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.4375,
|
|
"rewards/xmlcount_reward_func": 1.19140625,
|
|
"step": 128
|
|
},
|
|
{
|
|
"completion_length": 175.4375,
|
|
"epoch": 3.7142857142857144,
|
|
"grad_norm": 4472.33447265625,
|
|
"kl": 407.4183887737163,
|
|
"learning_rate": 2.95645903213222e-07,
|
|
"loss": 0.4074,
|
|
"reward": 6.523717999458313,
|
|
"reward_std": 0.23186340555548668,
|
|
"rewards/concensus_correctness_reward_func": 1.9444999862462282,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.8487492590211332,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.4375,
|
|
"rewards/xmlcount_reward_func": 1.16796875,
|
|
"step": 130
|
|
},
|
|
{
|
|
"completion_length": 119.59375,
|
|
"epoch": 3.7714285714285714,
|
|
"grad_norm": 14.829834938049316,
|
|
"kl": 43.95136128488707,
|
|
"learning_rate": 2.8097616157165885e-07,
|
|
"loss": 0.044,
|
|
"reward": 6.873107701539993,
|
|
"reward_std": 0.1839166151767131,
|
|
"rewards/concensus_correctness_reward_func": 2.2936249896883965,
|
|
"rewards/consensus_reward_func": 1.875,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.375,
|
|
"rewards/question_recreation_reward_func": 0.832482704310678,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.375,
|
|
"rewards/xmlcount_reward_func": 1.1220000013709068,
|
|
"step": 132
|
|
},
|
|
{
|
|
"completion_length": 132.0625,
|
|
"epoch": 3.8285714285714287,
|
|
"grad_norm": 10.728759765625,
|
|
"kl": 14.55994114780924,
|
|
"learning_rate": 2.665361456913797e-07,
|
|
"loss": 0.0146,
|
|
"reward": 6.96189820766449,
|
|
"reward_std": 0.15106727764941752,
|
|
"rewards/concensus_correctness_reward_func": 2.204374983906746,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.375,
|
|
"rewards/question_recreation_reward_func": 0.8239294798113406,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.390625,
|
|
"rewards/xmlcount_reward_func": 1.16796875,
|
|
"step": 134
|
|
},
|
|
{
|
|
"completion_length": 167.0625,
|
|
"epoch": 3.8857142857142857,
|
|
"grad_norm": 38.03074264526367,
|
|
"kl": 7.051730461721945,
|
|
"learning_rate": 2.523410011550064e-07,
|
|
"loss": 0.0071,
|
|
"reward": 6.516344875097275,
|
|
"reward_std": 0.12807103618979454,
|
|
"rewards/concensus_correctness_reward_func": 1.9381249845027924,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.8555636611999944,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.421875,
|
|
"rewards/xmlcount_reward_func": 1.17578125,
|
|
"step": 136
|
|
},
|
|
{
|
|
"completion_length": 118.5625,
|
|
"epoch": 3.942857142857143,
|
|
"grad_norm": 28.598356246948242,
|
|
"kl": 17.839532846181896,
|
|
"learning_rate": 2.3840561670893495e-07,
|
|
"loss": 0.0178,
|
|
"reward": 6.337287962436676,
|
|
"reward_std": 0.29263460593210766,
|
|
"rewards/concensus_correctness_reward_func": 1.9114999696612358,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.8437567495275289,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.40625,
|
|
"rewards/xmlcount_reward_func": 1.17578125,
|
|
"step": 138
|
|
},
|
|
{
|
|
"completion_length": 134.5625,
|
|
"epoch": 4.0,
|
|
"grad_norm": 62.3493537902832,
|
|
"kl": 14.68970625225824,
|
|
"learning_rate": 2.247446086470982e-07,
|
|
"loss": 0.0147,
|
|
"reward": 6.461593389511108,
|
|
"reward_std": 0.2324538507964462,
|
|
"rewards/concensus_correctness_reward_func": 2.0439999774098396,
|
|
"rewards/consensus_reward_func": 1.875,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0625,
|
|
"rewards/question_recreation_reward_func": 0.8629059121012688,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.4375,
|
|
"rewards/xmlcount_reward_func": 1.1796875,
|
|
"step": 140
|
|
},
|
|
{
|
|
"completion_length": 108.59375,
|
|
"epoch": 4.057142857142857,
|
|
"grad_norm": 23.657360076904297,
|
|
"kl": 0.6967418518526642,
|
|
"learning_rate": 2.113723054804904e-07,
|
|
"loss": 0.0007,
|
|
"reward": 6.604361176490784,
|
|
"reward_std": 0.08363489806652069,
|
|
"rewards/concensus_correctness_reward_func": 1.9134999737143517,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.9760174416005611,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.484375,
|
|
"rewards/xmlcount_reward_func": 1.23046875,
|
|
"step": 142
|
|
},
|
|
{
|
|
"completion_length": 168.4375,
|
|
"epoch": 4.114285714285714,
|
|
"grad_norm": 16.927154541015625,
|
|
"kl": 3.9155555481934243,
|
|
"learning_rate": 1.9830273290853766e-07,
|
|
"loss": 0.0039,
|
|
"reward": 6.76575893163681,
|
|
"reward_std": 0.10381055303150788,
|
|
"rewards/concensus_correctness_reward_func": 2.1677499637007713,
|
|
"rewards/consensus_reward_func": 1.875,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.25,
|
|
"rewards/question_recreation_reward_func": 0.8870714246877469,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.421875,
|
|
"rewards/xmlcount_reward_func": 1.1640625,
|
|
"step": 144
|
|
},
|
|
{
|
|
"completion_length": 207.34375,
|
|
"epoch": 4.171428571428572,
|
|
"grad_norm": 230.26246643066406,
|
|
"kl": 35.77139102048136,
|
|
"learning_rate": 1.8554959910807772e-07,
|
|
"loss": 0.0358,
|
|
"reward": 6.17943412065506,
|
|
"reward_std": 0.5001923546660691,
|
|
"rewards/concensus_correctness_reward_func": 1.8548749722540379,
|
|
"rewards/consensus_reward_func": 1.9375,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.8440904202871025,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.40625,
|
|
"rewards/xmlcount_reward_func": 1.13671875,
|
|
"step": 146
|
|
},
|
|
{
|
|
"completion_length": 179.96875,
|
|
"epoch": 4.228571428571429,
|
|
"grad_norm": 608.2833862304688,
|
|
"kl": 41.165939685676896,
|
|
"learning_rate": 1.7312628035537386e-07,
|
|
"loss": 0.0412,
|
|
"reward": 6.023386672139168,
|
|
"reward_std": 0.5373760673683137,
|
|
"rewards/concensus_correctness_reward_func": 1.79399998113513,
|
|
"rewards/consensus_reward_func": 1.9375,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.8114491496235132,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.375,
|
|
"rewards/xmlcount_reward_func": 1.1054375022649765,
|
|
"step": 148
|
|
},
|
|
{
|
|
"completion_length": 119.25,
|
|
"epoch": 4.285714285714286,
|
|
"grad_norm": 156.11758422851562,
|
|
"kl": 14.033162109145678,
|
|
"learning_rate": 1.6104580699624837e-07,
|
|
"loss": 0.014,
|
|
"reward": 6.490307152271271,
|
|
"reward_std": 0.4207246375735849,
|
|
"rewards/concensus_correctness_reward_func": 1.9844999872148037,
|
|
"rewards/consensus_reward_func": 1.875,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0625,
|
|
"rewards/question_recreation_reward_func": 0.9474009126424789,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.421875,
|
|
"rewards/xmlcount_reward_func": 1.1990312486886978,
|
|
"step": 150
|
|
},
|
|
{
|
|
"completion_length": 166.75,
|
|
"epoch": 4.3428571428571425,
|
|
"grad_norm": 59.2918815612793,
|
|
"kl": 15.187666200303994,
|
|
"learning_rate": 1.493208497790504e-07,
|
|
"loss": 0.0152,
|
|
"reward": 6.442292809486389,
|
|
"reward_std": 0.08591883815824986,
|
|
"rewards/concensus_correctness_reward_func": 1.9396249949932098,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.834699070896022,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.40625,
|
|
"rewards/xmlcount_reward_func": 1.13671875,
|
|
"step": 152
|
|
},
|
|
{
|
|
"completion_length": 151.84375,
|
|
"epoch": 4.4,
|
|
"grad_norm": 236.9644775390625,
|
|
"kl": 35.47643838436477,
|
|
"learning_rate": 1.3796370656478934e-07,
|
|
"loss": 0.0355,
|
|
"reward": 6.461767286062241,
|
|
"reward_std": 0.4090730301104486,
|
|
"rewards/concensus_correctness_reward_func": 2.0437499657273293,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.8281734781339765,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.359375,
|
|
"rewards/xmlcount_reward_func": 1.10546875,
|
|
"step": 154
|
|
},
|
|
{
|
|
"completion_length": 141.0,
|
|
"epoch": 4.457142857142857,
|
|
"grad_norm": 6.475616931915283,
|
|
"kl": 607.9583089863813,
|
|
"learning_rate": 1.2698628942837697e-07,
|
|
"loss": 0.608,
|
|
"reward": 6.360057026147842,
|
|
"reward_std": 0.10659515461156843,
|
|
"rewards/concensus_correctness_reward_func": 1.9188749715685844,
|
|
"rewards/consensus_reward_func": 1.875,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.9255570247769356,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.4375,
|
|
"rewards/xmlcount_reward_func": 1.203125,
|
|
"step": 156
|
|
},
|
|
{
|
|
"completion_length": 152.3125,
|
|
"epoch": 4.514285714285714,
|
|
"grad_norm": 2432.385986328125,
|
|
"kl": 223.98980636267672,
|
|
"learning_rate": 1.1640011216450691e-07,
|
|
"loss": 0.224,
|
|
"reward": 6.682069569826126,
|
|
"reward_std": 0.14451245963573456,
|
|
"rewards/concensus_correctness_reward_func": 2.0427499935030937,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.8815071193967015,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.453125,
|
|
"rewards/xmlcount_reward_func": 1.1796875,
|
|
"step": 158
|
|
},
|
|
{
|
|
"completion_length": 170.28125,
|
|
"epoch": 4.571428571428571,
|
|
"grad_norm": 66.50875091552734,
|
|
"kl": 179.85483134612986,
|
|
"learning_rate": 1.0621627821127288e-07,
|
|
"loss": 0.1799,
|
|
"reward": 6.952862948179245,
|
|
"reward_std": 0.34136643912643194,
|
|
"rewards/concensus_correctness_reward_func": 2.2936249896883965,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.3125,
|
|
"rewards/question_recreation_reward_func": 0.7928941965801641,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.40625,
|
|
"rewards/xmlcount_reward_func": 1.1475937515497208,
|
|
"step": 160
|
|
},
|
|
{
|
|
"completion_length": 150.375,
|
|
"epoch": 4.628571428571428,
|
|
"grad_norm": 69454.9453125,
|
|
"kl": 3160.4802792127484,
|
|
"learning_rate": 9.644546900419531e-08,
|
|
"loss": 3.1605,
|
|
"reward": 7.1283751130104065,
|
|
"reward_std": 0.16841159365139902,
|
|
"rewards/concensus_correctness_reward_func": 2.2938749864697456,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.375,
|
|
"rewards/question_recreation_reward_func": 0.8977188635617495,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.421875,
|
|
"rewards/xmlcount_reward_func": 1.1399062499403954,
|
|
"step": 162
|
|
},
|
|
{
|
|
"completion_length": 141.03125,
|
|
"epoch": 4.685714285714286,
|
|
"grad_norm": 0.005350879859179258,
|
|
"kl": 40.605473204805264,
|
|
"learning_rate": 8.70979327728718e-08,
|
|
"loss": 0.0406,
|
|
"reward": 6.543893337249756,
|
|
"reward_std": 0.008683009073138237,
|
|
"rewards/concensus_correctness_reward_func": 1.9179999753832817,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.9462057938799262,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.46875,
|
|
"rewards/xmlcount_reward_func": 1.2109375,
|
|
"step": 164
|
|
},
|
|
{
|
|
"completion_length": 172.5,
|
|
"epoch": 4.742857142857143,
|
|
"grad_norm": 842.9741821289062,
|
|
"kl": 79.63627462068501,
|
|
"learning_rate": 7.81834737919978e-08,
|
|
"loss": 0.0796,
|
|
"reward": 6.854692578315735,
|
|
"reward_std": 0.287606913712807,
|
|
"rewards/concensus_correctness_reward_func": 2.2923749834299088,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.25,
|
|
"rewards/question_recreation_reward_func": 0.7849738348741084,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.390625,
|
|
"rewards/xmlcount_reward_func": 1.13671875,
|
|
"step": 166
|
|
},
|
|
{
|
|
"completion_length": 206.53125,
|
|
"epoch": 4.8,
|
|
"grad_norm": 32.29534912109375,
|
|
"kl": 25.739543075352145,
|
|
"learning_rate": 6.971144209803736e-08,
|
|
"loss": 0.0257,
|
|
"reward": 6.660571306943893,
|
|
"reward_std": 0.22189393499866128,
|
|
"rewards/concensus_correctness_reward_func": 2.1687499657273293,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.25,
|
|
"rewards/question_recreation_reward_func": 0.7823212522780523,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.375,
|
|
"rewards/xmlcount_reward_func": 1.0844999998807907,
|
|
"step": 168
|
|
},
|
|
{
|
|
"completion_length": 158.78125,
|
|
"epoch": 4.857142857142857,
|
|
"grad_norm": 48.24919891357422,
|
|
"kl": 16.90938147829354,
|
|
"learning_rate": 6.16907236823262e-08,
|
|
"loss": 0.0169,
|
|
"reward": 6.540567606687546,
|
|
"reward_std": 0.11463689978700131,
|
|
"rewards/concensus_correctness_reward_func": 2.076999992132187,
|
|
"rewards/consensus_reward_func": 1.875,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.25,
|
|
"rewards/question_recreation_reward_func": 0.8065051874145865,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.390625,
|
|
"rewards/xmlcount_reward_func": 1.1414375007152557,
|
|
"step": 170
|
|
},
|
|
{
|
|
"completion_length": 140.40625,
|
|
"epoch": 4.914285714285715,
|
|
"grad_norm": 1.2339164018630981,
|
|
"kl": 38.32704227172388,
|
|
"learning_rate": 5.412973117089287e-08,
|
|
"loss": 0.0383,
|
|
"reward": 6.7995350658893585,
|
|
"reward_std": 0.22941981069743633,
|
|
"rewards/concensus_correctness_reward_func": 1.9438749924302101,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.3125,
|
|
"rewards/question_recreation_reward_func": 0.8947226200252771,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.453125,
|
|
"rewards/xmlcount_reward_func": 1.1953125,
|
|
"step": 172
|
|
},
|
|
{
|
|
"completion_length": 204.125,
|
|
"epoch": 4.9714285714285715,
|
|
"grad_norm": 83.86946868896484,
|
|
"kl": 16.977694245022008,
|
|
"learning_rate": 4.703639500077655e-08,
|
|
"loss": 0.017,
|
|
"reward": 6.165009528398514,
|
|
"reward_std": 0.3757715789251961,
|
|
"rewards/concensus_correctness_reward_func": 1.818874979391694,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.8160095855710097,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.375,
|
|
"rewards/xmlcount_reward_func": 1.1551249995827675,
|
|
"step": 174
|
|
},
|
|
{
|
|
"completion_length": 248.21875,
|
|
"epoch": 5.0285714285714285,
|
|
"grad_norm": 78.45205688476562,
|
|
"kl": 123.29459129135614,
|
|
"learning_rate": 4.041815510209395e-08,
|
|
"loss": 0.1233,
|
|
"reward": 6.23151370882988,
|
|
"reward_std": 0.30146771227009594,
|
|
"rewards/concensus_correctness_reward_func": 1.9547499865293503,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.7259825635701418,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.34375,
|
|
"rewards/xmlcount_reward_func": 1.08203125,
|
|
"step": 176
|
|
},
|
|
{
|
|
"completion_length": 115.59375,
|
|
"epoch": 5.085714285714285,
|
|
"grad_norm": 30.307924270629883,
|
|
"kl": 5.359901059065578,
|
|
"learning_rate": 3.4281953094578875e-08,
|
|
"loss": 0.0054,
|
|
"reward": 6.928922921419144,
|
|
"reward_std": 0.0711733426360297,
|
|
"rewards/concensus_correctness_reward_func": 2.1689999848604202,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.25,
|
|
"rewards/question_recreation_reward_func": 0.892735461005941,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.421875,
|
|
"rewards/xmlcount_reward_func": 1.1953125,
|
|
"step": 178
|
|
},
|
|
{
|
|
"completion_length": 155.28125,
|
|
"epoch": 5.142857142857143,
|
|
"grad_norm": 733.8907470703125,
|
|
"kl": 27.44156739033167,
|
|
"learning_rate": 2.8634225006782864e-08,
|
|
"loss": 0.0274,
|
|
"reward": 6.603150278329849,
|
|
"reward_std": 0.13888305581349414,
|
|
"rewards/concensus_correctness_reward_func": 2.042874976992607,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.8571502412669361,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.40625,
|
|
"rewards/xmlcount_reward_func": 1.171875,
|
|
"step": 180
|
|
},
|
|
{
|
|
"completion_length": 210.46875,
|
|
"epoch": 5.2,
|
|
"grad_norm": 193.40077209472656,
|
|
"kl": 32.62125267219369,
|
|
"learning_rate": 2.348089452556956e-08,
|
|
"loss": 0.0326,
|
|
"reward": 6.1919485330581665,
|
|
"reward_std": 0.3429363896648283,
|
|
"rewards/concensus_correctness_reward_func": 2.0476249754428864,
|
|
"rewards/consensus_reward_func": 1.875,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.7224485471379012,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.328125,
|
|
"rewards/xmlcount_reward_func": 1.09375,
|
|
"step": 182
|
|
},
|
|
{
|
|
"completion_length": 178.65625,
|
|
"epoch": 5.257142857142857,
|
|
"grad_norm": 195.80307006835938,
|
|
"kl": 32.1218818299501,
|
|
"learning_rate": 1.882736678298491e-08,
|
|
"loss": 0.0321,
|
|
"reward": 6.4709332883358,
|
|
"reward_std": 0.20533737423829734,
|
|
"rewards/concensus_correctness_reward_func": 2.041749984025955,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.8159645728301257,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.390625,
|
|
"rewards/xmlcount_reward_func": 1.0975937489420176,
|
|
"step": 184
|
|
},
|
|
{
|
|
"completion_length": 202.9375,
|
|
"epoch": 5.314285714285714,
|
|
"grad_norm": 6.5529704093933105,
|
|
"kl": 21.807212316646257,
|
|
"learning_rate": 1.4678522687020412e-08,
|
|
"loss": 0.0218,
|
|
"reward": 6.331148684024811,
|
|
"reward_std": 0.04444863580283709,
|
|
"rewards/concensus_correctness_reward_func": 1.9174999818205833,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.8277112208306789,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.421875,
|
|
"rewards/xmlcount_reward_func": 1.1640625,
|
|
"step": 186
|
|
},
|
|
{
|
|
"completion_length": 183.84375,
|
|
"epoch": 5.371428571428572,
|
|
"grad_norm": 14.336308479309082,
|
|
"kl": 11.637659553496746,
|
|
"learning_rate": 1.1038713802214717e-08,
|
|
"loss": 0.0116,
|
|
"reward": 6.26752108335495,
|
|
"reward_std": 0.21460022201063111,
|
|
"rewards/concensus_correctness_reward_func": 2.1094999872148037,
|
|
"rewards/consensus_reward_func": 1.8125,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.7791148920659907,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.34375,
|
|
"rewards/xmlcount_reward_func": 1.09765625,
|
|
"step": 188
|
|
},
|
|
{
|
|
"completion_length": 143.65625,
|
|
"epoch": 5.428571428571429,
|
|
"grad_norm": 35.581268310546875,
|
|
"kl": 22.53030281640372,
|
|
"learning_rate": 7.91175778546288e-09,
|
|
"loss": 0.0225,
|
|
"reward": 6.6998831778764725,
|
|
"reward_std": 0.5453806091099977,
|
|
"rewards/concensus_correctness_reward_func": 2.2333749793469906,
|
|
"rewards/consensus_reward_func": 1.875,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.3125,
|
|
"rewards/question_recreation_reward_func": 0.7986019123345613,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.375,
|
|
"rewards/xmlcount_reward_func": 1.1054062508046627,
|
|
"step": 190
|
|
},
|
|
{
|
|
"completion_length": 126.28125,
|
|
"epoch": 5.485714285714286,
|
|
"grad_norm": 1261.33740234375,
|
|
"kl": 216.40358931079027,
|
|
"learning_rate": 5.3009343818219975e-09,
|
|
"loss": 0.2164,
|
|
"reward": 6.315451234579086,
|
|
"reward_std": 0.19842870347201824,
|
|
"rewards/concensus_correctness_reward_func": 1.851999968290329,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.0,
|
|
"rewards/question_recreation_reward_func": 0.8775137448683381,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.40625,
|
|
"rewards/xmlcount_reward_func": 1.1796875,
|
|
"step": 192
|
|
},
|
|
{
|
|
"completion_length": 113.5625,
|
|
"epoch": 5.542857142857143,
|
|
"grad_norm": 38.23686981201172,
|
|
"kl": 26.032929529420585,
|
|
"learning_rate": 3.2089819845111944e-09,
|
|
"loss": 0.026,
|
|
"reward": 6.602267116308212,
|
|
"reward_std": 0.1286412663757801,
|
|
"rewards/concensus_correctness_reward_func": 2.169375002384186,
|
|
"rewards/consensus_reward_func": 1.875,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.8703921169508249,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.390625,
|
|
"rewards/xmlcount_reward_func": 1.171875,
|
|
"step": 194
|
|
},
|
|
{
|
|
"completion_length": 151.3125,
|
|
"epoch": 5.6,
|
|
"grad_norm": 0.003121931105852127,
|
|
"kl": 1.4879173506217853,
|
|
"learning_rate": 1.638094762715314e-09,
|
|
"loss": 0.0015,
|
|
"reward": 6.882100820541382,
|
|
"reward_std": 0.09168865531682968,
|
|
"rewards/concensus_correctness_reward_func": 1.9496249873191118,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.25,
|
|
"rewards/question_recreation_reward_func": 0.9676321819424629,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.484375,
|
|
"rewards/xmlcount_reward_func": 1.23046875,
|
|
"step": 196
|
|
},
|
|
{
|
|
"completion_length": 127.21875,
|
|
"epoch": 5.6571428571428575,
|
|
"grad_norm": 0.0015910969814285636,
|
|
"kl": 1.242647759863985,
|
|
"learning_rate": 5.899203602046654e-10,
|
|
"loss": 0.0012,
|
|
"reward": 6.5998228788375854,
|
|
"reward_std": 0.05409337143646553,
|
|
"rewards/concensus_correctness_reward_func": 1.8141249939799309,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.9653854127973318,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.46875,
|
|
"rewards/xmlcount_reward_func": 1.2265625,
|
|
"step": 198
|
|
},
|
|
{
|
|
"completion_length": 109.0625,
|
|
"epoch": 5.714285714285714,
|
|
"grad_norm": 0.0088896993547678,
|
|
"kl": 10.252777249065858,
|
|
"learning_rate": 6.555816718389895e-11,
|
|
"loss": 0.0103,
|
|
"reward": 6.735629111528397,
|
|
"reward_std": 0.09949736995622516,
|
|
"rewards/concensus_correctness_reward_func": 2.0439999774098396,
|
|
"rewards/consensus_reward_func": 2.0,
|
|
"rewards/cumulative_reward_2": 0.0,
|
|
"rewards/final_correctness_reward_func": 0.125,
|
|
"rewards/question_recreation_reward_func": 0.9142853366211057,
|
|
"rewards/soft_format_reward_func": 0.0,
|
|
"rewards/strict_format_reward_func": 0.453125,
|
|
"rewards/xmlcount_reward_func": 1.19921875,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 5.714285714285714,
|
|
"step": 200,
|
|
"total_flos": 0.0,
|
|
"train_loss": 149.93084794194365,
|
|
"train_runtime": 1724.5009,
|
|
"train_samples_per_second": 1.856,
|
|
"train_steps_per_second": 0.116
|
|
}
|
|
],
|
|
"logging_steps": 2,
|
|
"max_steps": 200,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 6,
|
|
"save_steps": 25,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 2,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|