2183 lines
83 KiB
JSON
2183 lines
83 KiB
JSON
|
{
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 5.7006369426751595,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 450,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"completion_length": 247.125,
|
||
|
|
"epoch": 0.050955414012738856,
|
||
|
|
"grad_norm": 5.409891128540039,
|
||
|
|
"kl": 2.317987572401762,
|
||
|
|
"learning_rate": 8.57142857142857e-08,
|
||
|
|
"loss": 0.0023,
|
||
|
|
"reward": 17.20423835515976,
|
||
|
|
"reward_std": 1.1507948003709316,
|
||
|
|
"rewards/concensus_correctness_reward_func": 12.706375002861023,
|
||
|
|
"rewards/consensus_reward_func": 1.25,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.25,
|
||
|
|
"rewards/question_recreation_reward_func": 0.7556760907173157,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.21875,
|
||
|
|
"rewards/xmlcount_reward_func": 1.023437537252903,
|
||
|
|
"step": 4
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 228.625,
|
||
|
|
"epoch": 0.10191082802547771,
|
||
|
|
"grad_norm": 4.3909173011779785,
|
||
|
|
"kl": 1.6623684950172901,
|
||
|
|
"learning_rate": 1.714285714285714e-07,
|
||
|
|
"loss": 0.0017,
|
||
|
|
"reward": 12.55974566936493,
|
||
|
|
"reward_std": 6.854554038494825,
|
||
|
|
"rewards/concensus_correctness_reward_func": 8.75,
|
||
|
|
"rewards/consensus_reward_func": 1.0,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.875,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5599956843070686,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1247500106692314,
|
||
|
|
"step": 8
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 194.375,
|
||
|
|
"epoch": 0.15286624203821655,
|
||
|
|
"grad_norm": 2.985093832015991,
|
||
|
|
"kl": 6.860304161906242,
|
||
|
|
"learning_rate": 2.571428571428571e-07,
|
||
|
|
"loss": 0.0069,
|
||
|
|
"reward": 16.054461300373077,
|
||
|
|
"reward_std": 9.667749413754791,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.670499965548515,
|
||
|
|
"rewards/consensus_reward_func": 1.125,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.125,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6383362491615117,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.28125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2143750190734863,
|
||
|
|
"step": 12
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 269.75,
|
||
|
|
"epoch": 0.20382165605095542,
|
||
|
|
"grad_norm": 3.6396210193634033,
|
||
|
|
"kl": 5.6482887379825115,
|
||
|
|
"learning_rate": 2.99984424553496e-07,
|
||
|
|
"loss": 0.0056,
|
||
|
|
"reward": 12.981057226657867,
|
||
|
|
"reward_std": 6.896631011739373,
|
||
|
|
"rewards/concensus_correctness_reward_func": 9.086749993264675,
|
||
|
|
"rewards/consensus_reward_func": 1.125,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.125,
|
||
|
|
"rewards/question_recreation_reward_func": 0.597869697958231,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.15625,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8901875242590904,
|
||
|
|
"step": 16
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 213.6875,
|
||
|
|
"epoch": 0.25477707006369427,
|
||
|
|
"grad_norm": 2.7366411685943604,
|
||
|
|
"kl": 8.909522596746683,
|
||
|
|
"learning_rate": 2.998598403883547e-07,
|
||
|
|
"loss": 0.0089,
|
||
|
|
"reward": 11.70977608859539,
|
||
|
|
"reward_std": 4.304964765789919,
|
||
|
|
"rewards/concensus_correctness_reward_func": 7.7207499742507935,
|
||
|
|
"rewards/consensus_reward_func": 1.0,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.0,
|
||
|
|
"rewards/question_recreation_reward_func": 0.46571373101323843,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.34375,
|
||
|
|
"rewards/xmlcount_reward_func": 1.179562509059906,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 250.625,
|
||
|
|
"epoch": 0.3057324840764331,
|
||
|
|
"grad_norm": 4.964949607849121,
|
||
|
|
"kl": 5.63778755068779,
|
||
|
|
"learning_rate": 2.9961077554357913e-07,
|
||
|
|
"loss": 0.0056,
|
||
|
|
"reward": 10.035876393318176,
|
||
|
|
"reward_std": 6.212635695934296,
|
||
|
|
"rewards/concensus_correctness_reward_func": 6.3186250030994415,
|
||
|
|
"rewards/consensus_reward_func": 1.125,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.625,
|
||
|
|
"rewards/question_recreation_reward_func": 0.7127514779567719,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.1875,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0670000314712524,
|
||
|
|
"step": 24
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 279.0,
|
||
|
|
"epoch": 0.35668789808917195,
|
||
|
|
"grad_norm": 2.4961581230163574,
|
||
|
|
"kl": 67.01326366513968,
|
||
|
|
"learning_rate": 2.9923743690422306e-07,
|
||
|
|
"loss": 0.067,
|
||
|
|
"reward": 16.109780073165894,
|
||
|
|
"reward_std": 5.918968662619591,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.880625024437904,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.875,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6405299380421638,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.21875,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1198750287294388,
|
||
|
|
"step": 28
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 273.6875,
|
||
|
|
"epoch": 0.40764331210191085,
|
||
|
|
"grad_norm": 5.211795330047607,
|
||
|
|
"kl": 5.373722583055496,
|
||
|
|
"learning_rate": 2.9874013458303877e-07,
|
||
|
|
"loss": 0.0054,
|
||
|
|
"reward": 17.488071978092194,
|
||
|
|
"reward_std": 4.414429775904864,
|
||
|
|
"rewards/concensus_correctness_reward_func": 12.954499989748001,
|
||
|
|
"rewards/consensus_reward_func": 1.25,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.375,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5903223338536918,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.15625,
|
||
|
|
"rewards/xmlcount_reward_func": 1.162000022828579,
|
||
|
|
"step": 32
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 241.6875,
|
||
|
|
"epoch": 0.4585987261146497,
|
||
|
|
"grad_norm": 1.7989726066589355,
|
||
|
|
"kl": 5.3555001355707645,
|
||
|
|
"learning_rate": 2.9811928166288275e-07,
|
||
|
|
"loss": 0.0053,
|
||
|
|
"reward": 15.544344544410706,
|
||
|
|
"reward_std": 2.0546061204513535,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.492375005036592,
|
||
|
|
"rewards/consensus_reward_func": 1.25,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.75,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5364694236777723,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.34375,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1717500165104866,
|
||
|
|
"step": 36
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 250.4375,
|
||
|
|
"epoch": 0.5095541401273885,
|
||
|
|
"grad_norm": 55.185237884521484,
|
||
|
|
"kl": 47.606556087732315,
|
||
|
|
"learning_rate": 2.973753938535892e-07,
|
||
|
|
"loss": 0.0476,
|
||
|
|
"reward": 12.839633375406265,
|
||
|
|
"reward_std": 2.927978876978159,
|
||
|
|
"rewards/concensus_correctness_reward_func": 8.834124997258186,
|
||
|
|
"rewards/consensus_reward_func": 1.5,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.5,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6650712452828884,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.1875,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1529375314712524,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 263.125,
|
||
|
|
"epoch": 0.5605095541401274,
|
||
|
|
"grad_norm": 3.3247878551483154,
|
||
|
|
"kl": 9.563120489940047,
|
||
|
|
"learning_rate": 2.9650908906359733e-07,
|
||
|
|
"loss": 0.0096,
|
||
|
|
"reward": 17.1003201007843,
|
||
|
|
"reward_std": 4.631968731526285,
|
||
|
|
"rewards/concensus_correctness_reward_func": 12.573749996721745,
|
||
|
|
"rewards/consensus_reward_func": 1.625,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.875,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6720699109137058,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func": 1.104500025510788,
|
||
|
|
"step": 44
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 290.75,
|
||
|
|
"epoch": 0.6114649681528662,
|
||
|
|
"grad_norm": 3.066596508026123,
|
||
|
|
"kl": 12.11058185622096,
|
||
|
|
"learning_rate": 2.955210868866872e-07,
|
||
|
|
"loss": 0.0121,
|
||
|
|
"reward": 19.856602907180786,
|
||
|
|
"reward_std": 8.727525525609963,
|
||
|
|
"rewards/concensus_correctness_reward_func": 15.0,
|
||
|
|
"rewards/consensus_reward_func": 1.5,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.625,
|
||
|
|
"rewards/question_recreation_reward_func": 0.700602900236845,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.21875,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8122500255703926,
|
||
|
|
"step": 48
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 229.4375,
|
||
|
|
"epoch": 0.6624203821656051,
|
||
|
|
"grad_norm": 2.433725118637085,
|
||
|
|
"kl": 1.3885855339467525,
|
||
|
|
"learning_rate": 2.944122080042512e-07,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"reward": 14.267784029245377,
|
||
|
|
"reward_std": 5.394140258431435,
|
||
|
|
"rewards/concensus_correctness_reward_func": 10.306124992668629,
|
||
|
|
"rewards/consensus_reward_func": 0.75,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.875,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6182841658592224,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.46875,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2496250122785568,
|
||
|
|
"step": 52
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 265.3125,
|
||
|
|
"epoch": 0.7133757961783439,
|
||
|
|
"grad_norm": 1.840900182723999,
|
||
|
|
"kl": 6.538037680089474,
|
||
|
|
"learning_rate": 2.93183373503597e-07,
|
||
|
|
"loss": 0.0065,
|
||
|
|
"reward": 16.03761088848114,
|
||
|
|
"reward_std": 10.041991762816906,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.560124978423119,
|
||
|
|
"rewards/consensus_reward_func": 1.25,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.125,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6827985420823097,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.3125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1071875169873238,
|
||
|
|
"step": 56
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 211.0,
|
||
|
|
"epoch": 0.7643312101910829,
|
||
|
|
"grad_norm": 1.3512978553771973,
|
||
|
|
"kl": 7.67945366166532,
|
||
|
|
"learning_rate": 2.91835604112849e-07,
|
||
|
|
"loss": 0.0077,
|
||
|
|
"reward": 19.18588936328888,
|
||
|
|
"reward_std": 6.0852950513362885,
|
||
|
|
"rewards/concensus_correctness_reward_func": 14.20750005543232,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.375,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6377642604056746,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.34375,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2468750029802322,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 209.0625,
|
||
|
|
"epoch": 0.8152866242038217,
|
||
|
|
"grad_norm": 2.5626749992370605,
|
||
|
|
"kl": 2.9191525876522064,
|
||
|
|
"learning_rate": 2.9037001935308313e-07,
|
||
|
|
"loss": 0.0029,
|
||
|
|
"reward": 14.56528091430664,
|
||
|
|
"reward_std": 4.982788998633623,
|
||
|
|
"rewards/concensus_correctness_reward_func": 10.215499997138977,
|
||
|
|
"rewards/consensus_reward_func": 1.25,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.875,
|
||
|
|
"rewards/question_recreation_reward_func": 0.7710309810936451,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2037499845027924,
|
||
|
|
"step": 64
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 215.3125,
|
||
|
|
"epoch": 0.8662420382165605,
|
||
|
|
"grad_norm": 3.383816719055176,
|
||
|
|
"kl": 3.1076866053044796,
|
||
|
|
"learning_rate": 2.8878783660839964e-07,
|
||
|
|
"loss": 0.0031,
|
||
|
|
"reward": 20.26084053516388,
|
||
|
|
"reward_std": 4.580408729612827,
|
||
|
|
"rewards/concensus_correctness_reward_func": 15.626375034451485,
|
||
|
|
"rewards/consensus_reward_func": 1.5,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.25,
|
||
|
|
"rewards/question_recreation_reward_func": 0.41534046828746796,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.34375,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1253750175237656,
|
||
|
|
"step": 68
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 245.3125,
|
||
|
|
"epoch": 0.9171974522292994,
|
||
|
|
"grad_norm": 11.53940200805664,
|
||
|
|
"kl": 36.37679213285446,
|
||
|
|
"learning_rate": 2.8709037011470624e-07,
|
||
|
|
"loss": 0.0364,
|
||
|
|
"reward": 12.166111320257187,
|
||
|
|
"reward_std": 7.469847652129829,
|
||
|
|
"rewards/concensus_correctness_reward_func": 8.189249992370605,
|
||
|
|
"rewards/consensus_reward_func": 1.0,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.0,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6527361150365323,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.28125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0428750179708004,
|
||
|
|
"step": 72
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 252.6875,
|
||
|
|
"epoch": 0.9681528662420382,
|
||
|
|
"grad_norm": 1.9474271535873413,
|
||
|
|
"kl": 5.756619825959206,
|
||
|
|
"learning_rate": 2.852790298680516e-07,
|
||
|
|
"loss": 0.0058,
|
||
|
|
"reward": 18.147495985031128,
|
||
|
|
"reward_std": 5.955903281457722,
|
||
|
|
"rewards/concensus_correctness_reward_func": 13.75,
|
||
|
|
"rewards/consensus_reward_func": 1.625,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.875,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5002461355179548,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.3125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.084750011563301,
|
||
|
|
"step": 76
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 231.14285714285714,
|
||
|
|
"epoch": 1.0127388535031847,
|
||
|
|
"grad_norm": 2.1662113666534424,
|
||
|
|
"kl": 4.730595575911658,
|
||
|
|
"learning_rate": 2.833553204534156e-07,
|
||
|
|
"loss": 0.0041,
|
||
|
|
"reward": 19.707179682595388,
|
||
|
|
"reward_std": 4.576129041612148,
|
||
|
|
"rewards/concensus_correctness_reward_func": 14.77628571646554,
|
||
|
|
"rewards/consensus_reward_func": 1.5714285714285714,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.1428571428571428,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6548936026436942,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.32142857142857145,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2402857201439994,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 181.9375,
|
||
|
|
"epoch": 1.0636942675159236,
|
||
|
|
"grad_norm": 3.710115909576416,
|
||
|
|
"kl": 7.103449460119009,
|
||
|
|
"learning_rate": 2.8132083979493016e-07,
|
||
|
|
"loss": 0.0071,
|
||
|
|
"reward": 15.854183077812195,
|
||
|
|
"reward_std": 8.889904107898474,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.673750042915344,
|
||
|
|
"rewards/consensus_reward_func": 1.125,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.875,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5401829741895199,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.40625,
|
||
|
|
"rewards/xmlcount_reward_func": 1.234000027179718,
|
||
|
|
"step": 84
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 250.4375,
|
||
|
|
"epoch": 1.1146496815286624,
|
||
|
|
"grad_norm": 4.344290256500244,
|
||
|
|
"kl": 3.607717391103506,
|
||
|
|
"learning_rate": 2.7917727782856764e-07,
|
||
|
|
"loss": 0.0036,
|
||
|
|
"reward": 13.101991266012192,
|
||
|
|
"reward_std": 6.616666306043044,
|
||
|
|
"rewards/concensus_correctness_reward_func": 8.970250010490417,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.75,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5953665748238564,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.21875,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1926250606775284,
|
||
|
|
"step": 88
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 224.5625,
|
||
|
|
"epoch": 1.1656050955414012,
|
||
|
|
"grad_norm": 1.8617960214614868,
|
||
|
|
"kl": 2.941370815038681,
|
||
|
|
"learning_rate": 2.769264150983997e-07,
|
||
|
|
"loss": 0.0029,
|
||
|
|
"reward": 18.346871256828308,
|
||
|
|
"reward_std": 3.0000894721597433,
|
||
|
|
"rewards/concensus_correctness_reward_func": 13.75,
|
||
|
|
"rewards/consensus_reward_func": 1.625,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.875,
|
||
|
|
"rewards/question_recreation_reward_func": 0.7451213523745537,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1017500162124634,
|
||
|
|
"step": 92
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 275.3125,
|
||
|
|
"epoch": 1.21656050955414,
|
||
|
|
"grad_norm": 2.1597092151641846,
|
||
|
|
"kl": 7.501682840287685,
|
||
|
|
"learning_rate": 2.745701212775937e-07,
|
||
|
|
"loss": 0.0075,
|
||
|
|
"reward": 15.922978341579437,
|
||
|
|
"reward_std": 10.099833406507969,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.25,
|
||
|
|
"rewards/consensus_reward_func": 1.25,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.375,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6603539576753974,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.28125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1063750311732292,
|
||
|
|
"step": 96
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 232.375,
|
||
|
|
"epoch": 1.267515923566879,
|
||
|
|
"grad_norm": 2.969134569168091,
|
||
|
|
"kl": 7.7300273440778255,
|
||
|
|
"learning_rate": 2.7211035361537307e-07,
|
||
|
|
"loss": 0.0077,
|
||
|
|
"reward": 11.717751204967499,
|
||
|
|
"reward_std": 7.577772298827767,
|
||
|
|
"rewards/concensus_correctness_reward_func": 7.94712507724762,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.5,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5368763818405569,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.21875,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1400000303983688,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 226.0625,
|
||
|
|
"epoch": 1.3184713375796178,
|
||
|
|
"grad_norm": 1.5485366582870483,
|
||
|
|
"kl": 2.664130464196205,
|
||
|
|
"learning_rate": 2.695491553112336e-07,
|
||
|
|
"loss": 0.0027,
|
||
|
|
"reward": 20.976629972457886,
|
||
|
|
"reward_std": 6.513700008392334,
|
||
|
|
"rewards/concensus_correctness_reward_func": 16.460624933242798,
|
||
|
|
"rewards/consensus_reward_func": 1.625,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.0,
|
||
|
|
"rewards/question_recreation_reward_func": 0.4631927441805601,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1778125315904617,
|
||
|
|
"step": 104
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 202.1875,
|
||
|
|
"epoch": 1.3694267515923566,
|
||
|
|
"grad_norm": 5.035473346710205,
|
||
|
|
"kl": 19.53648843243718,
|
||
|
|
"learning_rate": 2.6688865381776554e-07,
|
||
|
|
"loss": 0.0195,
|
||
|
|
"reward": 16.44457733631134,
|
||
|
|
"reward_std": 6.439986160025001,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.878250032663345,
|
||
|
|
"rewards/consensus_reward_func": 1.125,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.25,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6380774034187198,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.40625,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1470000073313713,
|
||
|
|
"step": 108
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 238.375,
|
||
|
|
"epoch": 1.4203821656050954,
|
||
|
|
"grad_norm": 2.5546350479125977,
|
||
|
|
"kl": 4.930681709200144,
|
||
|
|
"learning_rate": 2.6413105907349006e-07,
|
||
|
|
"loss": 0.0049,
|
||
|
|
"reward": 19.794534921646118,
|
||
|
|
"reward_std": 4.685703623574227,
|
||
|
|
"rewards/concensus_correctness_reward_func": 15.0,
|
||
|
|
"rewards/consensus_reward_func": 1.625,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.0,
|
||
|
|
"rewards/question_recreation_reward_func": 0.7159727662801743,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.28125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1723125129938126,
|
||
|
|
"step": 112
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 213.0625,
|
||
|
|
"epoch": 1.4713375796178343,
|
||
|
|
"grad_norm": 2.9883830547332764,
|
||
|
|
"kl": 3.1261193864047527,
|
||
|
|
"learning_rate": 2.612786616671803e-07,
|
||
|
|
"loss": 0.0031,
|
||
|
|
"reward": 13.908211499452591,
|
||
|
|
"reward_std": 4.823537867516279,
|
||
|
|
"rewards/concensus_correctness_reward_func": 10.0,
|
||
|
|
"rewards/consensus_reward_func": 1.125,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.625,
|
||
|
|
"rewards/question_recreation_reward_func": 0.689586691558361,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.3125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1561250165104866,
|
||
|
|
"step": 116
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 229.25,
|
||
|
|
"epoch": 1.5222929936305731,
|
||
|
|
"grad_norm": 2.6060760021209717,
|
||
|
|
"kl": 93.09971876814961,
|
||
|
|
"learning_rate": 2.583338309351894e-07,
|
||
|
|
"loss": 0.0931,
|
||
|
|
"reward": 14.722533643245697,
|
||
|
|
"reward_std": 3.7422131042694673,
|
||
|
|
"rewards/concensus_correctness_reward_func": 10.464500008150935,
|
||
|
|
"rewards/consensus_reward_func": 1.5,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.75,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5823462381958961,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.28125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.14443751424551,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 241.9375,
|
||
|
|
"epoch": 1.573248407643312,
|
||
|
|
"grad_norm": 3.838078260421753,
|
||
|
|
"kl": 6.129395470023155,
|
||
|
|
"learning_rate": 2.55299012993368e-07,
|
||
|
|
"loss": 0.0061,
|
||
|
|
"reward": 18.527520298957825,
|
||
|
|
"reward_std": 6.345299224369228,
|
||
|
|
"rewards/concensus_correctness_reward_func": 14.059624962508678,
|
||
|
|
"rewards/consensus_reward_func": 1.5,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.375,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5597703522071242,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.1875,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8456250205636024,
|
||
|
|
"step": 124
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 228.1875,
|
||
|
|
"epoch": 1.6242038216560508,
|
||
|
|
"grad_norm": 2.3569176197052,
|
||
|
|
"kl": 190.0281139537692,
|
||
|
|
"learning_rate": 2.5217672870520407e-07,
|
||
|
|
"loss": 0.19,
|
||
|
|
"reward": 19.29222685098648,
|
||
|
|
"reward_std": 0.9815351580036804,
|
||
|
|
"rewards/concensus_correctness_reward_func": 15.0,
|
||
|
|
"rewards/consensus_reward_func": 1.5,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.875,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5401642136275768,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.28125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0958125218749046,
|
||
|
|
"step": 128
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 216.75,
|
||
|
|
"epoch": 1.6751592356687897,
|
||
|
|
"grad_norm": 1.5537621974945068,
|
||
|
|
"kl": 2.767712377011776,
|
||
|
|
"learning_rate": 2.489695715878747e-07,
|
||
|
|
"loss": 0.0028,
|
||
|
|
"reward": 18.718332588672638,
|
||
|
|
"reward_std": 6.182364942971617,
|
||
|
|
"rewards/concensus_correctness_reward_func": 13.75,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.625,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5509578062337823,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1673750281333923,
|
||
|
|
"step": 132
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 257.9375,
|
||
|
|
"epoch": 1.7261146496815285,
|
||
|
|
"grad_norm": 3.8648273944854736,
|
||
|
|
"kl": 2.600251194089651,
|
||
|
|
"learning_rate": 2.4568020565794784e-07,
|
||
|
|
"loss": 0.0026,
|
||
|
|
"reward": 10.30246677994728,
|
||
|
|
"reward_std": 6.988724898546934,
|
||
|
|
"rewards/concensus_correctness_reward_func": 6.4593749940395355,
|
||
|
|
"rewards/consensus_reward_func": 0.875,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.0,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6528416611254215,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.21875,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0965000167489052,
|
||
|
|
"step": 136
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 242.6875,
|
||
|
|
"epoch": 1.7770700636942676,
|
||
|
|
"grad_norm": 54.2701530456543,
|
||
|
|
"kl": 31.665705926716328,
|
||
|
|
"learning_rate": 2.4231136321852394e-07,
|
||
|
|
"loss": 0.0317,
|
||
|
|
"reward": 20.147151827812195,
|
||
|
|
"reward_std": 7.880714102386264,
|
||
|
|
"rewards/concensus_correctness_reward_func": 15.221750020980835,
|
||
|
|
"rewards/consensus_reward_func": 1.5,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.375,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6395266838371754,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.3125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0983750149607658,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 229.6875,
|
||
|
|
"epoch": 1.8280254777070064,
|
||
|
|
"grad_norm": 2.66843581199646,
|
||
|
|
"kl": 3.5122765973210335,
|
||
|
|
"learning_rate": 2.3886584258965555e-07,
|
||
|
|
"loss": 0.0035,
|
||
|
|
"reward": 15.746762156486511,
|
||
|
|
"reward_std": 5.882234873250127,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.25,
|
||
|
|
"rewards/consensus_reward_func": 1.5,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.0,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5443250723183155,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.28125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1711875051259995,
|
||
|
|
"step": 144
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 270.125,
|
||
|
|
"epoch": 1.8789808917197452,
|
||
|
|
"grad_norm": 1.622960090637207,
|
||
|
|
"kl": 7.444879077374935,
|
||
|
|
"learning_rate": 2.3534650578393037e-07,
|
||
|
|
"loss": 0.0074,
|
||
|
|
"reward": 13.791082382202148,
|
||
|
|
"reward_std": 6.4896623231470585,
|
||
|
|
"rewards/concensus_correctness_reward_func": 9.334750011563301,
|
||
|
|
"rewards/consensus_reward_func": 1.25,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.25,
|
||
|
|
"rewards/question_recreation_reward_func": 0.698457557708025,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.21875,
|
||
|
|
"rewards/xmlcount_reward_func": 1.03912503272295,
|
||
|
|
"step": 148
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 199.625,
|
||
|
|
"epoch": 1.929936305732484,
|
||
|
|
"grad_norm": 2.772538900375366,
|
||
|
|
"kl": 1.7244556173682213,
|
||
|
|
"learning_rate": 2.3175627612914788e-07,
|
||
|
|
"loss": 0.0017,
|
||
|
|
"reward": 18.972005277872086,
|
||
|
|
"reward_std": 2.4147312035784125,
|
||
|
|
"rewards/concensus_correctness_reward_func": 13.971750006079674,
|
||
|
|
"rewards/consensus_reward_func": 1.625,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.25,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5098803527653217,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.40625,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2091249972581863,
|
||
|
|
"step": 152
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 244.0625,
|
||
|
|
"epoch": 1.980891719745223,
|
||
|
|
"grad_norm": 2.382359504699707,
|
||
|
|
"kl": 6.68058829754591,
|
||
|
|
"learning_rate": 2.2809813584006504e-07,
|
||
|
|
"loss": 0.0067,
|
||
|
|
"reward": 18.733210682868958,
|
||
|
|
"reward_std": 5.422195242252201,
|
||
|
|
"rewards/concensus_correctness_reward_func": 14.069375038146973,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.5,
|
||
|
|
"rewards/question_recreation_reward_func": 0.4836479816585779,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.21875,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0864375233650208,
|
||
|
|
"step": 156
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 189.71428571428572,
|
||
|
|
"epoch": 2.0254777070063694,
|
||
|
|
"grad_norm": 4.063220977783203,
|
||
|
|
"kl": 2.7896384043352946,
|
||
|
|
"learning_rate": 2.243751235412272e-07,
|
||
|
|
"loss": 0.0024,
|
||
|
|
"reward": 19.006362915039062,
|
||
|
|
"reward_std": 5.089762266193118,
|
||
|
|
"rewards/concensus_correctness_reward_func": 14.781285745756966,
|
||
|
|
"rewards/consensus_reward_func": 1.5714285714285714,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.8571428571428571,
|
||
|
|
"rewards/question_recreation_reward_func": 0.515291694019522,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.2857142857142857,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9955000025885445,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 213.4375,
|
||
|
|
"epoch": 2.0764331210191083,
|
||
|
|
"grad_norm": 45.27822494506836,
|
||
|
|
"kl": 18.627060689032078,
|
||
|
|
"learning_rate": 2.2059033174294308e-07,
|
||
|
|
"loss": 0.0186,
|
||
|
|
"reward": 17.683844089508057,
|
||
|
|
"reward_std": 10.830251511186361,
|
||
|
|
"rewards/concensus_correctness_reward_func": 13.190500020980835,
|
||
|
|
"rewards/consensus_reward_func": 1.125,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.375,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5491568585857749,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.28125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.162937492132187,
|
||
|
|
"step": 164
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 233.25,
|
||
|
|
"epoch": 2.127388535031847,
|
||
|
|
"grad_norm": 10.763078689575195,
|
||
|
|
"kl": 8.058194138109684,
|
||
|
|
"learning_rate": 2.167469042724989e-07,
|
||
|
|
"loss": 0.0081,
|
||
|
|
"reward": 16.62453094124794,
|
||
|
|
"reward_std": 2.8090486722067,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.670500010251999,
|
||
|
|
"rewards/consensus_reward_func": 1.5,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.5,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6700308993458748,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.21875,
|
||
|
|
"rewards/xmlcount_reward_func": 1.065250001847744,
|
||
|
|
"step": 168
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 239.9375,
|
||
|
|
"epoch": 2.178343949044586,
|
||
|
|
"grad_norm": 2.4544873237609863,
|
||
|
|
"kl": 4.6812648214399815,
|
||
|
|
"learning_rate": 2.1284803366274665e-07,
|
||
|
|
"loss": 0.0047,
|
||
|
|
"reward": 10.69625610113144,
|
||
|
|
"reward_std": 2.8211628943681717,
|
||
|
|
"rewards/concensus_correctness_reward_func": 6.55375000089407,
|
||
|
|
"rewards/consensus_reward_func": 1.0,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.125,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5454436782747507,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2220625132322311,
|
||
|
|
"step": 172
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 243.1875,
|
||
|
|
"epoch": 2.229299363057325,
|
||
|
|
"grad_norm": 2.8351073265075684,
|
||
|
|
"kl": 11.15944654121995,
|
||
|
|
"learning_rate": 2.0889695850023532e-07,
|
||
|
|
"loss": 0.0111,
|
||
|
|
"reward": 14.30355390906334,
|
||
|
|
"reward_std": 3.1945501037698705,
|
||
|
|
"rewards/concensus_correctness_reward_func": 10.470749974250793,
|
||
|
|
"rewards/consensus_reward_func": 1.25,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.75,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5688038393855095,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.21875,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0452500134706497,
|
||
|
|
"step": 176
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 227.25,
|
||
|
|
"epoch": 2.2802547770700636,
|
||
|
|
"grad_norm": 2.5390284061431885,
|
||
|
|
"kl": 4.675673048943281,
|
||
|
|
"learning_rate": 2.048969607350874e-07,
|
||
|
|
"loss": 0.0047,
|
||
|
|
"reward": 13.208377301692963,
|
||
|
|
"reward_std": 6.394375938922167,
|
||
|
|
"rewards/concensus_correctness_reward_func": 9.178874969482422,
|
||
|
|
"rewards/consensus_reward_func": 1.25,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.875,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5128772370517254,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.34375,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0478749945759773,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 241.5,
|
||
|
|
"epoch": 2.3312101910828025,
|
||
|
|
"grad_norm": 339.2472839355469,
|
||
|
|
"kl": 120.68484226241708,
|
||
|
|
"learning_rate": 2.008513629548557e-07,
|
||
|
|
"loss": 0.1207,
|
||
|
|
"reward": 12.759812444448471,
|
||
|
|
"reward_std": 5.974757697433233,
|
||
|
|
"rewards/concensus_correctness_reward_func": 9.136750057339668,
|
||
|
|
"rewards/consensus_reward_func": 1.125,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.625,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5173125192523003,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.28125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0745000019669533,
|
||
|
|
"step": 184
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 266.1875,
|
||
|
|
"epoch": 2.3821656050955413,
|
||
|
|
"grad_norm": 9.494641304016113,
|
||
|
|
"kl": 6.892231393605471,
|
||
|
|
"learning_rate": 1.967635256246248e-07,
|
||
|
|
"loss": 0.0069,
|
||
|
|
"reward": 18.39091458916664,
|
||
|
|
"reward_std": 3.120848624035716,
|
||
|
|
"rewards/concensus_correctness_reward_func": 13.973124995827675,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.125,
|
||
|
|
"rewards/question_recreation_reward_func": 0.630602179095149,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func": 1.03718750923872,
|
||
|
|
"step": 188
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 256.1875,
|
||
|
|
"epoch": 2.43312101910828,
|
||
|
|
"grad_norm": 3.3058788776397705,
|
||
|
|
"kl": 3.385168194770813,
|
||
|
|
"learning_rate": 1.9263684429564973e-07,
|
||
|
|
"loss": 0.0034,
|
||
|
|
"reward": 15.78951108455658,
|
||
|
|
"reward_std": 9.866211914690211,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.25,
|
||
|
|
"rewards/consensus_reward_func": 1.25,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.125,
|
||
|
|
"rewards/question_recreation_reward_func": 0.7607610449194908,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.3125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0912500247359276,
|
||
|
|
"step": 192
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 266.625,
|
||
|
|
"epoch": 2.484076433121019,
|
||
|
|
"grad_norm": 3.944892644882202,
|
||
|
|
"kl": 8.661903837695718,
|
||
|
|
"learning_rate": 1.8847474678485014e-07,
|
||
|
|
"loss": 0.0087,
|
||
|
|
"reward": 17.91079044342041,
|
||
|
|
"reward_std": 7.754661478335038,
|
||
|
|
"rewards/concensus_correctness_reward_func": 12.71512496471405,
|
||
|
|
"rewards/consensus_reward_func": 1.5,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.5,
|
||
|
|
"rewards/question_recreation_reward_func": 0.666103033348918,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.3125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2170625180006027,
|
||
|
|
"step": 196
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 225.8125,
|
||
|
|
"epoch": 2.535031847133758,
|
||
|
|
"grad_norm": 2.1859211921691895,
|
||
|
|
"kl": 4.23666650429368,
|
||
|
|
"learning_rate": 1.8428069032750338e-07,
|
||
|
|
"loss": 0.0042,
|
||
|
|
"reward": 19.735592007637024,
|
||
|
|
"reward_std": 7.996539369225502,
|
||
|
|
"rewards/concensus_correctness_reward_func": 15.209874987602234,
|
||
|
|
"rewards/consensus_reward_func": 1.25,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.25,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5920920763164759,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.28125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1523750126361847,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 256.375,
|
||
|
|
"epoch": 2.5859872611464967,
|
||
|
|
"grad_norm": 6.963754653930664,
|
||
|
|
"kl": 56.80678379908204,
|
||
|
|
"learning_rate": 1.8005815870550113e-07,
|
||
|
|
"loss": 0.0568,
|
||
|
|
"reward": 16.95860594511032,
|
||
|
|
"reward_std": 7.698275171220303,
|
||
|
|
"rewards/concensus_correctness_reward_func": 12.922000050544739,
|
||
|
|
"rewards/consensus_reward_func": 1.25,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.875,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5784185063093901,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.34375,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9894375093281269,
|
||
|
|
"step": 204
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 235.6875,
|
||
|
|
"epoch": 2.6369426751592355,
|
||
|
|
"grad_norm": 56.92842102050781,
|
||
|
|
"kl": 21.92546533420682,
|
||
|
|
"learning_rate": 1.7581065935355522e-07,
|
||
|
|
"loss": 0.0219,
|
||
|
|
"reward": 18.0313401222229,
|
||
|
|
"reward_std": 5.557751599699259,
|
||
|
|
"rewards/concensus_correctness_reward_func": 13.97724997997284,
|
||
|
|
"rewards/consensus_reward_func": 1.5,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.875,
|
||
|
|
"rewards/question_recreation_reward_func": 0.42977757239714265,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.15625,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0930624902248383,
|
||
|
|
"step": 208
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 239.125,
|
||
|
|
"epoch": 2.6878980891719744,
|
||
|
|
"grad_norm": 4.275964736938477,
|
||
|
|
"kl": 2.6714395098388195,
|
||
|
|
"learning_rate": 1.715417204457562e-07,
|
||
|
|
"loss": 0.0027,
|
||
|
|
"reward": 18.481036722660065,
|
||
|
|
"reward_std": 5.910495655843988,
|
||
|
|
"rewards/concensus_correctness_reward_func": 13.918125033378601,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.0,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6785994963720441,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.3125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1968125253915787,
|
||
|
|
"step": 212
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 238.1875,
|
||
|
|
"epoch": 2.738853503184713,
|
||
|
|
"grad_norm": 2.9757699966430664,
|
||
|
|
"kl": 4.989309325814247,
|
||
|
|
"learning_rate": 1.6725488796490503e-07,
|
||
|
|
"loss": 0.0051,
|
||
|
|
"reward": 16.237927079200745,
|
||
|
|
"reward_std": 5.6823064510535914,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.685875058174133,
|
||
|
|
"rewards/consensus_reward_func": 1.625,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.875,
|
||
|
|
"rewards/question_recreation_reward_func": 0.57005206332542,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2320000380277634,
|
||
|
|
"step": 216
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 210.75,
|
||
|
|
"epoch": 2.789808917197452,
|
||
|
|
"grad_norm": 6.644160270690918,
|
||
|
|
"kl": 28.66787153482437,
|
||
|
|
"learning_rate": 1.6295372275705162e-07,
|
||
|
|
"loss": 0.0287,
|
||
|
|
"reward": 15.902063801884651,
|
||
|
|
"reward_std": 2.8661335185170174,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.25,
|
||
|
|
"rewards/consensus_reward_func": 1.5,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.375,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5894390102475882,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.21875,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9688750207424164,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 225.875,
|
||
|
|
"epoch": 2.840764331210191,
|
||
|
|
"grad_norm": 2.6777331829071045,
|
||
|
|
"kl": 139.06324845552444,
|
||
|
|
"learning_rate": 1.5864179757368764e-07,
|
||
|
|
"loss": 0.1391,
|
||
|
|
"reward": 17.266418665647507,
|
||
|
|
"reward_std": 7.936541106551886,
|
||
|
|
"rewards/concensus_correctness_reward_func": 12.5,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.25,
|
||
|
|
"rewards/question_recreation_reward_func": 0.7135435082018375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1778749972581863,
|
||
|
|
"step": 224
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 223.4375,
|
||
|
|
"epoch": 2.8917197452229297,
|
||
|
|
"grad_norm": 5.439139366149902,
|
||
|
|
"kl": 3.8189243748784065,
|
||
|
|
"learning_rate": 1.5432269410404998e-07,
|
||
|
|
"loss": 0.0038,
|
||
|
|
"reward": 13.274927824735641,
|
||
|
|
"reward_std": 2.9763048992026597,
|
||
|
|
"rewards/concensus_correctness_reward_func": 9.352000012993813,
|
||
|
|
"rewards/consensus_reward_func": 1.25,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.75,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6428653616458178,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.15625,
|
||
|
|
"rewards/xmlcount_reward_func": 1.123812511563301,
|
||
|
|
"step": 228
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 247.5,
|
||
|
|
"epoch": 2.9426751592356686,
|
||
|
|
"grad_norm": 170.2266845703125,
|
||
|
|
"kl": 167.23719503730536,
|
||
|
|
"learning_rate": 1.5e-07,
|
||
|
|
"loss": 0.1672,
|
||
|
|
"reward": 14.749447345733643,
|
||
|
|
"reward_std": 7.275530036538839,
|
||
|
|
"rewards/concensus_correctness_reward_func": 10.215250015258789,
|
||
|
|
"rewards/consensus_reward_func": 1.5,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.0,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5158222373574972,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.3125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2058750241994858,
|
||
|
|
"step": 232
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 268.0,
|
||
|
|
"epoch": 2.9936305732484074,
|
||
|
|
"grad_norm": 1.7273995876312256,
|
||
|
|
"kl": 5.3727107755839825,
|
||
|
|
"learning_rate": 1.4567730589595006e-07,
|
||
|
|
"loss": 0.0054,
|
||
|
|
"reward": 19.68195617198944,
|
||
|
|
"reward_std": 0.35719050047919154,
|
||
|
|
"rewards/concensus_correctness_reward_func": 15.112999998033047,
|
||
|
|
"rewards/consensus_reward_func": 1.625,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.0,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6851438917219639,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.1875,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0713125243782997,
|
||
|
|
"step": 236
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 256.64285714285717,
|
||
|
|
"epoch": 3.038216560509554,
|
||
|
|
"grad_norm": 110.74983978271484,
|
||
|
|
"kl": 41.563892917973654,
|
||
|
|
"learning_rate": 1.4135820242631235e-07,
|
||
|
|
"loss": 0.0364,
|
||
|
|
"reward": 15.650528294699532,
|
||
|
|
"reward_std": 4.540684355156762,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.661857196262904,
|
||
|
|
"rewards/consensus_reward_func": 1.5714285714285714,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.7142857142857143,
|
||
|
|
"rewards/question_recreation_reward_func": 0.49124259342040333,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.17857142857142858,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0331428689616067,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 241.625,
|
||
|
|
"epoch": 3.089171974522293,
|
||
|
|
"grad_norm": 2.8471550941467285,
|
||
|
|
"kl": 26.10317723825574,
|
||
|
|
"learning_rate": 1.3704627724294842e-07,
|
||
|
|
"loss": 0.0261,
|
||
|
|
"reward": 15.622052431106567,
|
||
|
|
"reward_std": 6.783383652102202,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.25,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.0,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6173020545393229,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.28125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0985000133514404,
|
||
|
|
"step": 244
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 237.625,
|
||
|
|
"epoch": 3.140127388535032,
|
||
|
|
"grad_norm": 4.9774088859558105,
|
||
|
|
"kl": 2.7739184722304344,
|
||
|
|
"learning_rate": 1.3274511203509498e-07,
|
||
|
|
"loss": 0.0028,
|
||
|
|
"reward": 16.195281475782394,
|
||
|
|
"reward_std": 5.6568857207894325,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.760749965906143,
|
||
|
|
"rewards/consensus_reward_func": 1.0,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.375,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6031564082950354,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2063750326633453,
|
||
|
|
"step": 248
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 232.25,
|
||
|
|
"epoch": 3.1910828025477707,
|
||
|
|
"grad_norm": 3.6411895751953125,
|
||
|
|
"kl": 8.415857467800379,
|
||
|
|
"learning_rate": 1.2845827955424378e-07,
|
||
|
|
"loss": 0.0084,
|
||
|
|
"reward": 21.606714069843292,
|
||
|
|
"reward_std": 2.034520313842222,
|
||
|
|
"rewards/concensus_correctness_reward_func": 16.477874994277954,
|
||
|
|
"rewards/consensus_reward_func": 1.875,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.25,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5895260944962502,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.28125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1330625265836716,
|
||
|
|
"step": 252
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 219.5625,
|
||
|
|
"epoch": 3.2420382165605095,
|
||
|
|
"grad_norm": 1.8258823156356812,
|
||
|
|
"kl": 16.24698005989194,
|
||
|
|
"learning_rate": 1.241893406464448e-07,
|
||
|
|
"loss": 0.0162,
|
||
|
|
"reward": 18.671132683753967,
|
||
|
|
"reward_std": 5.8178422772325575,
|
||
|
|
"rewards/concensus_correctness_reward_func": 13.984624981880188,
|
||
|
|
"rewards/consensus_reward_func": 1.5,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.0,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6623825784772635,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.3125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2116250097751617,
|
||
|
|
"step": 256
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 207.75,
|
||
|
|
"epoch": 3.2929936305732483,
|
||
|
|
"grad_norm": 4.799376010894775,
|
||
|
|
"kl": 5.706813037395477,
|
||
|
|
"learning_rate": 1.199418412944989e-07,
|
||
|
|
"loss": 0.0058,
|
||
|
|
"reward": 17.392418444156647,
|
||
|
|
"reward_std": 4.783364934177371,
|
||
|
|
"rewards/concensus_correctness_reward_func": 12.703624963760376,
|
||
|
|
"rewards/consensus_reward_func": 1.25,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.25,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6976060681045055,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.3125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1786875426769257,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 222.8125,
|
||
|
|
"epoch": 3.343949044585987,
|
||
|
|
"grad_norm": 2.5605955123901367,
|
||
|
|
"kl": 9.123412758111954,
|
||
|
|
"learning_rate": 1.1571930967249665e-07,
|
||
|
|
"loss": 0.0091,
|
||
|
|
"reward": 14.08166092634201,
|
||
|
|
"reward_std": 6.08101740386337,
|
||
|
|
"rewards/concensus_correctness_reward_func": 9.220499992370605,
|
||
|
|
"rewards/consensus_reward_func": 1.5,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.125,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6586607862263918,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.34375,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2337500303983688,
|
||
|
|
"step": 264
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 289.375,
|
||
|
|
"epoch": 3.394904458598726,
|
||
|
|
"grad_norm": 1.0773470401763916,
|
||
|
|
"kl": 7.33378242328763,
|
||
|
|
"learning_rate": 1.1152525321514992e-07,
|
||
|
|
"loss": 0.0073,
|
||
|
|
"reward": 15.252871230244637,
|
||
|
|
"reward_std": 2.67856452986598,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.25,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.875,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5149340042844415,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.1875,
|
||
|
|
"rewards/xmlcount_reward_func": 1.050437517464161,
|
||
|
|
"step": 268
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 248.8125,
|
||
|
|
"epoch": 3.445859872611465,
|
||
|
|
"grad_norm": 3.0849928855895996,
|
||
|
|
"kl": 4.085891917347908,
|
||
|
|
"learning_rate": 1.0736315570435027e-07,
|
||
|
|
"loss": 0.0041,
|
||
|
|
"reward": 12.67966365814209,
|
||
|
|
"reward_std": 6.439146310091019,
|
||
|
|
"rewards/concensus_correctness_reward_func": 9.08587496727705,
|
||
|
|
"rewards/consensus_reward_func": 0.875,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.875,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5481634996831417,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.3125,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9831250309944153,
|
||
|
|
"step": 272
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 250.5,
|
||
|
|
"epoch": 3.4968152866242037,
|
||
|
|
"grad_norm": 2.536210536956787,
|
||
|
|
"kl": 2.399098828434944,
|
||
|
|
"learning_rate": 1.0323647437537522e-07,
|
||
|
|
"loss": 0.0024,
|
||
|
|
"reward": 14.057019233703613,
|
||
|
|
"reward_std": 11.72206101194024,
|
||
|
|
"rewards/concensus_correctness_reward_func": 10.0,
|
||
|
|
"rewards/consensus_reward_func": 1.125,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.875,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5792695693671703,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.3125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1652500182390213,
|
||
|
|
"step": 276
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 223.6875,
|
||
|
|
"epoch": 3.5477707006369426,
|
||
|
|
"grad_norm": 8.720197677612305,
|
||
|
|
"kl": 10.37247185409069,
|
||
|
|
"learning_rate": 9.914863704514432e-08,
|
||
|
|
"loss": 0.0104,
|
||
|
|
"reward": 18.15606963634491,
|
||
|
|
"reward_std": 7.864195089787245,
|
||
|
|
"rewards/concensus_correctness_reward_func": 13.134375005960464,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.5,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6000700099393725,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.375,
|
||
|
|
"rewards/xmlcount_reward_func": 1.171625018119812,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 241.5,
|
||
|
|
"epoch": 3.5987261146496814,
|
||
|
|
"grad_norm": 2.412242889404297,
|
||
|
|
"kl": 5.458444844931364,
|
||
|
|
"learning_rate": 9.510303926491262e-08,
|
||
|
|
"loss": 0.0055,
|
||
|
|
"reward": 15.639052152633667,
|
||
|
|
"reward_std": 8.874945568852127,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.855250000953674,
|
||
|
|
"rewards/consensus_reward_func": 1.125,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.625,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5041148196905851,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.34375,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1859375089406967,
|
||
|
|
"step": 284
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 262.0,
|
||
|
|
"epoch": 3.6496815286624202,
|
||
|
|
"grad_norm": 2.1715633869171143,
|
||
|
|
"kl": 3.581681564450264,
|
||
|
|
"learning_rate": 9.11030414997647e-08,
|
||
|
|
"loss": 0.0036,
|
||
|
|
"reward": 15.71645438671112,
|
||
|
|
"reward_std": 5.47196826711297,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.672625064849854,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.625,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6106421127915382,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.21875,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2144375145435333,
|
||
|
|
"step": 288
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 287.875,
|
||
|
|
"epoch": 3.700636942675159,
|
||
|
|
"grad_norm": 1.9456199407577515,
|
||
|
|
"kl": 37.41057540290058,
|
||
|
|
"learning_rate": 8.715196633725341e-08,
|
||
|
|
"loss": 0.0374,
|
||
|
|
"reward": 15.976769387722015,
|
||
|
|
"reward_std": 6.066280534490943,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.459874987602234,
|
||
|
|
"rewards/consensus_reward_func": 1.5,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.0,
|
||
|
|
"rewards/question_recreation_reward_func": 0.7050191890448332,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.15625,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1556250378489494,
|
||
|
|
"step": 292
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 189.875,
|
||
|
|
"epoch": 3.7515923566878984,
|
||
|
|
"grad_norm": 4.28585147857666,
|
||
|
|
"kl": 12.30507379770279,
|
||
|
|
"learning_rate": 8.325309572750114e-08,
|
||
|
|
"loss": 0.0123,
|
||
|
|
"reward": 18.18193393945694,
|
||
|
|
"reward_std": 6.495595270884223,
|
||
|
|
"rewards/concensus_correctness_reward_func": 13.848375000059605,
|
||
|
|
"rewards/consensus_reward_func": 1.5,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.75,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6111842412501574,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.3125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1598750203847885,
|
||
|
|
"step": 296
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 209.5625,
|
||
|
|
"epoch": 3.802547770700637,
|
||
|
|
"grad_norm": 2.9255335330963135,
|
||
|
|
"kl": 2.6841611191630363,
|
||
|
|
"learning_rate": 7.940966825705694e-08,
|
||
|
|
"loss": 0.0028,
|
||
|
|
"reward": 18.69688767194748,
|
||
|
|
"reward_std": 7.003051041887375,
|
||
|
|
"rewards/concensus_correctness_reward_func": 13.75,
|
||
|
|
"rewards/consensus_reward_func": 1.625,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.125,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6851999312639236,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.3125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1991875022649765,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 239.4375,
|
||
|
|
"epoch": 3.853503184713376,
|
||
|
|
"grad_norm": 1089.72265625,
|
||
|
|
"kl": 959.4794948212802,
|
||
|
|
"learning_rate": 7.562487645877278e-08,
|
||
|
|
"loss": 0.9595,
|
||
|
|
"reward": 15.485539972782135,
|
||
|
|
"reward_std": 0.4287999947555363,
|
||
|
|
"rewards/concensus_correctness_reward_func": 10.538875006139278,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.25,
|
||
|
|
"rewards/question_recreation_reward_func": 0.7834777049720287,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.34375,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1944375187158585,
|
||
|
|
"step": 304
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 183.5,
|
||
|
|
"epoch": 3.904458598726115,
|
||
|
|
"grad_norm": 2.360800266265869,
|
||
|
|
"kl": 7.300635326653719,
|
||
|
|
"learning_rate": 7.1901864159935e-08,
|
||
|
|
"loss": 0.0073,
|
||
|
|
"reward": 20.999223351478577,
|
||
|
|
"reward_std": 5.513432171428576,
|
||
|
|
"rewards/concensus_correctness_reward_func": 16.687874913215637,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.0,
|
||
|
|
"rewards/question_recreation_reward_func": 0.4483482171781361,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.28125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.206750027835369,
|
||
|
|
"step": 308
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 219.875,
|
||
|
|
"epoch": 3.9554140127388537,
|
||
|
|
"grad_norm": 1.964698314666748,
|
||
|
|
"kl": 1.7256745472550392,
|
||
|
|
"learning_rate": 6.824372387085212e-08,
|
||
|
|
"loss": 0.0017,
|
||
|
|
"reward": 13.001979649066925,
|
||
|
|
"reward_std": 9.3370072809048,
|
||
|
|
"rewards/concensus_correctness_reward_func": 9.080874972045422,
|
||
|
|
"rewards/consensus_reward_func": 1.0,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.875,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5092292642220855,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.3125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.224374994635582,
|
||
|
|
"step": 312
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 235.14285714285714,
|
||
|
|
"epoch": 4.0,
|
||
|
|
"grad_norm": 0.010738754644989967,
|
||
|
|
"kl": 12.37380084821156,
|
||
|
|
"learning_rate": 6.465349421606967e-08,
|
||
|
|
"loss": 0.0109,
|
||
|
|
"reward": 16.182638917650497,
|
||
|
|
"reward_std": 4.973704278768439,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.428571428571429,
|
||
|
|
"rewards/consensus_reward_func": 1.7142857142857142,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.7142857142857143,
|
||
|
|
"rewards/question_recreation_reward_func": 0.760781603200095,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.32142857142857145,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2432857240949358,
|
||
|
|
"step": 316
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 222.625,
|
||
|
|
"epoch": 4.050955414012739,
|
||
|
|
"grad_norm": 3.1885955333709717,
|
||
|
|
"kl": 10.466495990753174,
|
||
|
|
"learning_rate": 6.113415741034446e-08,
|
||
|
|
"loss": 0.0105,
|
||
|
|
"reward": 18.242493152618408,
|
||
|
|
"reward_std": 2.5054425531998277,
|
||
|
|
"rewards/concensus_correctness_reward_func": 13.963750004768372,
|
||
|
|
"rewards/consensus_reward_func": 1.5,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.75,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5736186504364014,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.28125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1738750040531158,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 211.25,
|
||
|
|
"epoch": 4.101910828025478,
|
||
|
|
"grad_norm": 2.3097496032714844,
|
||
|
|
"kl": 10.439931187778711,
|
||
|
|
"learning_rate": 5.768863678147609e-08,
|
||
|
|
"loss": 0.0104,
|
||
|
|
"reward": 15.326348930597305,
|
||
|
|
"reward_std": 4.878300238400698,
|
||
|
|
"rewards/concensus_correctness_reward_func": 10.434500008821487,
|
||
|
|
"rewards/consensus_reward_func": 1.5,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.25,
|
||
|
|
"rewards/question_recreation_reward_func": 0.7187239453196526,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.34375,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0793750137090683,
|
||
|
|
"step": 324
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 242.8125,
|
||
|
|
"epoch": 4.1528662420382165,
|
||
|
|
"grad_norm": 2.6428279876708984,
|
||
|
|
"kl": 31.37735340744257,
|
||
|
|
"learning_rate": 5.4319794342052186e-08,
|
||
|
|
"loss": 0.0314,
|
||
|
|
"reward": 13.825124025344849,
|
||
|
|
"reward_std": 9.31145294936141,
|
||
|
|
"rewards/concensus_correctness_reward_func": 9.380625009536743,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.0,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5849994292948395,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.28125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2032500058412552,
|
||
|
|
"step": 328
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 185.875,
|
||
|
|
"epoch": 4.203821656050955,
|
||
|
|
"grad_norm": 8.287538528442383,
|
||
|
|
"kl": 4.1289902329444885,
|
||
|
|
"learning_rate": 5.103042841212535e-08,
|
||
|
|
"loss": 0.0041,
|
||
|
|
"reward": 23.004461526870728,
|
||
|
|
"reward_std": 3.9006739165633917,
|
||
|
|
"rewards/concensus_correctness_reward_func": 17.7224999666214,
|
||
|
|
"rewards/consensus_reward_func": 1.75,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.375,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6572116315364838,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.28125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.218500018119812,
|
||
|
|
"step": 332
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 221.75,
|
||
|
|
"epoch": 4.254777070063694,
|
||
|
|
"grad_norm": 148.15139770507812,
|
||
|
|
"kl": 78.43285623937845,
|
||
|
|
"learning_rate": 4.782327129479596e-08,
|
||
|
|
"loss": 0.0784,
|
||
|
|
"reward": 17.33521395921707,
|
||
|
|
"reward_std": 4.419136416167021,
|
||
|
|
"rewards/concensus_correctness_reward_func": 12.714375019073486,
|
||
|
|
"rewards/consensus_reward_func": 1.625,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.875,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6673390492796898,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.34375,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1097500286996365,
|
||
|
|
"step": 336
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 229.9375,
|
||
|
|
"epoch": 4.305732484076433,
|
||
|
|
"grad_norm": 1.7480417490005493,
|
||
|
|
"kl": 28.588999539613724,
|
||
|
|
"learning_rate": 4.4700987006632047e-08,
|
||
|
|
"loss": 0.0286,
|
||
|
|
"reward": 22.500885009765625,
|
||
|
|
"reward_std": 4.617068930529058,
|
||
|
|
"rewards/concensus_correctness_reward_func": 17.5,
|
||
|
|
"rewards/consensus_reward_func": 1.75,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.125,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5405731201171875,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.34375,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2415625154972076,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 241.875,
|
||
|
|
"epoch": 4.356687898089172,
|
||
|
|
"grad_norm": 2.9678404331207275,
|
||
|
|
"kl": 11.135519992560148,
|
||
|
|
"learning_rate": 4.166616906481058e-08,
|
||
|
|
"loss": 0.0111,
|
||
|
|
"reward": 22.390071988105774,
|
||
|
|
"reward_std": 4.402062423527241,
|
||
|
|
"rewards/concensus_correctness_reward_func": 17.70799994468689,
|
||
|
|
"rewards/consensus_reward_func": 1.5,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.25,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6001966744661331,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0818749964237213,
|
||
|
|
"step": 344
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 237.4375,
|
||
|
|
"epoch": 4.407643312101911,
|
||
|
|
"grad_norm": 4.497431755065918,
|
||
|
|
"kl": 3.662317145615816,
|
||
|
|
"learning_rate": 3.872133833281973e-08,
|
||
|
|
"loss": 0.0037,
|
||
|
|
"reward": 18.673893690109253,
|
||
|
|
"reward_std": 5.760366526897997,
|
||
|
|
"rewards/concensus_correctness_reward_func": 14.171375036239624,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.25,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5415192926302552,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func": 1.086000021547079,
|
||
|
|
"step": 348
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 243.5625,
|
||
|
|
"epoch": 4.45859872611465,
|
||
|
|
"grad_norm": 3.805427312850952,
|
||
|
|
"kl": 9.271330177783966,
|
||
|
|
"learning_rate": 3.586894092650992e-08,
|
||
|
|
"loss": 0.0093,
|
||
|
|
"reward": 9.116474032402039,
|
||
|
|
"reward_std": 7.954511460848153,
|
||
|
|
"rewards/concensus_correctness_reward_func": 5.309750065207481,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.625,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5349741466343403,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.1875,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0842500105500221,
|
||
|
|
"step": 352
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 216.0,
|
||
|
|
"epoch": 4.509554140127388,
|
||
|
|
"grad_norm": 1.2827595472335815,
|
||
|
|
"kl": 586.344656933099,
|
||
|
|
"learning_rate": 3.311134618223448e-08,
|
||
|
|
"loss": 0.5863,
|
||
|
|
"reward": 18.384202480316162,
|
||
|
|
"reward_std": 6.0147824686719105,
|
||
|
|
"rewards/concensus_correctness_reward_func": 13.971250057220459,
|
||
|
|
"rewards/consensus_reward_func": 1.125,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.0,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6950150849297643,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.375,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2179375290870667,
|
||
|
|
"step": 356
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 238.25,
|
||
|
|
"epoch": 4.560509554140127,
|
||
|
|
"grad_norm": 2.5090322494506836,
|
||
|
|
"kl": 14.409381460398436,
|
||
|
|
"learning_rate": 3.045084468876636e-08,
|
||
|
|
"loss": 0.0144,
|
||
|
|
"reward": 11.096249029040337,
|
||
|
|
"reward_std": 5.002050343900919,
|
||
|
|
"rewards/concensus_correctness_reward_func": 7.652750000357628,
|
||
|
|
"rewards/consensus_reward_func": 1.0,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.625,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6521867662668228,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.1875,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9788125269114971,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 231.6875,
|
||
|
|
"epoch": 4.611464968152866,
|
||
|
|
"grad_norm": 5.940495014190674,
|
||
|
|
"kl": 5.444742456078529,
|
||
|
|
"learning_rate": 2.7889646384626974e-08,
|
||
|
|
"loss": 0.0054,
|
||
|
|
"reward": 16.203705072402954,
|
||
|
|
"reward_std": 6.409488521516323,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.46025002002716,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.125,
|
||
|
|
"rewards/question_recreation_reward_func": 0.7280802987515926,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.34375,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1716250255703926,
|
||
|
|
"step": 364
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 261.9375,
|
||
|
|
"epoch": 4.662420382165605,
|
||
|
|
"grad_norm": 4.413754463195801,
|
||
|
|
"kl": 40.07878900319338,
|
||
|
|
"learning_rate": 2.5429878722406294e-08,
|
||
|
|
"loss": 0.0401,
|
||
|
|
"reward": 12.807837784290314,
|
||
|
|
"reward_std": 5.907793074846268,
|
||
|
|
"rewards/concensus_correctness_reward_func": 9.157750010490417,
|
||
|
|
"rewards/consensus_reward_func": 1.25,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.625,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5562750669196248,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.1875,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0313125103712082,
|
||
|
|
"step": 368
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 246.5625,
|
||
|
|
"epoch": 4.713375796178344,
|
||
|
|
"grad_norm": 2.676518201828003,
|
||
|
|
"kl": 193.69527792930603,
|
||
|
|
"learning_rate": 2.3073584901600297e-08,
|
||
|
|
"loss": 0.1937,
|
||
|
|
"reward": 15.810337275266647,
|
||
|
|
"reward_std": 7.08709304407239,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.445250004529953,
|
||
|
|
"rewards/consensus_reward_func": 0.875,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.5,
|
||
|
|
"rewards/question_recreation_reward_func": 0.7263994300737977,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.1875,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0761875212192535,
|
||
|
|
"step": 372
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 220.375,
|
||
|
|
"epoch": 4.764331210191083,
|
||
|
|
"grad_norm": 15.561013221740723,
|
||
|
|
"kl": 10.431547999382019,
|
||
|
|
"learning_rate": 2.0822722171432373e-08,
|
||
|
|
"loss": 0.0104,
|
||
|
|
"reward": 16.888904064893723,
|
||
|
|
"reward_std": 4.636547790840268,
|
||
|
|
"rewards/concensus_correctness_reward_func": 12.990749955177307,
|
||
|
|
"rewards/consensus_reward_func": 1.25,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.0,
|
||
|
|
"rewards/question_recreation_reward_func": 0.42490432411432266,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0982500240206718,
|
||
|
|
"step": 376
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 284.375,
|
||
|
|
"epoch": 4.8152866242038215,
|
||
|
|
"grad_norm": 2.221562385559082,
|
||
|
|
"kl": 28.070011030882597,
|
||
|
|
"learning_rate": 1.867916020506981e-08,
|
||
|
|
"loss": 0.0281,
|
||
|
|
"reward": 17.162397533655167,
|
||
|
|
"reward_std": 4.522519879043102,
|
||
|
|
"rewards/concensus_correctness_reward_func": 12.688500046730042,
|
||
|
|
"rewards/consensus_reward_func": 1.5,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.0,
|
||
|
|
"rewards/question_recreation_reward_func": 0.600835133343935,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.3125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0605625286698341,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 255.9375,
|
||
|
|
"epoch": 4.86624203821656,
|
||
|
|
"grad_norm": 3.4525911808013916,
|
||
|
|
"kl": 13.200175400823355,
|
||
|
|
"learning_rate": 1.6644679546584393e-08,
|
||
|
|
"loss": 0.0132,
|
||
|
|
"reward": 20.264204382896423,
|
||
|
|
"reward_std": 3.887154497206211,
|
||
|
|
"rewards/concensus_correctness_reward_func": 15.187625050544739,
|
||
|
|
"rewards/consensus_reward_func": 1.625,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.25,
|
||
|
|
"rewards/question_recreation_reward_func": 0.729454442858696,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.3125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.159624993801117,
|
||
|
|
"step": 384
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 250.0625,
|
||
|
|
"epoch": 4.917197452229299,
|
||
|
|
"grad_norm": 1.8867841958999634,
|
||
|
|
"kl": 8.001008473336697,
|
||
|
|
"learning_rate": 1.4720970131948435e-08,
|
||
|
|
"loss": 0.008,
|
||
|
|
"reward": 18.74275043606758,
|
||
|
|
"reward_std": 2.8640489764511585,
|
||
|
|
"rewards/concensus_correctness_reward_func": 13.75,
|
||
|
|
"rewards/consensus_reward_func": 1.625,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.375,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6746251694858074,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0681250244379044,
|
||
|
|
"step": 388
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 244.5,
|
||
|
|
"epoch": 4.968152866242038,
|
||
|
|
"grad_norm": 3.219937801361084,
|
||
|
|
"kl": 2.38198983669281,
|
||
|
|
"learning_rate": 1.290962988529376e-08,
|
||
|
|
"loss": 0.0024,
|
||
|
|
"reward": 15.04402369260788,
|
||
|
|
"reward_std": 4.118689671158791,
|
||
|
|
"rewards/concensus_correctness_reward_func": 10.858500003814697,
|
||
|
|
"rewards/consensus_reward_func": 1.25,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.75,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6415863204747438,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.34375,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2001875191926956,
|
||
|
|
"step": 392
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 240.0,
|
||
|
|
"epoch": 5.012738853503185,
|
||
|
|
"grad_norm": 3.246076822280884,
|
||
|
|
"kl": 16.680181052003586,
|
||
|
|
"learning_rate": 1.1212163391600377e-08,
|
||
|
|
"loss": 0.0146,
|
||
|
|
"reward": 14.724290439060756,
|
||
|
|
"reward_std": 6.1574646064213345,
|
||
|
|
"rewards/concensus_correctness_reward_func": 10.47371428353446,
|
||
|
|
"rewards/consensus_reward_func": 1.2857142857142858,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.2857142857142858,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5896476145301547,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.17857142857142858,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9109285984720502,
|
||
|
|
"step": 396
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 235.25,
|
||
|
|
"epoch": 5.063694267515924,
|
||
|
|
"grad_norm": 2.46883487701416,
|
||
|
|
"kl": 2.3545443788170815,
|
||
|
|
"learning_rate": 9.629980646916868e-09,
|
||
|
|
"loss": 0.0024,
|
||
|
|
"reward": 16.084708213806152,
|
||
|
|
"reward_std": 8.817421786487103,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.687125086784363,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.125,
|
||
|
|
"rewards/question_recreation_reward_func": 0.4455205723643303,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2020625323057175,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 247.125,
|
||
|
|
"epoch": 5.114649681528663,
|
||
|
|
"grad_norm": 2.4745118618011475,
|
||
|
|
"kl": 2.9937894009053707,
|
||
|
|
"learning_rate": 8.164395887150988e-09,
|
||
|
|
"loss": 0.003,
|
||
|
|
"reward": 14.124228060245514,
|
||
|
|
"reward_std": 9.18977701291442,
|
||
|
|
"rewards/concensus_correctness_reward_func": 9.378499984741211,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.125,
|
||
|
|
"rewards/question_recreation_reward_func": 0.720603097230196,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.3125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2126250267028809,
|
||
|
|
"step": 404
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 250.1875,
|
||
|
|
"epoch": 5.165605095541402,
|
||
|
|
"grad_norm": 2.0114660263061523,
|
||
|
|
"kl": 21.8758534938097,
|
||
|
|
"learning_rate": 6.816626496402977e-09,
|
||
|
|
"loss": 0.0219,
|
||
|
|
"reward": 13.668376445770264,
|
||
|
|
"reward_std": 2.5443186461925507,
|
||
|
|
"rewards/concensus_correctness_reward_func": 9.617874965071678,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.75,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5688141155987978,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.28125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0754375234246254,
|
||
|
|
"step": 408
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 241.625,
|
||
|
|
"epoch": 5.2165605095541405,
|
||
|
|
"grad_norm": 2.3374829292297363,
|
||
|
|
"kl": 3.71017612144351,
|
||
|
|
"learning_rate": 5.5877919957487805e-09,
|
||
|
|
"loss": 0.0037,
|
||
|
|
"reward": 19.984151780605316,
|
||
|
|
"reward_std": 4.235110512934625,
|
||
|
|
"rewards/concensus_correctness_reward_func": 15.0,
|
||
|
|
"rewards/consensus_reward_func": 1.75,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.125,
|
||
|
|
"rewards/question_recreation_reward_func": 0.8057144843041897,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.21875,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0846875384449959,
|
||
|
|
"step": 412
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 218.5,
|
||
|
|
"epoch": 5.267515923566879,
|
||
|
|
"grad_norm": 1151.6732177734375,
|
||
|
|
"kl": 297.7076738625765,
|
||
|
|
"learning_rate": 4.478913113312771e-09,
|
||
|
|
"loss": 0.2977,
|
||
|
|
"reward": 13.16609039902687,
|
||
|
|
"reward_std": 9.443016417324543,
|
||
|
|
"rewards/concensus_correctness_reward_func": 8.948874950408936,
|
||
|
|
"rewards/consensus_reward_func": 0.875,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.125,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6226529218256474,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.40625,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1883125305175781,
|
||
|
|
"step": 416
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 222.9375,
|
||
|
|
"epoch": 5.318471337579618,
|
||
|
|
"grad_norm": 2.7844438552856445,
|
||
|
|
"kl": 9.570182114839554,
|
||
|
|
"learning_rate": 3.490910936402691e-09,
|
||
|
|
"loss": 0.0096,
|
||
|
|
"reward": 18.053339332342148,
|
||
|
|
"reward_std": 6.633316494524479,
|
||
|
|
"rewards/concensus_correctness_reward_func": 13.75,
|
||
|
|
"rewards/consensus_reward_func": 1.5,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.75,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6550895310938358,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.1875,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2107500284910202,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 213.75,
|
||
|
|
"epoch": 5.369426751592357,
|
||
|
|
"grad_norm": 11.249397277832031,
|
||
|
|
"kl": 17.067561771720648,
|
||
|
|
"learning_rate": 2.6246061464108014e-09,
|
||
|
|
"loss": 0.0171,
|
||
|
|
"reward": 17.524891823530197,
|
||
|
|
"reward_std": 4.37577617354691,
|
||
|
|
"rewards/concensus_correctness_reward_func": 12.710250005126,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.375,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6043295841664076,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2103125154972076,
|
||
|
|
"step": 424
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 209.3125,
|
||
|
|
"epoch": 5.420382165605096,
|
||
|
|
"grad_norm": 3.837719202041626,
|
||
|
|
"kl": 2.012119587510824,
|
||
|
|
"learning_rate": 1.8807183371172484e-09,
|
||
|
|
"loss": 0.002,
|
||
|
|
"reward": 17.180835396051407,
|
||
|
|
"reward_std": 4.515378940850496,
|
||
|
|
"rewards/concensus_correctness_reward_func": 12.949124962091446,
|
||
|
|
"rewards/consensus_reward_func": 0.75,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.375,
|
||
|
|
"rewards/question_recreation_reward_func": 0.5834604157134891,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.3125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.210750013589859,
|
||
|
|
"step": 428
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 205.6875,
|
||
|
|
"epoch": 5.471337579617835,
|
||
|
|
"grad_norm": 2.5619990825653076,
|
||
|
|
"kl": 13.116049729287624,
|
||
|
|
"learning_rate": 1.2598654169611977e-09,
|
||
|
|
"loss": 0.0131,
|
||
|
|
"reward": 18.689254820346832,
|
||
|
|
"reward_std": 5.8718895125202835,
|
||
|
|
"rewards/concensus_correctness_reward_func": 13.973749995231628,
|
||
|
|
"rewards/consensus_reward_func": 1.625,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.125,
|
||
|
|
"rewards/question_recreation_reward_func": 0.4517549993470311,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.28125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2325000166893005,
|
||
|
|
"step": 432
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 221.4375,
|
||
|
|
"epoch": 5.522292993630574,
|
||
|
|
"grad_norm": 4.5784711837768555,
|
||
|
|
"kl": 13.171938437968493,
|
||
|
|
"learning_rate": 7.625630957769357e-10,
|
||
|
|
"loss": 0.0132,
|
||
|
|
"reward": 12.938094705343246,
|
||
|
|
"reward_std": 9.368400201201439,
|
||
|
|
"rewards/concensus_correctness_reward_func": 9.259250115603209,
|
||
|
|
"rewards/consensus_reward_func": 0.875,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.625,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6295947674661875,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.34375,
|
||
|
|
"rewards/xmlcount_reward_func": 1.2055000215768814,
|
||
|
|
"step": 436
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 235.4375,
|
||
|
|
"epoch": 5.573248407643312,
|
||
|
|
"grad_norm": 16.68247413635254,
|
||
|
|
"kl": 12.497772630304098,
|
||
|
|
"learning_rate": 3.8922445642086687e-10,
|
||
|
|
"loss": 0.0125,
|
||
|
|
"reward": 15.425481706857681,
|
||
|
|
"reward_std": 2.6193624921143055,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.479375004768372,
|
||
|
|
"rewards/consensus_reward_func": 1.375,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 0.75,
|
||
|
|
"rewards/question_recreation_reward_func": 0.4780444288626313,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.15625,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1868125051259995,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 259.8125,
|
||
|
|
"epoch": 5.624203821656051,
|
||
|
|
"grad_norm": 2.506514549255371,
|
||
|
|
"kl": 2.1330077573657036,
|
||
|
|
"learning_rate": 1.4015961164528522e-10,
|
||
|
|
"loss": 0.0023,
|
||
|
|
"reward": 20.30971536040306,
|
||
|
|
"reward_std": 0.7802602792944526,
|
||
|
|
"rewards/concensus_correctness_reward_func": 15.0,
|
||
|
|
"rewards/consensus_reward_func": 1.875,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.375,
|
||
|
|
"rewards/question_recreation_reward_func": 0.6319650914520025,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.3125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.1152500286698341,
|
||
|
|
"step": 444
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 213.75,
|
||
|
|
"epoch": 5.67515923566879,
|
||
|
|
"grad_norm": 1.8979109525680542,
|
||
|
|
"kl": 5.903007443994284,
|
||
|
|
"learning_rate": 1.5575446504029198e-11,
|
||
|
|
"loss": 0.0059,
|
||
|
|
"reward": 15.859139263629913,
|
||
|
|
"reward_std": 3.203619036823511,
|
||
|
|
"rewards/concensus_correctness_reward_func": 11.463249996304512,
|
||
|
|
"rewards/consensus_reward_func": 1.25,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.125,
|
||
|
|
"rewards/question_recreation_reward_func": 0.637013990432024,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.3125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0713750198483467,
|
||
|
|
"step": 448
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 231.25,
|
||
|
|
"epoch": 5.7006369426751595,
|
||
|
|
"kl": 1.2166772335767746,
|
||
|
|
"reward": 22.533211946487427,
|
||
|
|
"reward_std": 5.739101491868496,
|
||
|
|
"rewards/concensus_correctness_reward_func": 17.5,
|
||
|
|
"rewards/consensus_reward_func": 1.75,
|
||
|
|
"rewards/cumulative_reward_2": 0.0,
|
||
|
|
"rewards/final_correctness_reward_func": 1.25,
|
||
|
|
"rewards/question_recreation_reward_func": 0.7859619110822678,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.25,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9972500205039978,
|
||
|
|
"step": 450,
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_loss": 0.03514481741314133,
|
||
|
|
"train_runtime": 2046.9514,
|
||
|
|
"train_samples_per_second": 0.879,
|
||
|
|
"train_steps_per_second": 0.22
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 4,
|
||
|
|
"max_steps": 450,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 6,
|
||
|
|
"save_steps": 40,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": true
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 2,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|