4834 lines
163 KiB
JSON
4834 lines
163 KiB
JSON
|
{
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 0.04014452027298274,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 300,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"completion_length": 247.375,
|
||
|
|
"epoch": 0.00013381506757660912,
|
||
|
|
"grad_norm": 0.43693211674690247,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 5.0000000000000004e-08,
|
||
|
|
"loss": -0.0,
|
||
|
|
"reward": 0.04712499678134918,
|
||
|
|
"reward_std": 1.0776702165603638,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.0625,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.2653749883174896,
|
||
|
|
"step": 1
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 299.125,
|
||
|
|
"epoch": 0.00026763013515321824,
|
||
|
|
"grad_norm": 0.8330063223838806,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 1.0000000000000001e-07,
|
||
|
|
"loss": -0.0,
|
||
|
|
"reward": -0.1561249941587448,
|
||
|
|
"reward_std": 0.6402567625045776,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.0,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.1561249941587448,
|
||
|
|
"step": 2
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 280.5,
|
||
|
|
"epoch": 0.0004014452027298274,
|
||
|
|
"grad_norm": 0.4297165274620056,
|
||
|
|
"kl": 5.41932513442589e-06,
|
||
|
|
"learning_rate": 1.5000000000000002e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": 0.1876250058412552,
|
||
|
|
"reward_std": 0.9614257216453552,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.0,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.312375009059906,
|
||
|
|
"step": 3
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 261.75,
|
||
|
|
"epoch": 0.0005352602703064365,
|
||
|
|
"grad_norm": 0.41665294766426086,
|
||
|
|
"kl": 5.1015522330999374e-06,
|
||
|
|
"learning_rate": 2.0000000000000002e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": 0.593874990940094,
|
||
|
|
"reward_std": 0.9993927478790283,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.0625,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.468625009059906,
|
||
|
|
"step": 4
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 128.25,
|
||
|
|
"epoch": 0.0006690753378830456,
|
||
|
|
"grad_norm": 0.6423615217208862,
|
||
|
|
"kl": 2.9761447422060883e-06,
|
||
|
|
"learning_rate": 2.5000000000000004e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": -0.04662501811981201,
|
||
|
|
"reward_std": 0.7257494926452637,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.421625018119812,
|
||
|
|
"step": 5
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 269.375,
|
||
|
|
"epoch": 0.0008028904054596548,
|
||
|
|
"grad_norm": 0.6059211492538452,
|
||
|
|
"kl": 4.0267409531224985e-06,
|
||
|
|
"learning_rate": 3.0000000000000004e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": 1.190999984741211,
|
||
|
|
"reward_std": 1.1969025135040283,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.43400001525878906,
|
||
|
|
"step": 6
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 275.875,
|
||
|
|
"epoch": 0.0009367054730362638,
|
||
|
|
"grad_norm": 0.36298441886901855,
|
||
|
|
"kl": 8.828338650346268e-06,
|
||
|
|
"learning_rate": 3.5000000000000004e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": 0.09387499839067459,
|
||
|
|
"reward_std": 0.9057357907295227,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.0625,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.468625009059906,
|
||
|
|
"step": 7
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 208.875,
|
||
|
|
"epoch": 0.001070520540612873,
|
||
|
|
"grad_norm": 0.5650742053985596,
|
||
|
|
"kl": 9.147981472779065e-06,
|
||
|
|
"learning_rate": 4.0000000000000003e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": 0.15625,
|
||
|
|
"reward_std": 0.9348175525665283,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.0,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.34375,
|
||
|
|
"step": 8
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 203.125,
|
||
|
|
"epoch": 0.0012043356081894822,
|
||
|
|
"grad_norm": 0.6047148704528809,
|
||
|
|
"kl": 1.0332691999792587e-05,
|
||
|
|
"learning_rate": 4.5000000000000003e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": 0.359624981880188,
|
||
|
|
"reward_std": 1.2167718410491943,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.265375018119812,
|
||
|
|
"step": 9
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 219.625,
|
||
|
|
"epoch": 0.0013381506757660913,
|
||
|
|
"grad_norm": 0.0003068444784730673,
|
||
|
|
"kl": 8.656875252199825e-06,
|
||
|
|
"learning_rate": 5.000000000000001e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": 1.5,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.0,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.5,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 168.375,
|
||
|
|
"epoch": 0.0014719657433427003,
|
||
|
|
"grad_norm": 0.47508102655410767,
|
||
|
|
"kl": 5.768194114352809e-06,
|
||
|
|
"learning_rate": 5.5e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": 1.2501249313354492,
|
||
|
|
"reward_std": 1.281990647315979,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.0625,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.312375009059906,
|
||
|
|
"step": 11
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 254.75,
|
||
|
|
"epoch": 0.0016057808109193096,
|
||
|
|
"grad_norm": 0.4676521420478821,
|
||
|
|
"kl": 8.639064617455006e-06,
|
||
|
|
"learning_rate": 6.000000000000001e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": 0.0,
|
||
|
|
"reward_std": 0.9258201122283936,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.0,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.5,
|
||
|
|
"step": 12
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 241.25,
|
||
|
|
"epoch": 0.0017395958784959186,
|
||
|
|
"grad_norm": 0.44435447454452515,
|
||
|
|
"kl": 9.67886717262445e-06,
|
||
|
|
"learning_rate": 6.5e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": 0.5,
|
||
|
|
"reward_std": 1.0690449476242065,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.0,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.5,
|
||
|
|
"step": 13
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 225.875,
|
||
|
|
"epoch": 0.0018734109460725277,
|
||
|
|
"grad_norm": 0.4902050495147705,
|
||
|
|
"kl": 1.7060545360436663e-05,
|
||
|
|
"learning_rate": 7.000000000000001e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": 0.437624990940094,
|
||
|
|
"reward_std": 1.0155839920043945,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.0,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.312375009059906,
|
||
|
|
"step": 14
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 297.625,
|
||
|
|
"epoch": 0.002007226013649137,
|
||
|
|
"grad_norm": 0.3076633810997009,
|
||
|
|
"kl": 1.0535066394368187e-05,
|
||
|
|
"learning_rate": 7.5e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": 1.0938749313354492,
|
||
|
|
"reward_std": 0.7783224582672119,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.0625,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.468625009059906,
|
||
|
|
"step": 15
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 241.625,
|
||
|
|
"epoch": 0.002141041081225746,
|
||
|
|
"grad_norm": 0.44562768936157227,
|
||
|
|
"kl": 1.70583080034703e-05,
|
||
|
|
"learning_rate": 8.000000000000001e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": 0.562874972820282,
|
||
|
|
"reward_std": 1.1241743564605713,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.0625,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.24962499737739563,
|
||
|
|
"step": 16
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 239.375,
|
||
|
|
"epoch": 0.002274856148802355,
|
||
|
|
"grad_norm": 0.32838791608810425,
|
||
|
|
"kl": 2.1659421690856107e-05,
|
||
|
|
"learning_rate": 8.500000000000001e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": 1.5001249313354492,
|
||
|
|
"reward_std": 1.0693122148513794,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.0625,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.312375009059906,
|
||
|
|
"step": 17
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 209.625,
|
||
|
|
"epoch": 0.0024086712163789645,
|
||
|
|
"grad_norm": 0.5566530227661133,
|
||
|
|
"kl": 9.372454223921522e-05,
|
||
|
|
"learning_rate": 9.000000000000001e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": 1.8756250143051147,
|
||
|
|
"reward_std": 1.6641279458999634,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.1875,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.43812501430511475,
|
||
|
|
"step": 18
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 341.75,
|
||
|
|
"epoch": 0.0025424862839555735,
|
||
|
|
"grad_norm": 0.33729103207588196,
|
||
|
|
"kl": 3.0826518923277035e-05,
|
||
|
|
"learning_rate": 9.500000000000001e-07,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": -0.328000009059906,
|
||
|
|
"reward_std": 0.4864894151687622,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.0,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.328000009059906,
|
||
|
|
"step": 19
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 320.25,
|
||
|
|
"epoch": 0.0026763013515321826,
|
||
|
|
"grad_norm": 0.5381105542182922,
|
||
|
|
"kl": 9.824618609854952e-05,
|
||
|
|
"learning_rate": 1.0000000000000002e-06,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": 0.12524999678134918,
|
||
|
|
"reward_std": 0.8765599727630615,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.0,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.12474998831748962,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 324.0,
|
||
|
|
"epoch": 0.0028101164191087916,
|
||
|
|
"grad_norm": 0.36464422941207886,
|
||
|
|
"kl": 8.919827814679593e-05,
|
||
|
|
"learning_rate": 1.0500000000000001e-06,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": 1.3910000324249268,
|
||
|
|
"reward_std": 0.885222315788269,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.23399999737739563,
|
||
|
|
"step": 21
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 103.125,
|
||
|
|
"epoch": 0.0029439314866854006,
|
||
|
|
"grad_norm": 1.0258393287658691,
|
||
|
|
"kl": 0.0007803550106473267,
|
||
|
|
"learning_rate": 1.1e-06,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": 0.041125014424324036,
|
||
|
|
"reward_std": 0.7970256805419922,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.0,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.041125014424324036,
|
||
|
|
"step": 22
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 199.25,
|
||
|
|
"epoch": 0.00307774655426201,
|
||
|
|
"grad_norm": 0.5802411437034607,
|
||
|
|
"kl": 0.0005780195351690054,
|
||
|
|
"learning_rate": 1.1500000000000002e-06,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": 0.6317499876022339,
|
||
|
|
"reward_std": 1.0433696508407593,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.2432500123977661,
|
||
|
|
"step": 23
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 234.5,
|
||
|
|
"epoch": 0.003211561621838619,
|
||
|
|
"grad_norm": 0.7017332911491394,
|
||
|
|
"kl": 0.0006519687594845891,
|
||
|
|
"learning_rate": 1.2000000000000002e-06,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": -0.16749998927116394,
|
||
|
|
"reward_std": 0.6224183440208435,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.0,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.16749998927116394,
|
||
|
|
"step": 24
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 214.0,
|
||
|
|
"epoch": 0.003345376689415228,
|
||
|
|
"grad_norm": 0.5245077013969421,
|
||
|
|
"kl": 0.0009261792874895036,
|
||
|
|
"learning_rate": 1.25e-06,
|
||
|
|
"loss": 0.0,
|
||
|
|
"reward": 1.266124963760376,
|
||
|
|
"reward_std": 1.027535080909729,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.0625,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.04637499153614044,
|
||
|
|
"step": 25
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 272.625,
|
||
|
|
"epoch": 0.0034791917569918372,
|
||
|
|
"grad_norm": 0.735785186290741,
|
||
|
|
"kl": 0.0016067801043391228,
|
||
|
|
"learning_rate": 1.3e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"reward": -0.12462500482797623,
|
||
|
|
"reward_std": 0.7074856758117676,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.24962501227855682,
|
||
|
|
"step": 26
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 198.625,
|
||
|
|
"epoch": 0.0036130068245684463,
|
||
|
|
"grad_norm": 0.8707014918327332,
|
||
|
|
"kl": 0.003043584758415818,
|
||
|
|
"learning_rate": 1.3500000000000002e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"reward": 2.0633749961853027,
|
||
|
|
"reward_std": 1.5223981142044067,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.25,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8133749961853027,
|
||
|
|
"step": 27
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 170.25,
|
||
|
|
"epoch": 0.0037468218921450553,
|
||
|
|
"grad_norm": 0.7808597683906555,
|
||
|
|
"kl": 0.0034616258926689625,
|
||
|
|
"learning_rate": 1.4000000000000001e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"reward": 1.0472500324249268,
|
||
|
|
"reward_std": 1.5982364416122437,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": -0.07774998992681503,
|
||
|
|
"step": 28
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 181.5,
|
||
|
|
"epoch": 0.003880636959721665,
|
||
|
|
"grad_norm": 0.37707751989364624,
|
||
|
|
"kl": 0.008707987144589424,
|
||
|
|
"learning_rate": 1.45e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"reward": 0.8757500648498535,
|
||
|
|
"reward_std": 0.5826664566993713,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.0,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.6257500648498535,
|
||
|
|
"step": 29
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 191.625,
|
||
|
|
"epoch": 0.004014452027298274,
|
||
|
|
"grad_norm": 0.6155921220779419,
|
||
|
|
"kl": 0.009353982284665108,
|
||
|
|
"learning_rate": 1.5e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"reward": 1.4383749961853027,
|
||
|
|
"reward_std": 1.4438849687576294,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.1875,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.5008749961853027,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 112.875,
|
||
|
|
"epoch": 0.004148267094874883,
|
||
|
|
"grad_norm": 0.8173586130142212,
|
||
|
|
"kl": 0.012922466732561588,
|
||
|
|
"learning_rate": 1.5500000000000002e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"reward": 1.7816250324249268,
|
||
|
|
"reward_std": 1.2780566215515137,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.0625,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.21912501752376556,
|
||
|
|
"step": 31
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 313.375,
|
||
|
|
"epoch": 0.004282082162451492,
|
||
|
|
"grad_norm": 0.46694669127464294,
|
||
|
|
"kl": 0.009532835334539413,
|
||
|
|
"learning_rate": 1.6000000000000001e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"reward": 1.1568750143051147,
|
||
|
|
"reward_std": 1.2886663675308228,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.3125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.5943750143051147,
|
||
|
|
"step": 32
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 176.125,
|
||
|
|
"epoch": 0.004415897230028101,
|
||
|
|
"grad_norm": 0.6538305282592773,
|
||
|
|
"kl": 0.012952692806720734,
|
||
|
|
"learning_rate": 1.6500000000000003e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"reward": 2.813499927520752,
|
||
|
|
"reward_std": 0.9977653622627258,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.3125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 33
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 171.5,
|
||
|
|
"epoch": 0.00454971229760471,
|
||
|
|
"grad_norm": 0.5880665183067322,
|
||
|
|
"kl": 0.01218588650226593,
|
||
|
|
"learning_rate": 1.7000000000000002e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"reward": 2.125999927520752,
|
||
|
|
"reward_std": 1.1572750806808472,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.3125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0625,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 34
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 244.5,
|
||
|
|
"epoch": 0.004683527365181319,
|
||
|
|
"grad_norm": 0.6010124087333679,
|
||
|
|
"kl": 0.011459747329354286,
|
||
|
|
"learning_rate": 1.75e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"reward": 2.1167500019073486,
|
||
|
|
"reward_std": 0.9670493006706238,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9917500019073486,
|
||
|
|
"step": 35
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 170.625,
|
||
|
|
"epoch": 0.004817342432757929,
|
||
|
|
"grad_norm": 0.5059515833854675,
|
||
|
|
"kl": 0.039504192769527435,
|
||
|
|
"learning_rate": 1.8000000000000001e-06,
|
||
|
|
"loss": 0.0016,
|
||
|
|
"reward": 0.6885000467300415,
|
||
|
|
"reward_std": 0.5786375403404236,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.0,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.6884999871253967,
|
||
|
|
"step": 36
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 242.25,
|
||
|
|
"epoch": 0.004951157500334538,
|
||
|
|
"grad_norm": 0.4785184860229492,
|
||
|
|
"kl": 0.01279086247086525,
|
||
|
|
"learning_rate": 1.85e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"reward": 1.5764999389648438,
|
||
|
|
"reward_std": 0.9564229249954224,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0625,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8265000581741333,
|
||
|
|
"step": 37
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 127.125,
|
||
|
|
"epoch": 0.005084972567911147,
|
||
|
|
"grad_norm": 1.1302545070648193,
|
||
|
|
"kl": 0.02101985551416874,
|
||
|
|
"learning_rate": 1.9000000000000002e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 1.1102499961853027,
|
||
|
|
"reward_std": 0.20507541298866272,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9852499961853027,
|
||
|
|
"step": 38
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 219.25,
|
||
|
|
"epoch": 0.005218787635487756,
|
||
|
|
"grad_norm": 0.3671993017196655,
|
||
|
|
"kl": 0.018780428916215897,
|
||
|
|
"learning_rate": 1.9500000000000004e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 1.3914999961853027,
|
||
|
|
"reward_std": 0.8697662949562073,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.1875,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9539999961853027,
|
||
|
|
"step": 39
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 239.875,
|
||
|
|
"epoch": 0.005352602703064365,
|
||
|
|
"grad_norm": 0.43140241503715515,
|
||
|
|
"kl": 0.011521664448082447,
|
||
|
|
"learning_rate": 2.0000000000000003e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"reward": 1.813499927520752,
|
||
|
|
"reward_std": 0.9234058856964111,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.3125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 208.0,
|
||
|
|
"epoch": 0.005486417770640974,
|
||
|
|
"grad_norm": 0.639629602432251,
|
||
|
|
"kl": 0.008469691500067711,
|
||
|
|
"learning_rate": 2.05e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"reward": 2.625999927520752,
|
||
|
|
"reward_std": 0.9543134570121765,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 41
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 165.0,
|
||
|
|
"epoch": 0.005620232838217583,
|
||
|
|
"grad_norm": 0.8204224109649658,
|
||
|
|
"kl": 0.021016664803028107,
|
||
|
|
"learning_rate": 2.1000000000000002e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 1.7762500047683716,
|
||
|
|
"reward_std": 1.1011048555374146,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.3125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9637500047683716,
|
||
|
|
"step": 42
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 204.625,
|
||
|
|
"epoch": 0.005754047905794192,
|
||
|
|
"grad_norm": 0.008070758543908596,
|
||
|
|
"kl": 0.01050515379756689,
|
||
|
|
"learning_rate": 2.15e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"reward": 1.0010000467300415,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.0,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 43
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 245.625,
|
||
|
|
"epoch": 0.005887862973370801,
|
||
|
|
"grad_norm": 0.7679343223571777,
|
||
|
|
"kl": 0.012935507111251354,
|
||
|
|
"learning_rate": 2.2e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"reward": 2.688499927520752,
|
||
|
|
"reward_std": 0.9234058856964111,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.1875,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 44
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 254.0,
|
||
|
|
"epoch": 0.00602167804094741,
|
||
|
|
"grad_norm": 0.615129828453064,
|
||
|
|
"kl": 0.01200153212994337,
|
||
|
|
"learning_rate": 2.25e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"reward": 0.9628750085830688,
|
||
|
|
"reward_std": 0.355848491191864,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.0625,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9003750085830688,
|
||
|
|
"step": 45
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 92.25,
|
||
|
|
"epoch": 0.00615549310852402,
|
||
|
|
"grad_norm": 0.7606546878814697,
|
||
|
|
"kl": 0.03429767116904259,
|
||
|
|
"learning_rate": 2.3000000000000004e-06,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"reward": 1.9192500114440918,
|
||
|
|
"reward_std": 1.267798662185669,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.3125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8567500710487366,
|
||
|
|
"step": 46
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 145.25,
|
||
|
|
"epoch": 0.006289308176100629,
|
||
|
|
"grad_norm": 0.49140965938568115,
|
||
|
|
"kl": 0.022705648094415665,
|
||
|
|
"learning_rate": 2.35e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 3.3002500534057617,
|
||
|
|
"reward_std": 0.27898576855659485,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9252500534057617,
|
||
|
|
"step": 47
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 212.0,
|
||
|
|
"epoch": 0.006423123243677238,
|
||
|
|
"grad_norm": 0.5346264839172363,
|
||
|
|
"kl": 0.01612384431064129,
|
||
|
|
"learning_rate": 2.4000000000000003e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"reward": 2.1816248893737793,
|
||
|
|
"reward_std": 0.9994299411773682,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.3125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8691250681877136,
|
||
|
|
"step": 48
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 203.75,
|
||
|
|
"epoch": 0.006556938311253847,
|
||
|
|
"grad_norm": 0.45523107051849365,
|
||
|
|
"kl": 0.010947800241410732,
|
||
|
|
"learning_rate": 2.4500000000000003e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"reward": 1.500999927520752,
|
||
|
|
"reward_std": 0.6546536087989807,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.25,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 49
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 236.25,
|
||
|
|
"epoch": 0.006690753378830456,
|
||
|
|
"grad_norm": 0.5265190601348877,
|
||
|
|
"kl": 0.023704925552010536,
|
||
|
|
"learning_rate": 2.5e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 2.2196249961853027,
|
||
|
|
"reward_std": 1.012678623199463,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.0,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9696249961853027,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 171.125,
|
||
|
|
"epoch": 0.006824568446407065,
|
||
|
|
"grad_norm": 0.8060426712036133,
|
||
|
|
"kl": 0.01678396947681904,
|
||
|
|
"learning_rate": 2.55e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"reward": 2.375999927520752,
|
||
|
|
"reward_std": 1.2174328565597534,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 51
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 422.125,
|
||
|
|
"epoch": 0.0069583835139836745,
|
||
|
|
"grad_norm": 0.41455376148223877,
|
||
|
|
"kl": 0.00810204166918993,
|
||
|
|
"learning_rate": 2.6e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"reward": 0.9853750467300415,
|
||
|
|
"reward_std": 0.5019493699073792,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8603750467300415,
|
||
|
|
"step": 52
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 206.5,
|
||
|
|
"epoch": 0.0070921985815602835,
|
||
|
|
"grad_norm": 0.6518821716308594,
|
||
|
|
"kl": 0.010900408960878849,
|
||
|
|
"learning_rate": 2.6500000000000005e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"reward": 1.621500015258789,
|
||
|
|
"reward_std": 0.7783805131912231,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9965000152587891,
|
||
|
|
"step": 53
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 179.5,
|
||
|
|
"epoch": 0.0072260136491368926,
|
||
|
|
"grad_norm": 0.6649825572967529,
|
||
|
|
"kl": 0.02075210213661194,
|
||
|
|
"learning_rate": 2.7000000000000004e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 1.790374994277954,
|
||
|
|
"reward_std": 0.9680113196372986,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.0625,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9778749942779541,
|
||
|
|
"step": 54
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 176.375,
|
||
|
|
"epoch": 0.007359828716713502,
|
||
|
|
"grad_norm": 0.37156742811203003,
|
||
|
|
"kl": 0.021078450605273247,
|
||
|
|
"learning_rate": 2.7500000000000004e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 1.313499927520752,
|
||
|
|
"reward_std": 0.8838834166526794,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.0625,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 55
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 218.5,
|
||
|
|
"epoch": 0.007493643784290111,
|
||
|
|
"grad_norm": 0.47643041610717773,
|
||
|
|
"kl": 0.020204594358801842,
|
||
|
|
"learning_rate": 2.8000000000000003e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 2.250999927520752,
|
||
|
|
"reward_std": 1.0350983142852783,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 56
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 226.125,
|
||
|
|
"epoch": 0.0076274588518667205,
|
||
|
|
"grad_norm": 0.495337575674057,
|
||
|
|
"kl": 0.010943291708827019,
|
||
|
|
"learning_rate": 2.85e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"reward": 1.3143749237060547,
|
||
|
|
"reward_std": 0.9964520931243896,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.1875,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8768750429153442,
|
||
|
|
"step": 57
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 155.0,
|
||
|
|
"epoch": 0.00776127391944333,
|
||
|
|
"grad_norm": 0.57661372423172,
|
||
|
|
"kl": 0.024568796157836914,
|
||
|
|
"learning_rate": 2.9e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 1.313499927520752,
|
||
|
|
"reward_std": 0.8838834166526794,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.0625,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 58
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 268.375,
|
||
|
|
"epoch": 0.007895088987019938,
|
||
|
|
"grad_norm": 0.4335046410560608,
|
||
|
|
"kl": 0.014941551722586155,
|
||
|
|
"learning_rate": 2.95e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"reward": 1.813499927520752,
|
||
|
|
"reward_std": 1.0669549703598022,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.3125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 59
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 98.0,
|
||
|
|
"epoch": 0.008028904054596548,
|
||
|
|
"grad_norm": 1.121963381767273,
|
||
|
|
"kl": 0.03911125659942627,
|
||
|
|
"learning_rate": 3e-06,
|
||
|
|
"loss": 0.0016,
|
||
|
|
"reward": 0.905500054359436,
|
||
|
|
"reward_std": 0.17846010625362396,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.0,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.905500054359436,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 202.25,
|
||
|
|
"epoch": 0.008162719122173156,
|
||
|
|
"grad_norm": 0.6488782167434692,
|
||
|
|
"kl": 0.01559341512620449,
|
||
|
|
"learning_rate": 3.05e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"reward": 1.1260000467300415,
|
||
|
|
"reward_std": 0.2314550280570984,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 61
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 234.5,
|
||
|
|
"epoch": 0.008296534189749766,
|
||
|
|
"grad_norm": 0.4507755637168884,
|
||
|
|
"kl": 0.014328347519040108,
|
||
|
|
"learning_rate": 3.1000000000000004e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"reward": 2.438499927520752,
|
||
|
|
"reward_std": 1.0155047178268433,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 62
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 204.375,
|
||
|
|
"epoch": 0.008430349257326376,
|
||
|
|
"grad_norm": 0.44619619846343994,
|
||
|
|
"kl": 0.01781797781586647,
|
||
|
|
"learning_rate": 3.1500000000000003e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"reward": 2.4227499961853027,
|
||
|
|
"reward_std": 1.1080801486968994,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.1875,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9852499961853027,
|
||
|
|
"step": 63
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 216.875,
|
||
|
|
"epoch": 0.008564164324902984,
|
||
|
|
"grad_norm": 0.5863662958145142,
|
||
|
|
"kl": 0.009704262018203735,
|
||
|
|
"learning_rate": 3.2000000000000003e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"reward": 2.1727499961853027,
|
||
|
|
"reward_std": 1.1200695037841797,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9852499961853027,
|
||
|
|
"step": 64
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 214.375,
|
||
|
|
"epoch": 0.008697979392479594,
|
||
|
|
"grad_norm": 0.4384564757347107,
|
||
|
|
"kl": 0.01359130721539259,
|
||
|
|
"learning_rate": 3.2500000000000002e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"reward": 1.625999927520752,
|
||
|
|
"reward_std": 0.7905693650245667,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 65
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 160.625,
|
||
|
|
"epoch": 0.008831794460056202,
|
||
|
|
"grad_norm": 0.6219623684883118,
|
||
|
|
"kl": 0.021640820428729057,
|
||
|
|
"learning_rate": 3.3000000000000006e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 2.438499927520752,
|
||
|
|
"reward_std": 1.1475906372070312,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 66
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 180.875,
|
||
|
|
"epoch": 0.008965609527632812,
|
||
|
|
"grad_norm": 0.4888530969619751,
|
||
|
|
"kl": 0.023110317066311836,
|
||
|
|
"learning_rate": 3.3500000000000005e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 3.000999927520752,
|
||
|
|
"reward_std": 0.9258200526237488,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 67
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 195.0,
|
||
|
|
"epoch": 0.00909942459520942,
|
||
|
|
"grad_norm": 0.4218730628490448,
|
||
|
|
"kl": 0.02685842104256153,
|
||
|
|
"learning_rate": 3.4000000000000005e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"reward": 1.622999906539917,
|
||
|
|
"reward_std": 0.7911567687988281,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9980000853538513,
|
||
|
|
"step": 68
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 172.0,
|
||
|
|
"epoch": 0.00923323966278603,
|
||
|
|
"grad_norm": 0.4772073030471802,
|
||
|
|
"kl": 0.028861787170171738,
|
||
|
|
"learning_rate": 3.45e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"reward": 2.563499927520752,
|
||
|
|
"reward_std": 1.0500850677490234,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.3125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 69
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 149.875,
|
||
|
|
"epoch": 0.009367054730362638,
|
||
|
|
"grad_norm": 0.6557480096817017,
|
||
|
|
"kl": 0.01907029002904892,
|
||
|
|
"learning_rate": 3.5e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 2.241499900817871,
|
||
|
|
"reward_std": 1.3466919660568237,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.25,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9915000200271606,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 147.375,
|
||
|
|
"epoch": 0.009500869797939248,
|
||
|
|
"grad_norm": 0.8631268739700317,
|
||
|
|
"kl": 0.02935231290757656,
|
||
|
|
"learning_rate": 3.5500000000000003e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"reward": 1.665624976158142,
|
||
|
|
"reward_std": 1.180916428565979,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.0625,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8531250357627869,
|
||
|
|
"step": 71
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 193.0,
|
||
|
|
"epoch": 0.009634684865515858,
|
||
|
|
"grad_norm": 0.8472118973731995,
|
||
|
|
"kl": 0.022116979584097862,
|
||
|
|
"learning_rate": 3.6000000000000003e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 2.1202499866485596,
|
||
|
|
"reward_std": 1.1495558023452759,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9952501058578491,
|
||
|
|
"step": 72
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 138.125,
|
||
|
|
"epoch": 0.009768499933092466,
|
||
|
|
"grad_norm": 1.0678242444992065,
|
||
|
|
"kl": 0.03614279255270958,
|
||
|
|
"learning_rate": 3.65e-06,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"reward": 1.6882500648498535,
|
||
|
|
"reward_std": 0.7522825598716736,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.000749945640564,
|
||
|
|
"step": 73
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 189.875,
|
||
|
|
"epoch": 0.009902315000669076,
|
||
|
|
"grad_norm": 0.7392526865005493,
|
||
|
|
"kl": 0.034408796578645706,
|
||
|
|
"learning_rate": 3.7e-06,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"reward": 1.1260000467300415,
|
||
|
|
"reward_std": 0.2314550280570984,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 74
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 185.0,
|
||
|
|
"epoch": 0.010036130068245684,
|
||
|
|
"grad_norm": 0.42845863103866577,
|
||
|
|
"kl": 0.02572033368051052,
|
||
|
|
"learning_rate": 3.7500000000000005e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 1.688499927520752,
|
||
|
|
"reward_std": 0.752970278263092,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 75
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 112.875,
|
||
|
|
"epoch": 0.010169945135822294,
|
||
|
|
"grad_norm": 2.002657890319824,
|
||
|
|
"kl": 0.11890066415071487,
|
||
|
|
"learning_rate": 3.8000000000000005e-06,
|
||
|
|
"loss": 0.0048,
|
||
|
|
"reward": 2.766624927520752,
|
||
|
|
"reward_std": 1.3569226264953613,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8291250467300415,
|
||
|
|
"step": 76
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 233.875,
|
||
|
|
"epoch": 0.010303760203398902,
|
||
|
|
"grad_norm": 0.5970933437347412,
|
||
|
|
"kl": 0.025483284145593643,
|
||
|
|
"learning_rate": 3.85e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 1.4385000467300415,
|
||
|
|
"reward_std": 0.1767766922712326,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 77
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 182.25,
|
||
|
|
"epoch": 0.010437575270975512,
|
||
|
|
"grad_norm": 0.8580113053321838,
|
||
|
|
"kl": 0.028447438031435013,
|
||
|
|
"learning_rate": 3.900000000000001e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"reward": 1.1260000467300415,
|
||
|
|
"reward_std": 0.2314550280570984,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 78
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 159.375,
|
||
|
|
"epoch": 0.01057139033855212,
|
||
|
|
"grad_norm": 0.8452702164649963,
|
||
|
|
"kl": 0.03436105698347092,
|
||
|
|
"learning_rate": 3.95e-06,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"reward": 3.3652501106262207,
|
||
|
|
"reward_std": 0.23232780396938324,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9277499914169312,
|
||
|
|
"step": 79
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 232.0,
|
||
|
|
"epoch": 0.01070520540612873,
|
||
|
|
"grad_norm": 0.3852091133594513,
|
||
|
|
"kl": 0.026189465075731277,
|
||
|
|
"learning_rate": 4.000000000000001e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 2.063499927520752,
|
||
|
|
"reward_std": 1.0835621356964111,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.3125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 126.5,
|
||
|
|
"epoch": 0.010839020473705338,
|
||
|
|
"grad_norm": 1.0608493089675903,
|
||
|
|
"kl": 0.03948404639959335,
|
||
|
|
"learning_rate": 4.05e-06,
|
||
|
|
"loss": 0.0016,
|
||
|
|
"reward": 2.9744999408721924,
|
||
|
|
"reward_std": 0.91214519739151,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9745000600814819,
|
||
|
|
"step": 81
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 134.5,
|
||
|
|
"epoch": 0.010972835541281948,
|
||
|
|
"grad_norm": 1.0002614259719849,
|
||
|
|
"kl": 0.046765152364969254,
|
||
|
|
"learning_rate": 4.1e-06,
|
||
|
|
"loss": 0.0019,
|
||
|
|
"reward": 1.7273750305175781,
|
||
|
|
"reward_std": 0.7195501923561096,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9773749709129333,
|
||
|
|
"step": 82
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 198.625,
|
||
|
|
"epoch": 0.011106650608858558,
|
||
|
|
"grad_norm": 0.5600577592849731,
|
||
|
|
"kl": 0.022656453773379326,
|
||
|
|
"learning_rate": 4.15e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 2.2507500648498535,
|
||
|
|
"reward_std": 1.0691120624542236,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0625,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0007500648498535,
|
||
|
|
"step": 83
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 197.5,
|
||
|
|
"epoch": 0.011240465676435166,
|
||
|
|
"grad_norm": 0.4727917015552521,
|
||
|
|
"kl": 0.018861515447497368,
|
||
|
|
"learning_rate": 4.2000000000000004e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 2.188499927520752,
|
||
|
|
"reward_std": 1.0999188423156738,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 84
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 169.375,
|
||
|
|
"epoch": 0.011374280744011776,
|
||
|
|
"grad_norm": 0.6654386520385742,
|
||
|
|
"kl": 0.023982375860214233,
|
||
|
|
"learning_rate": 4.25e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 1.4539999961853027,
|
||
|
|
"reward_std": 1.0523241758346558,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8289999961853027,
|
||
|
|
"step": 85
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 102.625,
|
||
|
|
"epoch": 0.011508095811588384,
|
||
|
|
"grad_norm": 1.203064203262329,
|
||
|
|
"kl": 0.0678047314286232,
|
||
|
|
"learning_rate": 4.3e-06,
|
||
|
|
"loss": 0.0027,
|
||
|
|
"reward": 3.304999828338623,
|
||
|
|
"reward_std": 0.7506926655769348,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0625,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9924999475479126,
|
||
|
|
"step": 86
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 157.625,
|
||
|
|
"epoch": 0.011641910879164994,
|
||
|
|
"grad_norm": 0.6558554172515869,
|
||
|
|
"kl": 0.035465698689222336,
|
||
|
|
"learning_rate": 4.350000000000001e-06,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"reward": 1.938499927520752,
|
||
|
|
"reward_std": 1.2938729524612427,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.1875,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 87
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 170.875,
|
||
|
|
"epoch": 0.011775725946741603,
|
||
|
|
"grad_norm": 0.467671662569046,
|
||
|
|
"kl": 0.03049163520336151,
|
||
|
|
"learning_rate": 4.4e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"reward": 3.188499927520752,
|
||
|
|
"reward_std": 0.8838834762573242,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 88
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 163.875,
|
||
|
|
"epoch": 0.011909541014318212,
|
||
|
|
"grad_norm": 0.7639308571815491,
|
||
|
|
"kl": 0.02996567264199257,
|
||
|
|
"learning_rate": 4.450000000000001e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"reward": 1.750999927520752,
|
||
|
|
"reward_std": 1.1019463539123535,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.25,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 89
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 212.375,
|
||
|
|
"epoch": 0.01204335608189482,
|
||
|
|
"grad_norm": 0.5778034925460815,
|
||
|
|
"kl": 0.03728095442056656,
|
||
|
|
"learning_rate": 4.5e-06,
|
||
|
|
"loss": 0.0015,
|
||
|
|
"reward": 2.6665000915527344,
|
||
|
|
"reward_std": 1.3489927053451538,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0625,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8540000319480896,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 132.0,
|
||
|
|
"epoch": 0.01217717114947143,
|
||
|
|
"grad_norm": 0.6906542181968689,
|
||
|
|
"kl": 0.04584599286317825,
|
||
|
|
"learning_rate": 4.5500000000000005e-06,
|
||
|
|
"loss": 0.0018,
|
||
|
|
"reward": 3.062999963760376,
|
||
|
|
"reward_std": 0.9793040752410889,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0625,
|
||
|
|
"rewards/xmlcount_reward_func": 1.000499963760376,
|
||
|
|
"step": 91
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 174.75,
|
||
|
|
"epoch": 0.01231098621704804,
|
||
|
|
"grad_norm": 0.4605662524700165,
|
||
|
|
"kl": 0.015500452369451523,
|
||
|
|
"learning_rate": 4.600000000000001e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"reward": 2.500999927520752,
|
||
|
|
"reward_std": 1.069044828414917,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 92
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 190.625,
|
||
|
|
"epoch": 0.012444801284624649,
|
||
|
|
"grad_norm": 0.017340591177344322,
|
||
|
|
"kl": 0.03146103397011757,
|
||
|
|
"learning_rate": 4.65e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"reward": 3.500999927520752,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 93
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 154.375,
|
||
|
|
"epoch": 0.012578616352201259,
|
||
|
|
"grad_norm": 0.5866317749023438,
|
||
|
|
"kl": 0.024630675092339516,
|
||
|
|
"learning_rate": 4.7e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 2.250999927520752,
|
||
|
|
"reward_std": 1.0350983142852783,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 94
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 156.5,
|
||
|
|
"epoch": 0.012712431419777867,
|
||
|
|
"grad_norm": 0.6401786804199219,
|
||
|
|
"kl": 0.011824361979961395,
|
||
|
|
"learning_rate": 4.75e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"reward": 2.000999927520752,
|
||
|
|
"reward_std": 0.9258200526237488,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 95
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 197.125,
|
||
|
|
"epoch": 0.012846246487354477,
|
||
|
|
"grad_norm": 0.5072675347328186,
|
||
|
|
"kl": 0.023525703698396683,
|
||
|
|
"learning_rate": 4.800000000000001e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 2.2507500648498535,
|
||
|
|
"reward_std": 1.0691120624542236,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0625,
|
||
|
|
"rewards/xmlcount_reward_func": 1.000749945640564,
|
||
|
|
"step": 96
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 154.25,
|
||
|
|
"epoch": 0.012980061554931085,
|
||
|
|
"grad_norm": 0.7046185731887817,
|
||
|
|
"kl": 0.03401239216327667,
|
||
|
|
"learning_rate": 4.85e-06,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"reward": 2.250999927520752,
|
||
|
|
"reward_std": 1.1019463539123535,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.25,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 97
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 120.125,
|
||
|
|
"epoch": 0.013113876622507695,
|
||
|
|
"grad_norm": 0.8547447919845581,
|
||
|
|
"kl": 0.031828928738832474,
|
||
|
|
"learning_rate": 4.9000000000000005e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"reward": 3.2507500648498535,
|
||
|
|
"reward_std": 0.7070060968399048,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.000749945640564,
|
||
|
|
"step": 98
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 136.875,
|
||
|
|
"epoch": 0.013247691690084303,
|
||
|
|
"grad_norm": 0.701848566532135,
|
||
|
|
"kl": 0.04948738217353821,
|
||
|
|
"learning_rate": 4.95e-06,
|
||
|
|
"loss": 0.002,
|
||
|
|
"reward": 2.1882500648498535,
|
||
|
|
"reward_std": 1.1002274751663208,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0007500648498535,
|
||
|
|
"step": 99
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 158.5,
|
||
|
|
"epoch": 0.013381506757660913,
|
||
|
|
"grad_norm": 0.5703092813491821,
|
||
|
|
"kl": 0.03237658366560936,
|
||
|
|
"learning_rate": 5e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"reward": 2.5625,
|
||
|
|
"reward_std": 1.147528886795044,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0625,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 196.25,
|
||
|
|
"epoch": 0.013515321825237521,
|
||
|
|
"grad_norm": 0.7725698947906494,
|
||
|
|
"kl": 0.03819940611720085,
|
||
|
|
"learning_rate": 4.999984769144476e-06,
|
||
|
|
"loss": 0.0015,
|
||
|
|
"reward": 1.3757500648498535,
|
||
|
|
"reward_std": 0.3530486226081848,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.3125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0625,
|
||
|
|
"rewards/xmlcount_reward_func": 1.000749945640564,
|
||
|
|
"step": 101
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 284.75,
|
||
|
|
"epoch": 0.01364913689281413,
|
||
|
|
"grad_norm": 0.47911059856414795,
|
||
|
|
"kl": 0.02090694010257721,
|
||
|
|
"learning_rate": 4.999939076763487e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 2.7507500648498535,
|
||
|
|
"reward_std": 1.0348914861679077,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0007500648498535,
|
||
|
|
"step": 102
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 108.125,
|
||
|
|
"epoch": 0.01378295196039074,
|
||
|
|
"grad_norm": 0.6628210544586182,
|
||
|
|
"kl": 0.04560092091560364,
|
||
|
|
"learning_rate": 4.999862923413781e-06,
|
||
|
|
"loss": 0.0018,
|
||
|
|
"reward": 3.3117499351501465,
|
||
|
|
"reward_std": 0.9606176614761353,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.125,
|
||
|
|
"rewards/xmlcount_reward_func": 0.999250054359436,
|
||
|
|
"step": 103
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 190.25,
|
||
|
|
"epoch": 0.013916767027967349,
|
||
|
|
"grad_norm": 0.47198909521102905,
|
||
|
|
"kl": 0.04583700746297836,
|
||
|
|
"learning_rate": 4.999756310023261e-06,
|
||
|
|
"loss": 0.0018,
|
||
|
|
"reward": 3.0002498626708984,
|
||
|
|
"reward_std": 0.9259749054908752,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.000249981880188,
|
||
|
|
"step": 104
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 210.625,
|
||
|
|
"epoch": 0.014050582095543959,
|
||
|
|
"grad_norm": 0.4388250410556793,
|
||
|
|
"kl": 0.024925414472818375,
|
||
|
|
"learning_rate": 4.9996192378909785e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 2.250499963760376,
|
||
|
|
"reward_std": 1.0349606275558472,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.000499963760376,
|
||
|
|
"step": 105
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 172.375,
|
||
|
|
"epoch": 0.014184397163120567,
|
||
|
|
"grad_norm": 0.6656742691993713,
|
||
|
|
"kl": 0.06062869355082512,
|
||
|
|
"learning_rate": 4.999451708687114e-06,
|
||
|
|
"loss": 0.0024,
|
||
|
|
"reward": 3.3122501373291016,
|
||
|
|
"reward_std": 0.75318443775177,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0625,
|
||
|
|
"rewards/xmlcount_reward_func": 0.999750018119812,
|
||
|
|
"step": 106
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 93.5,
|
||
|
|
"epoch": 0.014318212230697177,
|
||
|
|
"grad_norm": 0.8294744491577148,
|
||
|
|
"kl": 0.10790253430604935,
|
||
|
|
"learning_rate": 4.9992537244529585e-06,
|
||
|
|
"loss": 0.0043,
|
||
|
|
"reward": 3.561500072479248,
|
||
|
|
"reward_std": 0.1767766922712326,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0625,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9990000128746033,
|
||
|
|
"step": 107
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 269.625,
|
||
|
|
"epoch": 0.014452027298273785,
|
||
|
|
"grad_norm": 0.2980615794658661,
|
||
|
|
"kl": 0.017784131690859795,
|
||
|
|
"learning_rate": 4.999025287600886e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"reward": 2.750499963760376,
|
||
|
|
"reward_std": 1.034684658050537,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.000499963760376,
|
||
|
|
"step": 108
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 129.625,
|
||
|
|
"epoch": 0.014585842365850395,
|
||
|
|
"grad_norm": 0.9553698897361755,
|
||
|
|
"kl": 0.0943451002240181,
|
||
|
|
"learning_rate": 4.998766400914329e-06,
|
||
|
|
"loss": 0.0038,
|
||
|
|
"reward": 3.6242499351501465,
|
||
|
|
"reward_std": 0.23130176961421967,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.125,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9992499947547913,
|
||
|
|
"step": 109
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 163.625,
|
||
|
|
"epoch": 0.014719657433427003,
|
||
|
|
"grad_norm": 0.45802226662635803,
|
||
|
|
"kl": 0.05015110969543457,
|
||
|
|
"learning_rate": 4.99847706754774e-06,
|
||
|
|
"loss": 0.002,
|
||
|
|
"reward": 3.1872501373291016,
|
||
|
|
"reward_std": 0.8841871023178101,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.999750018119812,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 237.75,
|
||
|
|
"epoch": 0.014853472501003613,
|
||
|
|
"grad_norm": 0.3533436954021454,
|
||
|
|
"kl": 0.047210708260536194,
|
||
|
|
"learning_rate": 4.998157291026553e-06,
|
||
|
|
"loss": 0.0019,
|
||
|
|
"reward": 2.906374931335449,
|
||
|
|
"reward_std": 1.0347106456756592,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9688750505447388,
|
||
|
|
"step": 111
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 158.25,
|
||
|
|
"epoch": 0.014987287568580221,
|
||
|
|
"grad_norm": 0.9160925149917603,
|
||
|
|
"kl": 0.04597921669483185,
|
||
|
|
"learning_rate": 4.997807075247147e-06,
|
||
|
|
"loss": 0.0018,
|
||
|
|
"reward": 3.3897500038146973,
|
||
|
|
"reward_std": 0.9479596018791199,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.1875,
|
||
|
|
"rewards/xmlcount_reward_func": 0.952250063419342,
|
||
|
|
"step": 112
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 158.875,
|
||
|
|
"epoch": 0.015121102636156831,
|
||
|
|
"grad_norm": 0.5840315818786621,
|
||
|
|
"kl": 0.0752396360039711,
|
||
|
|
"learning_rate": 4.997426424476787e-06,
|
||
|
|
"loss": 0.003,
|
||
|
|
"reward": 2.8127498626708984,
|
||
|
|
"reward_std": 1.251542329788208,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0002501010894775,
|
||
|
|
"step": 113
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 220.125,
|
||
|
|
"epoch": 0.015254917703733441,
|
||
|
|
"grad_norm": 0.5775883197784424,
|
||
|
|
"kl": 0.04557417705655098,
|
||
|
|
"learning_rate": 4.9970153433535855e-06,
|
||
|
|
"loss": 0.0018,
|
||
|
|
"reward": 2.5002498626708984,
|
||
|
|
"reward_std": 1.0693126916885376,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.000249981880188,
|
||
|
|
"step": 114
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 212.375,
|
||
|
|
"epoch": 0.01538873277131005,
|
||
|
|
"grad_norm": 0.5855643153190613,
|
||
|
|
"kl": 0.0537395142018795,
|
||
|
|
"learning_rate": 4.9965738368864345e-06,
|
||
|
|
"loss": 0.0021,
|
||
|
|
"reward": 2.297874927520752,
|
||
|
|
"reward_std": 1.0647298097610474,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8603750467300415,
|
||
|
|
"step": 115
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 195.125,
|
||
|
|
"epoch": 0.01552254783888666,
|
||
|
|
"grad_norm": 0.41128572821617126,
|
||
|
|
"kl": 0.046660423278808594,
|
||
|
|
"learning_rate": 4.996101910454953e-06,
|
||
|
|
"loss": 0.0019,
|
||
|
|
"reward": 3.1882500648498535,
|
||
|
|
"reward_std": 0.8837827444076538,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.000749945640564,
|
||
|
|
"step": 116
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 154.625,
|
||
|
|
"epoch": 0.01565636290646327,
|
||
|
|
"grad_norm": 0.5770456790924072,
|
||
|
|
"kl": 0.035205211490392685,
|
||
|
|
"learning_rate": 4.995599569809414e-06,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"reward": 3.2507500648498535,
|
||
|
|
"reward_std": 0.7070060968399048,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0007500648498535,
|
||
|
|
"step": 117
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 350.75,
|
||
|
|
"epoch": 0.015790177974039876,
|
||
|
|
"grad_norm": 0.3080421984195709,
|
||
|
|
"kl": 0.02211996726691723,
|
||
|
|
"learning_rate": 4.9950668210706795e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 1.2037500143051147,
|
||
|
|
"reward_std": 0.6614398956298828,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8287500143051147,
|
||
|
|
"step": 118
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 164.625,
|
||
|
|
"epoch": 0.015923993041616485,
|
||
|
|
"grad_norm": 0.6492892503738403,
|
||
|
|
"kl": 0.02696000412106514,
|
||
|
|
"learning_rate": 4.994503670730126e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"reward": 2.8132500648498535,
|
||
|
|
"reward_std": 0.9615465998649597,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0625,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0007500648498535,
|
||
|
|
"step": 119
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 123.375,
|
||
|
|
"epoch": 0.016057808109193095,
|
||
|
|
"grad_norm": 0.7609626054763794,
|
||
|
|
"kl": 0.05437001213431358,
|
||
|
|
"learning_rate": 4.993910125649561e-06,
|
||
|
|
"loss": 0.0022,
|
||
|
|
"reward": 3.5627498626708984,
|
||
|
|
"reward_std": 0.17627395689487457,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0625,
|
||
|
|
"rewards/xmlcount_reward_func": 1.000249981880188,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 246.0,
|
||
|
|
"epoch": 0.016191623176769705,
|
||
|
|
"grad_norm": 0.5615448355674744,
|
||
|
|
"kl": 0.032086554914712906,
|
||
|
|
"learning_rate": 4.993286193061145e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"reward": 1.1882500648498535,
|
||
|
|
"reward_std": 0.2584303021430969,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.1875,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0007500648498535,
|
||
|
|
"step": 121
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 338.75,
|
||
|
|
"epoch": 0.01632543824434631,
|
||
|
|
"grad_norm": 0.7422879338264465,
|
||
|
|
"kl": 0.028502581641077995,
|
||
|
|
"learning_rate": 4.992631880567301e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"reward": 1.5007500648498535,
|
||
|
|
"reward_std": 0.0007071277359500527,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.000749945640564,
|
||
|
|
"step": 122
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 179.625,
|
||
|
|
"epoch": 0.01645925331192292,
|
||
|
|
"grad_norm": 0.6192591190338135,
|
||
|
|
"kl": 0.03176417201757431,
|
||
|
|
"learning_rate": 4.991947196140619e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"reward": 2.360374927520752,
|
||
|
|
"reward_std": 1.273769497871399,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8603750467300415,
|
||
|
|
"step": 123
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 186.5,
|
||
|
|
"epoch": 0.01659306837949953,
|
||
|
|
"grad_norm": 0.5681326985359192,
|
||
|
|
"kl": 0.03496817499399185,
|
||
|
|
"learning_rate": 4.9912321481237616e-06,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"reward": 1.750999927520752,
|
||
|
|
"reward_std": 0.7071067094802856,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 124
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 255.625,
|
||
|
|
"epoch": 0.01672688344707614,
|
||
|
|
"grad_norm": 0.4468543529510498,
|
||
|
|
"kl": 0.022565122693777084,
|
||
|
|
"learning_rate": 4.990486745229364e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 1.938499927520752,
|
||
|
|
"reward_std": 1.2938729524612427,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.1875,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 125
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 95.0,
|
||
|
|
"epoch": 0.01686069851465275,
|
||
|
|
"grad_norm": 0.977435290813446,
|
||
|
|
"kl": 0.08776085078716278,
|
||
|
|
"learning_rate": 4.989710996539926e-06,
|
||
|
|
"loss": 0.0035,
|
||
|
|
"reward": 3.625499963760376,
|
||
|
|
"reward_std": 0.23052926361560822,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.000499963760376,
|
||
|
|
"step": 126
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 192.125,
|
||
|
|
"epoch": 0.016994513582229358,
|
||
|
|
"grad_norm": 0.015967685729265213,
|
||
|
|
"kl": 0.0297485813498497,
|
||
|
|
"learning_rate": 4.9889049115077e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"reward": 3.500999927520752,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 127
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 130.375,
|
||
|
|
"epoch": 0.017128328649805968,
|
||
|
|
"grad_norm": 0.6309700608253479,
|
||
|
|
"kl": 0.06466387212276459,
|
||
|
|
"learning_rate": 4.988068499954578e-06,
|
||
|
|
"loss": 0.0026,
|
||
|
|
"reward": 2.9382500648498535,
|
||
|
|
"reward_std": 1.050476312637329,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0625,
|
||
|
|
"rewards/xmlcount_reward_func": 1.000749945640564,
|
||
|
|
"step": 128
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 200.375,
|
||
|
|
"epoch": 0.017262143717382578,
|
||
|
|
"grad_norm": 0.4523550570011139,
|
||
|
|
"kl": 0.038704678416252136,
|
||
|
|
"learning_rate": 4.987201772071971e-06,
|
||
|
|
"loss": 0.0015,
|
||
|
|
"reward": 3.000999927520752,
|
||
|
|
"reward_std": 0.9258200526237488,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 129
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 268.125,
|
||
|
|
"epoch": 0.017395958784959187,
|
||
|
|
"grad_norm": 0.4572976529598236,
|
||
|
|
"kl": 0.02546817809343338,
|
||
|
|
"learning_rate": 4.986304738420684e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 1.3757500648498535,
|
||
|
|
"reward_std": 0.23130175471305847,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.000749945640564,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 138.375,
|
||
|
|
"epoch": 0.017529773852535794,
|
||
|
|
"grad_norm": 1.002087116241455,
|
||
|
|
"kl": 0.054114002734422684,
|
||
|
|
"learning_rate": 4.985377409930789e-06,
|
||
|
|
"loss": 0.0022,
|
||
|
|
"reward": 2.937999963760376,
|
||
|
|
"reward_std": 1.1154919862747192,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.3125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.125,
|
||
|
|
"rewards/xmlcount_reward_func": 1.000499963760376,
|
||
|
|
"step": 131
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 202.75,
|
||
|
|
"epoch": 0.017663588920112404,
|
||
|
|
"grad_norm": 0.013795514591038227,
|
||
|
|
"kl": 0.016482144594192505,
|
||
|
|
"learning_rate": 4.984419797901491e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"reward": 1.5010000467300415,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 132
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 162.75,
|
||
|
|
"epoch": 0.017797403987689014,
|
||
|
|
"grad_norm": 0.5195367336273193,
|
||
|
|
"kl": 0.04107159376144409,
|
||
|
|
"learning_rate": 4.983431914000991e-06,
|
||
|
|
"loss": 0.0016,
|
||
|
|
"reward": 3.3132500648498535,
|
||
|
|
"reward_std": 0.7527097463607788,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0625,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0007500648498535,
|
||
|
|
"step": 133
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 150.75,
|
||
|
|
"epoch": 0.017931219055265624,
|
||
|
|
"grad_norm": 0.6692448258399963,
|
||
|
|
"kl": 0.07387490570545197,
|
||
|
|
"learning_rate": 4.9824137702663424e-06,
|
||
|
|
"loss": 0.003,
|
||
|
|
"reward": 3.1252498626708984,
|
||
|
|
"reward_std": 0.8768863081932068,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.1875,
|
||
|
|
"rewards/xmlcount_reward_func": 1.000249981880188,
|
||
|
|
"step": 134
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 136.75,
|
||
|
|
"epoch": 0.018065034122842234,
|
||
|
|
"grad_norm": 0.704760730266571,
|
||
|
|
"kl": 0.05747693404555321,
|
||
|
|
"learning_rate": 4.981365379103306e-06,
|
||
|
|
"loss": 0.0023,
|
||
|
|
"reward": 3.500499963760376,
|
||
|
|
"reward_std": 0.0009257529745809734,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0005000829696655,
|
||
|
|
"step": 135
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 218.75,
|
||
|
|
"epoch": 0.01819884919041884,
|
||
|
|
"grad_norm": 0.40006327629089355,
|
||
|
|
"kl": 0.08134910464286804,
|
||
|
|
"learning_rate": 4.980286753286196e-06,
|
||
|
|
"loss": 0.0033,
|
||
|
|
"reward": 1.797374963760376,
|
||
|
|
"reward_std": 1.221002459526062,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0625,
|
||
|
|
"rewards/xmlcount_reward_func": 0.859874963760376,
|
||
|
|
"step": 136
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 170.25,
|
||
|
|
"epoch": 0.01833266425799545,
|
||
|
|
"grad_norm": 0.49556273221969604,
|
||
|
|
"kl": 0.04155727103352547,
|
||
|
|
"learning_rate": 4.979177905957726e-06,
|
||
|
|
"loss": 0.0017,
|
||
|
|
"reward": 2.688499927520752,
|
||
|
|
"reward_std": 1.1319231986999512,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 137
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 199.75,
|
||
|
|
"epoch": 0.01846647932557206,
|
||
|
|
"grad_norm": 0.46643561124801636,
|
||
|
|
"kl": 0.032832756638526917,
|
||
|
|
"learning_rate": 4.978038850628855e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"reward": 2.250999927520752,
|
||
|
|
"reward_std": 1.0350983142852783,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 138
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 127.5,
|
||
|
|
"epoch": 0.01860029439314867,
|
||
|
|
"grad_norm": 0.6091315150260925,
|
||
|
|
"kl": 0.032538849860429764,
|
||
|
|
"learning_rate": 4.9768696011786095e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"reward": 3.000999927520752,
|
||
|
|
"reward_std": 0.9258200526237488,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 139
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 172.375,
|
||
|
|
"epoch": 0.018734109460725276,
|
||
|
|
"grad_norm": 0.5669727325439453,
|
||
|
|
"kl": 0.032948385924100876,
|
||
|
|
"learning_rate": 4.975670171853926e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"reward": 2.438499927520752,
|
||
|
|
"reward_std": 1.1475906372070312,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 196.25,
|
||
|
|
"epoch": 0.018867924528301886,
|
||
|
|
"grad_norm": 0.7394277453422546,
|
||
|
|
"kl": 0.043011508882045746,
|
||
|
|
"learning_rate": 4.974440577269473e-06,
|
||
|
|
"loss": 0.0017,
|
||
|
|
"reward": 2.7351250648498535,
|
||
|
|
"reward_std": 1.483136534690857,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.7976250648498535,
|
||
|
|
"step": 141
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 299.25,
|
||
|
|
"epoch": 0.019001739595878496,
|
||
|
|
"grad_norm": 1.0737040042877197,
|
||
|
|
"kl": 0.07280484586954117,
|
||
|
|
"learning_rate": 4.973180832407471e-06,
|
||
|
|
"loss": 0.0029,
|
||
|
|
"reward": 1.2976250648498535,
|
||
|
|
"reward_std": 1.0223388671875,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.1875,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8601250648498535,
|
||
|
|
"step": 142
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 218.375,
|
||
|
|
"epoch": 0.019135554663455106,
|
||
|
|
"grad_norm": 0.47138193249702454,
|
||
|
|
"kl": 0.0328100360929966,
|
||
|
|
"learning_rate": 4.971890952617515e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"reward": 2.500999927520752,
|
||
|
|
"reward_std": 1.069044828414917,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 143
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 115.875,
|
||
|
|
"epoch": 0.019269369731031716,
|
||
|
|
"grad_norm": 0.809634804725647,
|
||
|
|
"kl": 0.05107220634818077,
|
||
|
|
"learning_rate": 4.970570953616383e-06,
|
||
|
|
"loss": 0.002,
|
||
|
|
"reward": 1.875749945640564,
|
||
|
|
"reward_std": 0.6945196986198425,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0007500648498535,
|
||
|
|
"step": 144
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 139.0,
|
||
|
|
"epoch": 0.019403184798608322,
|
||
|
|
"grad_norm": 1.0910069942474365,
|
||
|
|
"kl": 0.07320874184370041,
|
||
|
|
"learning_rate": 4.9692208514878445e-06,
|
||
|
|
"loss": 0.0029,
|
||
|
|
"reward": 3.360374927520752,
|
||
|
|
"reward_std": 0.39774757623672485,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8603750467300415,
|
||
|
|
"step": 145
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 204.5,
|
||
|
|
"epoch": 0.019536999866184932,
|
||
|
|
"grad_norm": 0.5369763374328613,
|
||
|
|
"kl": 0.036634840071201324,
|
||
|
|
"learning_rate": 4.96784066268247e-06,
|
||
|
|
"loss": 0.0015,
|
||
|
|
"reward": 2.250999927520752,
|
||
|
|
"reward_std": 1.0350983142852783,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 146
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 112.875,
|
||
|
|
"epoch": 0.019670814933761542,
|
||
|
|
"grad_norm": 0.6824727654457092,
|
||
|
|
"kl": 0.10541833192110062,
|
||
|
|
"learning_rate": 4.966430404017424e-06,
|
||
|
|
"loss": 0.0042,
|
||
|
|
"reward": 3.250999927520752,
|
||
|
|
"reward_std": 0.7071067690849304,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 147
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 270.25,
|
||
|
|
"epoch": 0.019804630001338152,
|
||
|
|
"grad_norm": 0.47248849272727966,
|
||
|
|
"kl": 0.040871020406484604,
|
||
|
|
"learning_rate": 4.964990092676263e-06,
|
||
|
|
"loss": 0.0016,
|
||
|
|
"reward": 1.938499927520752,
|
||
|
|
"reward_std": 0.9797046780586243,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 148
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 163.625,
|
||
|
|
"epoch": 0.01993844506891476,
|
||
|
|
"grad_norm": 0.07421360909938812,
|
||
|
|
"kl": 0.051940809935331345,
|
||
|
|
"learning_rate": 4.963519746208726e-06,
|
||
|
|
"loss": 0.0021,
|
||
|
|
"reward": 3.500999927520752,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 149
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 220.125,
|
||
|
|
"epoch": 0.02007226013649137,
|
||
|
|
"grad_norm": 0.469992458820343,
|
||
|
|
"kl": 0.045063313096761703,
|
||
|
|
"learning_rate": 4.962019382530521e-06,
|
||
|
|
"loss": 0.0018,
|
||
|
|
"reward": 1.938499927520752,
|
||
|
|
"reward_std": 0.979704737663269,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 156.875,
|
||
|
|
"epoch": 0.020206075204067978,
|
||
|
|
"grad_norm": 0.5472331643104553,
|
||
|
|
"kl": 0.03409579023718834,
|
||
|
|
"learning_rate": 4.960489019923105e-06,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"reward": 2.500999927520752,
|
||
|
|
"reward_std": 1.069044828414917,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 151
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 292.5,
|
||
|
|
"epoch": 0.020339890271644588,
|
||
|
|
"grad_norm": 0.41060271859169006,
|
||
|
|
"kl": 0.03718462586402893,
|
||
|
|
"learning_rate": 4.958928677033465e-06,
|
||
|
|
"loss": 0.0015,
|
||
|
|
"reward": 2.7196249961853027,
|
||
|
|
"reward_std": 1.0811783075332642,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9696249961853027,
|
||
|
|
"step": 152
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 270.75,
|
||
|
|
"epoch": 0.020473705339221198,
|
||
|
|
"grad_norm": 0.017522625625133514,
|
||
|
|
"kl": 0.02355693280696869,
|
||
|
|
"learning_rate": 4.957338372873886e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 1.5010000467300415,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 153
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 345.125,
|
||
|
|
"epoch": 0.020607520406797804,
|
||
|
|
"grad_norm": 0.04261169955134392,
|
||
|
|
"kl": 0.03182287886738777,
|
||
|
|
"learning_rate": 4.9557181268217225e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"reward": 1.5010000467300415,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 154
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 201.125,
|
||
|
|
"epoch": 0.020741335474374414,
|
||
|
|
"grad_norm": 0.5202696919441223,
|
||
|
|
"kl": 0.03061300329864025,
|
||
|
|
"learning_rate": 4.9540679586191605e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"reward": 3.188499927520752,
|
||
|
|
"reward_std": 0.8838834762573242,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 155
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 214.25,
|
||
|
|
"epoch": 0.020875150541951024,
|
||
|
|
"grad_norm": 0.5046383142471313,
|
||
|
|
"kl": 0.027153637260198593,
|
||
|
|
"learning_rate": 4.9523878883729794e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"reward": 2.375999927520752,
|
||
|
|
"reward_std": 1.2174328565597534,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 156
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 612.75,
|
||
|
|
"epoch": 0.021008965609527634,
|
||
|
|
"grad_norm": 0.21087080240249634,
|
||
|
|
"kl": 0.012561831623315811,
|
||
|
|
"learning_rate": 4.9506779365543054e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"reward": 1.625999927520752,
|
||
|
|
"reward_std": 0.7905693650245667,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 157
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 253.75,
|
||
|
|
"epoch": 0.02114278067710424,
|
||
|
|
"grad_norm": 0.5047801733016968,
|
||
|
|
"kl": 0.015701811760663986,
|
||
|
|
"learning_rate": 4.94893812399836e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"reward": 2.438499927520752,
|
||
|
|
"reward_std": 1.1475906372070312,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 158
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 204.125,
|
||
|
|
"epoch": 0.02127659574468085,
|
||
|
|
"grad_norm": 0.5075508952140808,
|
||
|
|
"kl": 0.015691544860601425,
|
||
|
|
"learning_rate": 4.947168471904213e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"reward": 2.750999927520752,
|
||
|
|
"reward_std": 1.0350983142852783,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 159
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 172.25,
|
||
|
|
"epoch": 0.02141041081225746,
|
||
|
|
"grad_norm": 0.49671560525894165,
|
||
|
|
"kl": 0.020630527287721634,
|
||
|
|
"learning_rate": 4.9453690018345144e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 2.750999927520752,
|
||
|
|
"reward_std": 1.0350983142852783,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 267.375,
|
||
|
|
"epoch": 0.02154422587983407,
|
||
|
|
"grad_norm": 0.350994735956192,
|
||
|
|
"kl": 0.017818449065089226,
|
||
|
|
"learning_rate": 4.9435397357152406e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"reward": 1.750999927520752,
|
||
|
|
"reward_std": 1.1019463539123535,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.25,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 161
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 190.625,
|
||
|
|
"epoch": 0.021678040947410677,
|
||
|
|
"grad_norm": 0.4397769868373871,
|
||
|
|
"kl": 0.02272423729300499,
|
||
|
|
"learning_rate": 4.9416806958354206e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 2.6571249961853027,
|
||
|
|
"reward_std": 1.1874943971633911,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9696249961853027,
|
||
|
|
"step": 162
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 239.375,
|
||
|
|
"epoch": 0.021811856014987287,
|
||
|
|
"grad_norm": 0.012928323820233345,
|
||
|
|
"kl": 0.018880505114793777,
|
||
|
|
"learning_rate": 4.939791904846869e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 1.0010000467300415,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.0,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 163
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 164.0,
|
||
|
|
"epoch": 0.021945671082563897,
|
||
|
|
"grad_norm": 0.015614047646522522,
|
||
|
|
"kl": 0.013937260955572128,
|
||
|
|
"learning_rate": 4.937873385763909e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"reward": 3.500999927520752,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 164
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 264.0,
|
||
|
|
"epoch": 0.022079486150140507,
|
||
|
|
"grad_norm": 0.44026902318000793,
|
||
|
|
"kl": 0.023002952337265015,
|
||
|
|
"learning_rate": 4.935925161963089e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 2.000999927520752,
|
||
|
|
"reward_std": 0.9258200526237488,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 165
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 168.75,
|
||
|
|
"epoch": 0.022213301217717116,
|
||
|
|
"grad_norm": 0.36826014518737793,
|
||
|
|
"kl": 0.026351258158683777,
|
||
|
|
"learning_rate": 4.933947257182901e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"reward": 2.000999927520752,
|
||
|
|
"reward_std": 0.9258200526237488,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 166
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 281.375,
|
||
|
|
"epoch": 0.022347116285293723,
|
||
|
|
"grad_norm": 0.4348958730697632,
|
||
|
|
"kl": 0.03400744870305061,
|
||
|
|
"learning_rate": 4.9319396955234925e-06,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"reward": 2.125999927520752,
|
||
|
|
"reward_std": 1.02643620967865,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 167
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 178.0,
|
||
|
|
"epoch": 0.022480931352870333,
|
||
|
|
"grad_norm": 0.013454057276248932,
|
||
|
|
"kl": 0.020782005041837692,
|
||
|
|
"learning_rate": 4.9299025014463665e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 3.500999927520752,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 168
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 99.125,
|
||
|
|
"epoch": 0.022614746420446943,
|
||
|
|
"grad_norm": 0.6706786155700684,
|
||
|
|
"kl": 0.037762995809316635,
|
||
|
|
"learning_rate": 4.92783569977409e-06,
|
||
|
|
"loss": 0.0015,
|
||
|
|
"reward": 3.235374927520752,
|
||
|
|
"reward_std": 0.7021570801734924,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9853750467300415,
|
||
|
|
"step": 169
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 246.625,
|
||
|
|
"epoch": 0.022748561488023553,
|
||
|
|
"grad_norm": 0.4395011067390442,
|
||
|
|
"kl": 0.021541399881243706,
|
||
|
|
"learning_rate": 4.925739315689991e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 2.485374927520752,
|
||
|
|
"reward_std": 1.0531362295150757,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9853750467300415,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 145.375,
|
||
|
|
"epoch": 0.02288237655560016,
|
||
|
|
"grad_norm": 0.5895448327064514,
|
||
|
|
"kl": 0.017903313040733337,
|
||
|
|
"learning_rate": 4.923613374737848e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"reward": 2.500999927520752,
|
||
|
|
"reward_std": 1.069044828414917,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 171
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 241.625,
|
||
|
|
"epoch": 0.02301619162317677,
|
||
|
|
"grad_norm": 0.5434019565582275,
|
||
|
|
"kl": 0.012325316667556763,
|
||
|
|
"learning_rate": 4.921457902821578e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"reward": 3.250999927520752,
|
||
|
|
"reward_std": 0.7071067690849304,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 172
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 119.125,
|
||
|
|
"epoch": 0.02315000669075338,
|
||
|
|
"grad_norm": 0.6367537379264832,
|
||
|
|
"kl": 0.021066242828965187,
|
||
|
|
"learning_rate": 4.9192729262049285e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 2.985374927520752,
|
||
|
|
"reward_std": 0.9171907305717468,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9853750467300415,
|
||
|
|
"step": 173
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 404.25,
|
||
|
|
"epoch": 0.02328382175832999,
|
||
|
|
"grad_norm": 0.326046347618103,
|
||
|
|
"kl": 0.023576462641358376,
|
||
|
|
"learning_rate": 4.917058471511149e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 2.000999927520752,
|
||
|
|
"reward_std": 0.9258200526237488,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 174
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 265.0,
|
||
|
|
"epoch": 0.0234176368259066,
|
||
|
|
"grad_norm": 0.6007720232009888,
|
||
|
|
"kl": 0.04033298417925835,
|
||
|
|
"learning_rate": 4.914814565722671e-06,
|
||
|
|
"loss": 0.0016,
|
||
|
|
"reward": 1.9272499084472656,
|
||
|
|
"reward_std": 1.335482120513916,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.25,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9272500276565552,
|
||
|
|
"step": 175
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 166.375,
|
||
|
|
"epoch": 0.023551451893483205,
|
||
|
|
"grad_norm": 0.622722327709198,
|
||
|
|
"kl": 0.040124136954545975,
|
||
|
|
"learning_rate": 4.912541236180779e-06,
|
||
|
|
"loss": 0.0016,
|
||
|
|
"reward": 3.4695000648498535,
|
||
|
|
"reward_std": 0.058329079300165176,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9695000648498535,
|
||
|
|
"step": 176
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 164.75,
|
||
|
|
"epoch": 0.023685266961059815,
|
||
|
|
"grad_norm": 0.46375563740730286,
|
||
|
|
"kl": 0.03894059360027313,
|
||
|
|
"learning_rate": 4.910238510585275e-06,
|
||
|
|
"loss": 0.0016,
|
||
|
|
"reward": 2.969749927520752,
|
||
|
|
"reward_std": 0.6572059392929077,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.25,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9697500467300415,
|
||
|
|
"step": 177
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 159.0,
|
||
|
|
"epoch": 0.023819082028636425,
|
||
|
|
"grad_norm": 0.5922391414642334,
|
||
|
|
"kl": 0.04015544056892395,
|
||
|
|
"learning_rate": 4.907906416994146e-06,
|
||
|
|
"loss": 0.0016,
|
||
|
|
"reward": 2.938499927520752,
|
||
|
|
"reward_std": 1.0500850677490234,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 178
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 286.25,
|
||
|
|
"epoch": 0.023952897096213035,
|
||
|
|
"grad_norm": 0.00922817550599575,
|
||
|
|
"kl": 0.010824710130691528,
|
||
|
|
"learning_rate": 4.905544983823214e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"reward": 1.5010000467300415,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 179
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 177.375,
|
||
|
|
"epoch": 0.02408671216378964,
|
||
|
|
"grad_norm": 0.014155122451484203,
|
||
|
|
"kl": 0.028690611943602562,
|
||
|
|
"learning_rate": 4.903154239845798e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"reward": 3.500999927520752,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 160.5,
|
||
|
|
"epoch": 0.02422052723136625,
|
||
|
|
"grad_norm": 0.00964143592864275,
|
||
|
|
"kl": 0.027186835184693336,
|
||
|
|
"learning_rate": 4.900734214192358e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"reward": 3.500999927520752,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 181
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 181.25,
|
||
|
|
"epoch": 0.02435434229894286,
|
||
|
|
"grad_norm": 0.41265276074409485,
|
||
|
|
"kl": 0.03371923044323921,
|
||
|
|
"learning_rate": 4.898284936350144e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"reward": 3.125999927520752,
|
||
|
|
"reward_std": 0.876274585723877,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 182
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 182.0,
|
||
|
|
"epoch": 0.02448815736651947,
|
||
|
|
"grad_norm": 0.09521566331386566,
|
||
|
|
"kl": 0.03205399587750435,
|
||
|
|
"learning_rate": 4.8958064361628334e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"reward": 3.500999927520752,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 183
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 314.125,
|
||
|
|
"epoch": 0.02462197243409608,
|
||
|
|
"grad_norm": 0.42055413126945496,
|
||
|
|
"kl": 0.021630937233567238,
|
||
|
|
"learning_rate": 4.893298743830168e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 1.563499927520752,
|
||
|
|
"reward_std": 0.8210402727127075,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.3125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 184
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 256.625,
|
||
|
|
"epoch": 0.024755787501672687,
|
||
|
|
"grad_norm": 0.011799737811088562,
|
||
|
|
"kl": 0.022686151787638664,
|
||
|
|
"learning_rate": 4.890761889907589e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 3.500999927520752,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 185
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 214.875,
|
||
|
|
"epoch": 0.024889602569249297,
|
||
|
|
"grad_norm": 0.46379536390304565,
|
||
|
|
"kl": 0.015875784680247307,
|
||
|
|
"learning_rate": 4.888195905305859e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"reward": 2.500999927520752,
|
||
|
|
"reward_std": 1.069044828414917,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 186
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 360.25,
|
||
|
|
"epoch": 0.025023417636825907,
|
||
|
|
"grad_norm": 0.38108423352241516,
|
||
|
|
"kl": 0.012030637823045254,
|
||
|
|
"learning_rate": 4.885600821290692e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"reward": 1.4385000467300415,
|
||
|
|
"reward_std": 0.1767766922712326,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 187
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 322.375,
|
||
|
|
"epoch": 0.025157232704402517,
|
||
|
|
"grad_norm": 0.3240382969379425,
|
||
|
|
"kl": 0.019951753318309784,
|
||
|
|
"learning_rate": 4.882976669482368e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 1.4385000467300415,
|
||
|
|
"reward_std": 0.1767766922712326,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 188
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 232.5,
|
||
|
|
"epoch": 0.025291047771979124,
|
||
|
|
"grad_norm": 0.5204270482063293,
|
||
|
|
"kl": 0.02518557757139206,
|
||
|
|
"learning_rate": 4.880323481855347e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 2.938499927520752,
|
||
|
|
"reward_std": 0.9038607478141785,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 189
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 330.875,
|
||
|
|
"epoch": 0.025424862839555733,
|
||
|
|
"grad_norm": 0.3633680045604706,
|
||
|
|
"kl": 0.012905788607895374,
|
||
|
|
"learning_rate": 4.8776412907378845e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"reward": 2.500999927520752,
|
||
|
|
"reward_std": 1.069044828414917,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 163.25,
|
||
|
|
"epoch": 0.025558677907132343,
|
||
|
|
"grad_norm": 0.01740916818380356,
|
||
|
|
"kl": 0.03026902675628662,
|
||
|
|
"learning_rate": 4.874930128811631e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"reward": 3.500999927520752,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 191
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 274.5,
|
||
|
|
"epoch": 0.025692492974708953,
|
||
|
|
"grad_norm": 0.01146768406033516,
|
||
|
|
"kl": 0.013433395884931087,
|
||
|
|
"learning_rate": 4.8721900291112415e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"reward": 1.5010000467300415,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 192
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 426.0,
|
||
|
|
"epoch": 0.02582630804228556,
|
||
|
|
"grad_norm": 0.009323939681053162,
|
||
|
|
"kl": 0.01329454593360424,
|
||
|
|
"learning_rate": 4.869421025023965e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"reward": 1.5010000467300415,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 193
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 158.375,
|
||
|
|
"epoch": 0.02596012310986217,
|
||
|
|
"grad_norm": 0.6743643283843994,
|
||
|
|
"kl": 0.03743596747517586,
|
||
|
|
"learning_rate": 4.866623150289241e-06,
|
||
|
|
"loss": 0.0015,
|
||
|
|
"reward": 3.250999927520752,
|
||
|
|
"reward_std": 0.7071067690849304,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 194
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 158.875,
|
||
|
|
"epoch": 0.02609393817743878,
|
||
|
|
"grad_norm": 0.049696680158376694,
|
||
|
|
"kl": 0.0463409386575222,
|
||
|
|
"learning_rate": 4.863796438998293e-06,
|
||
|
|
"loss": 0.0019,
|
||
|
|
"reward": 3.500999927520752,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 195
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 235.25,
|
||
|
|
"epoch": 0.02622775324501539,
|
||
|
|
"grad_norm": 0.4329129755496979,
|
||
|
|
"kl": 0.02660982497036457,
|
||
|
|
"learning_rate": 4.860940925593703e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"reward": 3.063499927520752,
|
||
|
|
"reward_std": 0.979704737663269,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0625,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 196
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 141.875,
|
||
|
|
"epoch": 0.026361568312592,
|
||
|
|
"grad_norm": 0.08335603028535843,
|
||
|
|
"kl": 0.0469173900783062,
|
||
|
|
"learning_rate": 4.858056644869002e-06,
|
||
|
|
"loss": 0.0019,
|
||
|
|
"reward": 3.500999927520752,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0625,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 197
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 248.625,
|
||
|
|
"epoch": 0.026495383380168606,
|
||
|
|
"grad_norm": 0.454050213098526,
|
||
|
|
"kl": 0.018201926723122597,
|
||
|
|
"learning_rate": 4.855143631968242e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"reward": 2.500999927520752,
|
||
|
|
"reward_std": 1.069044828414917,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 198
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 178.5,
|
||
|
|
"epoch": 0.026629198447745216,
|
||
|
|
"grad_norm": 0.4429324269294739,
|
||
|
|
"kl": 0.03411182388663292,
|
||
|
|
"learning_rate": 4.852201922385564e-06,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"reward": 3.188499927520752,
|
||
|
|
"reward_std": 0.8838834762573242,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 199
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 259.5,
|
||
|
|
"epoch": 0.026763013515321826,
|
||
|
|
"grad_norm": 0.48248985409736633,
|
||
|
|
"kl": 0.02396260015666485,
|
||
|
|
"learning_rate": 4.849231551964771e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 2.219749927520752,
|
||
|
|
"reward_std": 0.9949649572372437,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8447500467300415,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 178.875,
|
||
|
|
"epoch": 0.026896828582898435,
|
||
|
|
"grad_norm": 0.022656133398413658,
|
||
|
|
"kl": 0.01943058706820011,
|
||
|
|
"learning_rate": 4.84623255689889e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 3.500999927520752,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 201
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 273.0,
|
||
|
|
"epoch": 0.027030643650475042,
|
||
|
|
"grad_norm": 0.3635607659816742,
|
||
|
|
"kl": 0.019030285999178886,
|
||
|
|
"learning_rate": 4.84320497372973e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 1.688499927520752,
|
||
|
|
"reward_std": 0.752970278263092,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 202
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 225.875,
|
||
|
|
"epoch": 0.027164458718051652,
|
||
|
|
"grad_norm": 0.3966084420681,
|
||
|
|
"kl": 0.027967505156993866,
|
||
|
|
"learning_rate": 4.840148839347434e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"reward": 3.1832499504089355,
|
||
|
|
"reward_std": 0.8818846344947815,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9957500696182251,
|
||
|
|
"step": 203
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 161.75,
|
||
|
|
"epoch": 0.02729827378562826,
|
||
|
|
"grad_norm": 0.5269073843955994,
|
||
|
|
"kl": 0.029133355244994164,
|
||
|
|
"learning_rate": 4.837064190990036e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"reward": 3.250999927520752,
|
||
|
|
"reward_std": 0.7071067690849304,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 204
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 290.75,
|
||
|
|
"epoch": 0.02743208885320487,
|
||
|
|
"grad_norm": 0.4223637282848358,
|
||
|
|
"kl": 0.014362221583724022,
|
||
|
|
"learning_rate": 4.833951066243004e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"reward": 1.938499927520752,
|
||
|
|
"reward_std": 1.2938729524612427,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.1875,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 205
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 242.75,
|
||
|
|
"epoch": 0.02756590392078148,
|
||
|
|
"grad_norm": 0.31881266832351685,
|
||
|
|
"kl": 0.013570988550782204,
|
||
|
|
"learning_rate": 4.830809503038781e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"reward": 3.250999927520752,
|
||
|
|
"reward_std": 0.7071067690849304,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 206
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 240.0,
|
||
|
|
"epoch": 0.027699718988358088,
|
||
|
|
"grad_norm": 0.4299936294555664,
|
||
|
|
"kl": 0.019979460164904594,
|
||
|
|
"learning_rate": 4.8276395396563215e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 3.438499927520752,
|
||
|
|
"reward_std": 0.1767766922712326,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 207
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 221.25,
|
||
|
|
"epoch": 0.027833534055934698,
|
||
|
|
"grad_norm": 0.43808406591415405,
|
||
|
|
"kl": 0.020733119919896126,
|
||
|
|
"learning_rate": 4.824441214720629e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 2.297874927520752,
|
||
|
|
"reward_std": 1.3918408155441284,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8603750467300415,
|
||
|
|
"step": 208
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 256.125,
|
||
|
|
"epoch": 0.027967349123511308,
|
||
|
|
"grad_norm": 0.3329388499259949,
|
||
|
|
"kl": 0.024192897602915764,
|
||
|
|
"learning_rate": 4.821214567202284e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 2.438499927520752,
|
||
|
|
"reward_std": 1.1475906372070312,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 209
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 212.0,
|
||
|
|
"epoch": 0.028101164191087918,
|
||
|
|
"grad_norm": 0.3484155535697937,
|
||
|
|
"kl": 0.02922155149281025,
|
||
|
|
"learning_rate": 4.817959636416969e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"reward": 3.360374927520752,
|
||
|
|
"reward_std": 0.39774757623672485,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8603750467300415,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 323.0,
|
||
|
|
"epoch": 0.028234979258664524,
|
||
|
|
"grad_norm": 0.4701886475086212,
|
||
|
|
"kl": 0.020680323243141174,
|
||
|
|
"learning_rate": 4.814676462024988e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 1.375999927520752,
|
||
|
|
"reward_std": 0.8762745261192322,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 211
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 215.5,
|
||
|
|
"epoch": 0.028368794326241134,
|
||
|
|
"grad_norm": 0.5657159686088562,
|
||
|
|
"kl": 0.040002256631851196,
|
||
|
|
"learning_rate": 4.811365084030784e-06,
|
||
|
|
"loss": 0.0016,
|
||
|
|
"reward": 2.250999927520752,
|
||
|
|
"reward_std": 1.3363062143325806,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.25,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 212
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 164.625,
|
||
|
|
"epoch": 0.028502609393817744,
|
||
|
|
"grad_norm": 0.5063055753707886,
|
||
|
|
"kl": 0.02591283805668354,
|
||
|
|
"learning_rate": 4.808025542782453e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 3.000999927520752,
|
||
|
|
"reward_std": 0.9258200526237488,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 213
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 203.25,
|
||
|
|
"epoch": 0.028636424461394354,
|
||
|
|
"grad_norm": 0.4244440197944641,
|
||
|
|
"kl": 0.03919763118028641,
|
||
|
|
"learning_rate": 4.804657878971252e-06,
|
||
|
|
"loss": 0.0016,
|
||
|
|
"reward": 3.047874927520752,
|
||
|
|
"reward_std": 1.2816309928894043,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8603750467300415,
|
||
|
|
"step": 214
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 137.75,
|
||
|
|
"epoch": 0.028770239528970964,
|
||
|
|
"grad_norm": 0.4569053053855896,
|
||
|
|
"kl": 0.03501063957810402,
|
||
|
|
"learning_rate": 4.801262133631101e-06,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"reward": 2.250999927520752,
|
||
|
|
"reward_std": 1.0350983142852783,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 215
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 164.375,
|
||
|
|
"epoch": 0.02890405459654757,
|
||
|
|
"grad_norm": 0.5390671491622925,
|
||
|
|
"kl": 0.038795534521341324,
|
||
|
|
"learning_rate": 4.7978383481380865e-06,
|
||
|
|
"loss": 0.0016,
|
||
|
|
"reward": 2.688499927520752,
|
||
|
|
"reward_std": 0.997765302658081,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 216
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 250.75,
|
||
|
|
"epoch": 0.02903786966412418,
|
||
|
|
"grad_norm": 0.5104800462722778,
|
||
|
|
"kl": 0.020015213638544083,
|
||
|
|
"learning_rate": 4.794386564209953e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 2.313499927520752,
|
||
|
|
"reward_std": 1.4126343727111816,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.25,
|
||
|
|
"rewards/soft_format_reward_func": 0.0625,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 217
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 253.875,
|
||
|
|
"epoch": 0.02917168473170079,
|
||
|
|
"grad_norm": 0.39514511823654175,
|
||
|
|
"kl": 0.03106546960771084,
|
||
|
|
"learning_rate": 4.790906823905599e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"reward": 2.125999927520752,
|
||
|
|
"reward_std": 0.8762745261192322,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.125,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 218
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 243.75,
|
||
|
|
"epoch": 0.0293054997992774,
|
||
|
|
"grad_norm": 0.5085269212722778,
|
||
|
|
"kl": 0.04808986932039261,
|
||
|
|
"learning_rate": 4.787399169624562e-06,
|
||
|
|
"loss": 0.0019,
|
||
|
|
"reward": 1.688499927520752,
|
||
|
|
"reward_std": 0.752970278263092,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 219
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 197.75,
|
||
|
|
"epoch": 0.029439314866854006,
|
||
|
|
"grad_norm": 0.484157532453537,
|
||
|
|
"kl": 0.031817518174648285,
|
||
|
|
"learning_rate": 4.783863644106502e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"reward": 2.688499927520752,
|
||
|
|
"reward_std": 1.1319231986999512,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0625,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 272.875,
|
||
|
|
"epoch": 0.029573129934430616,
|
||
|
|
"grad_norm": 0.4163767397403717,
|
||
|
|
"kl": 0.031002264469861984,
|
||
|
|
"learning_rate": 4.780300290430683e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"reward": 1.1885000467300415,
|
||
|
|
"reward_std": 0.2587745785713196,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.1875,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 221
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 269.375,
|
||
|
|
"epoch": 0.029706945002007226,
|
||
|
|
"grad_norm": 0.35876283049583435,
|
||
|
|
"kl": 0.01797083392739296,
|
||
|
|
"learning_rate": 4.776709152015443e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"reward": 2.188499927520752,
|
||
|
|
"reward_std": 0.9613048434257507,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 222
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 197.625,
|
||
|
|
"epoch": 0.029840760069583836,
|
||
|
|
"grad_norm": 0.39538952708244324,
|
||
|
|
"kl": 0.03638344630599022,
|
||
|
|
"learning_rate": 4.773090272617672e-06,
|
||
|
|
"loss": 0.0015,
|
||
|
|
"reward": 1.563499927520752,
|
||
|
|
"reward_std": 0.8210402727127075,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.3125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 223
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 159.125,
|
||
|
|
"epoch": 0.029974575137160443,
|
||
|
|
"grad_norm": 1.2572689056396484,
|
||
|
|
"kl": 0.0397665910422802,
|
||
|
|
"learning_rate": 4.769443696332272e-06,
|
||
|
|
"loss": 0.0016,
|
||
|
|
"reward": 3.250999927520752,
|
||
|
|
"reward_std": 0.7071067690849304,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 224
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 310.5,
|
||
|
|
"epoch": 0.030108390204737052,
|
||
|
|
"grad_norm": 0.34139129519462585,
|
||
|
|
"kl": 0.015533320605754852,
|
||
|
|
"learning_rate": 4.765769467591626e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"reward": 3.250999927520752,
|
||
|
|
"reward_std": 0.7071067690849304,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 225
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 243.5,
|
||
|
|
"epoch": 0.030242205272313662,
|
||
|
|
"grad_norm": 0.5063166618347168,
|
||
|
|
"kl": 0.04719982668757439,
|
||
|
|
"learning_rate": 4.762067631165049e-06,
|
||
|
|
"loss": 0.0019,
|
||
|
|
"reward": 2.0633749961853027,
|
||
|
|
"reward_std": 1.2082972526550293,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.25,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8133750557899475,
|
||
|
|
"step": 226
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 229.625,
|
||
|
|
"epoch": 0.030376020339890272,
|
||
|
|
"grad_norm": 0.45795556902885437,
|
||
|
|
"kl": 0.09192857146263123,
|
||
|
|
"learning_rate": 4.7583382321582525e-06,
|
||
|
|
"loss": 0.0037,
|
||
|
|
"reward": 2.5008749961853027,
|
||
|
|
"reward_std": 1.5121413469314575,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8133749961853027,
|
||
|
|
"step": 227
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 246.5,
|
||
|
|
"epoch": 0.030509835407466882,
|
||
|
|
"grad_norm": 0.5579359531402588,
|
||
|
|
"kl": 0.041984669864177704,
|
||
|
|
"learning_rate": 4.754581316012785e-06,
|
||
|
|
"loss": 0.0017,
|
||
|
|
"reward": 1.3447500467300415,
|
||
|
|
"reward_std": 0.4419417679309845,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8447500467300415,
|
||
|
|
"step": 228
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 254.625,
|
||
|
|
"epoch": 0.03064365047504349,
|
||
|
|
"grad_norm": 0.38114693760871887,
|
||
|
|
"kl": 0.034735728055238724,
|
||
|
|
"learning_rate": 4.750796928505484e-06,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"reward": 2.313499927520752,
|
||
|
|
"reward_std": 1.1319231986999512,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0625,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 229
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 258.75,
|
||
|
|
"epoch": 0.0307774655426201,
|
||
|
|
"grad_norm": 0.3544849455356598,
|
||
|
|
"kl": 0.02206849493086338,
|
||
|
|
"learning_rate": 4.746985115747918e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 2.750999927520752,
|
||
|
|
"reward_std": 1.0350983142852783,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 294.125,
|
||
|
|
"epoch": 0.03091128061019671,
|
||
|
|
"grad_norm": 0.35905495285987854,
|
||
|
|
"kl": 0.023520147427916527,
|
||
|
|
"learning_rate": 4.743145924185821e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 3.000999927520752,
|
||
|
|
"reward_std": 0.9258200526237488,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 231
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 334.25,
|
||
|
|
"epoch": 0.03104509567777332,
|
||
|
|
"grad_norm": 0.3567132353782654,
|
||
|
|
"kl": 0.02822481282055378,
|
||
|
|
"learning_rate": 4.7392794005985324e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"reward": 2.9383749961853027,
|
||
|
|
"reward_std": 0.9040879607200623,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8133749961853027,
|
||
|
|
"step": 232
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 183.5,
|
||
|
|
"epoch": 0.031178910745349925,
|
||
|
|
"grad_norm": 0.5352622270584106,
|
||
|
|
"kl": 0.03570713847875595,
|
||
|
|
"learning_rate": 4.735385592098421e-06,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"reward": 3.313499927520752,
|
||
|
|
"reward_std": 0.5303300619125366,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0625,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 233
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 234.5,
|
||
|
|
"epoch": 0.03131272581292654,
|
||
|
|
"grad_norm": 0.5144317746162415,
|
||
|
|
"kl": 0.02466505579650402,
|
||
|
|
"learning_rate": 4.731464546130315e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 2.5008749961853027,
|
||
|
|
"reward_std": 1.5121413469314575,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8133749961853027,
|
||
|
|
"step": 234
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 254.5,
|
||
|
|
"epoch": 0.031446540880503145,
|
||
|
|
"grad_norm": 0.41540685296058655,
|
||
|
|
"kl": 0.025191426277160645,
|
||
|
|
"learning_rate": 4.72751631047092e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 1.4383749961853027,
|
||
|
|
"reward_std": 1.0838176012039185,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8133750557899475,
|
||
|
|
"step": 235
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 203.25,
|
||
|
|
"epoch": 0.03158035594807975,
|
||
|
|
"grad_norm": 0.0266204085201025,
|
||
|
|
"kl": 0.04067916423082352,
|
||
|
|
"learning_rate": 4.723540933228245e-06,
|
||
|
|
"loss": 0.0016,
|
||
|
|
"reward": 3.500999927520752,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 236
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 202.125,
|
||
|
|
"epoch": 0.031714171015656364,
|
||
|
|
"grad_norm": 0.4182632565498352,
|
||
|
|
"kl": 0.04802641272544861,
|
||
|
|
"learning_rate": 4.719538462841003e-06,
|
||
|
|
"loss": 0.0019,
|
||
|
|
"reward": 2.7508749961853027,
|
||
|
|
"reward_std": 1.0352709293365479,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8133749961853027,
|
||
|
|
"step": 237
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 149.75,
|
||
|
|
"epoch": 0.03184798608323297,
|
||
|
|
"grad_norm": 0.025329239666461945,
|
||
|
|
"kl": 0.048463426530361176,
|
||
|
|
"learning_rate": 4.715508948078037e-06,
|
||
|
|
"loss": 0.0019,
|
||
|
|
"reward": 3.500999927520752,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 238
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 314.75,
|
||
|
|
"epoch": 0.031981801150809584,
|
||
|
|
"grad_norm": 0.35496988892555237,
|
||
|
|
"kl": 0.02924046851694584,
|
||
|
|
"learning_rate": 4.71145243803771e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"reward": 1.5008749961853027,
|
||
|
|
"reward_std": 1.0693122148513794,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8133749961853027,
|
||
|
|
"step": 239
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 219.375,
|
||
|
|
"epoch": 0.03211561621838619,
|
||
|
|
"grad_norm": 0.021066131070256233,
|
||
|
|
"kl": 0.03024008870124817,
|
||
|
|
"learning_rate": 4.707368982147318e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"reward": 3.500999927520752,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 259.75,
|
||
|
|
"epoch": 0.0322494312859628,
|
||
|
|
"grad_norm": 0.30919742584228516,
|
||
|
|
"kl": 0.025379128754138947,
|
||
|
|
"learning_rate": 4.703258630162481e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 3.188499927520752,
|
||
|
|
"reward_std": 0.8838834762573242,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 241
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 202.0,
|
||
|
|
"epoch": 0.03238324635353941,
|
||
|
|
"grad_norm": 0.4490697979927063,
|
||
|
|
"kl": 0.038147423416376114,
|
||
|
|
"learning_rate": 4.699121432166542e-06,
|
||
|
|
"loss": 0.0015,
|
||
|
|
"reward": 2.750999927520752,
|
||
|
|
"reward_std": 1.0350983142852783,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 242
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 149.0,
|
||
|
|
"epoch": 0.03251706142111602,
|
||
|
|
"grad_norm": 0.038720302283763885,
|
||
|
|
"kl": 0.05375644937157631,
|
||
|
|
"learning_rate": 4.6949574385699514e-06,
|
||
|
|
"loss": 0.0022,
|
||
|
|
"reward": 3.500999927520752,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 243
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 259.25,
|
||
|
|
"epoch": 0.03265087648869262,
|
||
|
|
"grad_norm": 0.39240700006484985,
|
||
|
|
"kl": 0.03470877557992935,
|
||
|
|
"learning_rate": 4.690766700109659e-06,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"reward": 3.000999927520752,
|
||
|
|
"reward_std": 0.9258200526237488,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 244
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 286.0,
|
||
|
|
"epoch": 0.03278469155626924,
|
||
|
|
"grad_norm": 0.3337673544883728,
|
||
|
|
"kl": 0.024394266307353973,
|
||
|
|
"learning_rate": 4.68654926784849e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 1.750999927520752,
|
||
|
|
"reward_std": 0.7071067094802856,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 245
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 398.375,
|
||
|
|
"epoch": 0.03291850662384584,
|
||
|
|
"grad_norm": 0.008737333118915558,
|
||
|
|
"kl": 0.017680775374174118,
|
||
|
|
"learning_rate": 4.682305193174524e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"reward": 1.5010000467300415,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 246
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 269.25,
|
||
|
|
"epoch": 0.03305232169142246,
|
||
|
|
"grad_norm": 0.3497251570224762,
|
||
|
|
"kl": 0.030157841742038727,
|
||
|
|
"learning_rate": 4.6780345278004744e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"reward": 3.250999927520752,
|
||
|
|
"reward_std": 0.7071067690849304,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 247
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 346.75,
|
||
|
|
"epoch": 0.03318613675899906,
|
||
|
|
"grad_norm": 0.008697928860783577,
|
||
|
|
"kl": 0.01776600070297718,
|
||
|
|
"learning_rate": 4.673737323763048e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"reward": 1.5010000467300415,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 248
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 242.75,
|
||
|
|
"epoch": 0.03331995182657567,
|
||
|
|
"grad_norm": 0.01581292599439621,
|
||
|
|
"kl": 0.04547512158751488,
|
||
|
|
"learning_rate": 4.669413633422322e-06,
|
||
|
|
"loss": 0.0018,
|
||
|
|
"reward": 1.5010000467300415,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 249
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 405.5,
|
||
|
|
"epoch": 0.03345376689415228,
|
||
|
|
"grad_norm": 0.3250972032546997,
|
||
|
|
"kl": 0.027322612702846527,
|
||
|
|
"learning_rate": 4.665063509461098e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"reward": 1.688499927520752,
|
||
|
|
"reward_std": 0.752970278263092,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 378.25,
|
||
|
|
"epoch": 0.03358758196172889,
|
||
|
|
"grad_norm": 0.42158758640289307,
|
||
|
|
"kl": 0.022155875340104103,
|
||
|
|
"learning_rate": 4.6606870048842626e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 2.563499927520752,
|
||
|
|
"reward_std": 1.2938729524612427,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.3125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 251
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 336.0,
|
||
|
|
"epoch": 0.0337213970293055,
|
||
|
|
"grad_norm": 0.21698202192783356,
|
||
|
|
"kl": 0.02450788952410221,
|
||
|
|
"learning_rate": 4.656284173018144e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 3.188499927520752,
|
||
|
|
"reward_std": 0.8838834762573242,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 252
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 188.375,
|
||
|
|
"epoch": 0.03385521209688211,
|
||
|
|
"grad_norm": 0.45206162333488464,
|
||
|
|
"kl": 0.05112133175134659,
|
||
|
|
"learning_rate": 4.65185506750986e-06,
|
||
|
|
"loss": 0.002,
|
||
|
|
"reward": 3.250999927520752,
|
||
|
|
"reward_std": 0.7071067690849304,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 253
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 234.0,
|
||
|
|
"epoch": 0.033989027164458716,
|
||
|
|
"grad_norm": 0.43149563670158386,
|
||
|
|
"kl": 0.02526533044874668,
|
||
|
|
"learning_rate": 4.6473997423266615e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 3.250999927520752,
|
||
|
|
"reward_std": 0.7071067690849304,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 254
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 287.875,
|
||
|
|
"epoch": 0.03412284223203533,
|
||
|
|
"grad_norm": 0.36620572209358215,
|
||
|
|
"kl": 0.016464661806821823,
|
||
|
|
"learning_rate": 4.642918251755281e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"reward": 2.250999927520752,
|
||
|
|
"reward_std": 1.0350983142852783,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 255
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 286.75,
|
||
|
|
"epoch": 0.034256657299611935,
|
||
|
|
"grad_norm": 0.29181429743766785,
|
||
|
|
"kl": 0.027665331959724426,
|
||
|
|
"learning_rate": 4.638410650401267e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"reward": 3.188499927520752,
|
||
|
|
"reward_std": 0.8838834762573242,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 256
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 258.625,
|
||
|
|
"epoch": 0.03439047236718855,
|
||
|
|
"grad_norm": 0.36111629009246826,
|
||
|
|
"kl": 0.022748613730072975,
|
||
|
|
"learning_rate": 4.633876993188319e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 2.000999927520752,
|
||
|
|
"reward_std": 0.9258200526237488,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 257
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 279.0,
|
||
|
|
"epoch": 0.034524287434765155,
|
||
|
|
"grad_norm": 0.015732252970337868,
|
||
|
|
"kl": 0.02347162365913391,
|
||
|
|
"learning_rate": 4.62931733535762e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 3.500999927520752,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 258
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 370.625,
|
||
|
|
"epoch": 0.03465810250234176,
|
||
|
|
"grad_norm": 0.31918811798095703,
|
||
|
|
"kl": 0.027137352153658867,
|
||
|
|
"learning_rate": 4.62473173246716e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"reward": 2.610374927520752,
|
||
|
|
"reward_std": 1.2772694826126099,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8603750467300415,
|
||
|
|
"step": 259
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 225.875,
|
||
|
|
"epoch": 0.034791917569918375,
|
||
|
|
"grad_norm": 0.5192745923995972,
|
||
|
|
"kl": 0.025867221876978874,
|
||
|
|
"learning_rate": 4.620120240391065e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 3.000999927520752,
|
||
|
|
"reward_std": 0.9258200526237488,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 254.0,
|
||
|
|
"epoch": 0.03492573263749498,
|
||
|
|
"grad_norm": 0.36218902468681335,
|
||
|
|
"kl": 0.017309075221419334,
|
||
|
|
"learning_rate": 4.6154829153189105e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"reward": 3.000999927520752,
|
||
|
|
"reward_std": 0.9258200526237488,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 261
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 211.375,
|
||
|
|
"epoch": 0.03505954770507159,
|
||
|
|
"grad_norm": 0.5987810492515564,
|
||
|
|
"kl": 0.03272293880581856,
|
||
|
|
"learning_rate": 4.610819813755038e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"reward": 3.250999927520752,
|
||
|
|
"reward_std": 0.7071067690849304,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 262
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 274.875,
|
||
|
|
"epoch": 0.0351933627726482,
|
||
|
|
"grad_norm": 0.5228343605995178,
|
||
|
|
"kl": 0.02676045522093773,
|
||
|
|
"learning_rate": 4.60613099251787e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"reward": 1.422874927520752,
|
||
|
|
"reward_std": 1.006535530090332,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.3125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8603750467300415,
|
||
|
|
"step": 263
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 212.625,
|
||
|
|
"epoch": 0.03532717784022481,
|
||
|
|
"grad_norm": 0.48773717880249023,
|
||
|
|
"kl": 0.08342333137989044,
|
||
|
|
"learning_rate": 4.601416508739211e-06,
|
||
|
|
"loss": 0.0033,
|
||
|
|
"reward": 2.938499927520752,
|
||
|
|
"reward_std": 0.9038607478141785,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 264
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 317.0,
|
||
|
|
"epoch": 0.03546099290780142,
|
||
|
|
"grad_norm": 0.2805244028568268,
|
||
|
|
"kl": 0.04268364980816841,
|
||
|
|
"learning_rate": 4.596676419863561e-06,
|
||
|
|
"loss": 0.0017,
|
||
|
|
"reward": 2.688499927520752,
|
||
|
|
"reward_std": 1.1319231986999512,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 265
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 218.75,
|
||
|
|
"epoch": 0.03559480797537803,
|
||
|
|
"grad_norm": 0.5128685235977173,
|
||
|
|
"kl": 0.0328143872320652,
|
||
|
|
"learning_rate": 4.591910783647405e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"reward": 3.250999927520752,
|
||
|
|
"reward_std": 0.7071067690849304,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 266
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 309.0,
|
||
|
|
"epoch": 0.035728623042954634,
|
||
|
|
"grad_norm": 0.3302849531173706,
|
||
|
|
"kl": 0.02219189517199993,
|
||
|
|
"learning_rate": 4.587119658158517e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 1.750999927520752,
|
||
|
|
"reward_std": 0.7071067094802856,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 267
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 216.75,
|
||
|
|
"epoch": 0.03586243811053125,
|
||
|
|
"grad_norm": 0.35290464758872986,
|
||
|
|
"kl": 0.02897994965314865,
|
||
|
|
"learning_rate": 4.582303101775249e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"reward": 3.250999927520752,
|
||
|
|
"reward_std": 0.7071067690849304,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 268
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 189.25,
|
||
|
|
"epoch": 0.035996253178107854,
|
||
|
|
"grad_norm": 0.48176464438438416,
|
||
|
|
"kl": 0.026264041662216187,
|
||
|
|
"learning_rate": 4.577461173185821e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"reward": 3.250999927520752,
|
||
|
|
"reward_std": 0.7071067690849304,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 269
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 344.625,
|
||
|
|
"epoch": 0.03613006824568447,
|
||
|
|
"grad_norm": 0.35015684366226196,
|
||
|
|
"kl": 0.025320082902908325,
|
||
|
|
"learning_rate": 4.572593931387604e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 2.375999927520752,
|
||
|
|
"reward_std": 1.2174328565597534,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 196.0,
|
||
|
|
"epoch": 0.036263883313261074,
|
||
|
|
"grad_norm": 0.37403514981269836,
|
||
|
|
"kl": 0.025439651682972908,
|
||
|
|
"learning_rate": 4.567701435686405e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 3.360374927520752,
|
||
|
|
"reward_std": 0.39774757623672485,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8603750467300415,
|
||
|
|
"step": 271
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 314.125,
|
||
|
|
"epoch": 0.03639769838083768,
|
||
|
|
"grad_norm": 0.524261474609375,
|
||
|
|
"kl": 0.0216533113270998,
|
||
|
|
"learning_rate": 4.562783745695738e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 1.735374927520752,
|
||
|
|
"reward_std": 1.2218937873840332,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8603750467300415,
|
||
|
|
"step": 272
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 408.375,
|
||
|
|
"epoch": 0.03653151344841429,
|
||
|
|
"grad_norm": 0.2718545198440552,
|
||
|
|
"kl": 0.01605556346476078,
|
||
|
|
"learning_rate": 4.5578409213361055e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"reward": 1.688499927520752,
|
||
|
|
"reward_std": 0.752970278263092,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 273
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 303.875,
|
||
|
|
"epoch": 0.0366653285159909,
|
||
|
|
"grad_norm": 0.43400371074676514,
|
||
|
|
"kl": 0.020561426877975464,
|
||
|
|
"learning_rate": 4.55287302283426e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 2.500999927520752,
|
||
|
|
"reward_std": 1.069044828414917,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 274
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 213.75,
|
||
|
|
"epoch": 0.036799143583567506,
|
||
|
|
"grad_norm": 0.42370080947875977,
|
||
|
|
"kl": 0.025757426396012306,
|
||
|
|
"learning_rate": 4.54788011072248e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 2.500999927520752,
|
||
|
|
"reward_std": 1.069044828414917,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 275
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 263.375,
|
||
|
|
"epoch": 0.03693295865114412,
|
||
|
|
"grad_norm": 0.26540645956993103,
|
||
|
|
"kl": 0.018899496644735336,
|
||
|
|
"learning_rate": 4.542862245837821e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 3.250999927520752,
|
||
|
|
"reward_std": 0.7071067690849304,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 276
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 308.625,
|
||
|
|
"epoch": 0.037066773718720726,
|
||
|
|
"grad_norm": 0.318920761346817,
|
||
|
|
"kl": 0.01868232898414135,
|
||
|
|
"learning_rate": 4.537819489321385e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"reward": 3.000999927520752,
|
||
|
|
"reward_std": 0.9258200526237488,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 277
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 252.125,
|
||
|
|
"epoch": 0.03720058878629734,
|
||
|
|
"grad_norm": 0.37147581577301025,
|
||
|
|
"kl": 0.024237308651208878,
|
||
|
|
"learning_rate": 4.5327519026175694e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 2.875999927520752,
|
||
|
|
"reward_std": 1.1572750806808472,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 278
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 196.375,
|
||
|
|
"epoch": 0.037334403853873946,
|
||
|
|
"grad_norm": 0.4675554037094116,
|
||
|
|
"kl": 0.05026804655790329,
|
||
|
|
"learning_rate": 4.527659547473317e-06,
|
||
|
|
"loss": 0.002,
|
||
|
|
"reward": 2.687999963760376,
|
||
|
|
"reward_std": 0.9974435567855835,
|
||
|
|
"rewards/correctness_reward_func": 1.25,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.000499963760376,
|
||
|
|
"step": 279
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 310.0,
|
||
|
|
"epoch": 0.03746821892145055,
|
||
|
|
"grad_norm": 0.3709997236728668,
|
||
|
|
"kl": 0.01588054746389389,
|
||
|
|
"learning_rate": 4.522542485937369e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"reward": 2.500999927520752,
|
||
|
|
"reward_std": 1.069044828414917,
|
||
|
|
"rewards/correctness_reward_func": 1.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 306.5,
|
||
|
|
"epoch": 0.037602033989027166,
|
||
|
|
"grad_norm": 0.8552827835083008,
|
||
|
|
"kl": 0.02514849603176117,
|
||
|
|
"learning_rate": 4.517400780359505e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 3.250999927520752,
|
||
|
|
"reward_std": 0.7071067690849304,
|
||
|
|
"rewards/correctness_reward_func": 1.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 281
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 406.5,
|
||
|
|
"epoch": 0.03773584905660377,
|
||
|
|
"grad_norm": 0.3171614110469818,
|
||
|
|
"kl": 0.01633845828473568,
|
||
|
|
"learning_rate": 4.512234493389785e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"reward": 2.125999927520752,
|
||
|
|
"reward_std": 1.1572750806808472,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 282
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 456.875,
|
||
|
|
"epoch": 0.037869664124180386,
|
||
|
|
"grad_norm": 0.23401294648647308,
|
||
|
|
"kl": 0.02182060480117798,
|
||
|
|
"learning_rate": 4.507043687977787e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 1.0635000467300415,
|
||
|
|
"reward_std": 0.1767766922712326,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.0625,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 283
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 258.5,
|
||
|
|
"epoch": 0.03800347919175699,
|
||
|
|
"grad_norm": 0.35936811566352844,
|
||
|
|
"kl": 0.02936350367963314,
|
||
|
|
"learning_rate": 4.501828427371834e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"reward": 1.750999927520752,
|
||
|
|
"reward_std": 0.7071067094802856,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 284
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 251.375,
|
||
|
|
"epoch": 0.0381372942593336,
|
||
|
|
"grad_norm": 0.2574705183506012,
|
||
|
|
"kl": 0.024459898471832275,
|
||
|
|
"learning_rate": 4.496588775118232e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 2.875999927520752,
|
||
|
|
"reward_std": 1.1572750806808472,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 285
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 425.375,
|
||
|
|
"epoch": 0.03827110932691021,
|
||
|
|
"grad_norm": 0.3120448887348175,
|
||
|
|
"kl": 0.016995567828416824,
|
||
|
|
"learning_rate": 4.491324795060491e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"reward": 1.7664999961853027,
|
||
|
|
"reward_std": 1.0987950563430786,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.1875,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8289999961853027,
|
||
|
|
"step": 286
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 321.75,
|
||
|
|
"epoch": 0.03840492439448682,
|
||
|
|
"grad_norm": 0.29984933137893677,
|
||
|
|
"kl": 0.020925011485815048,
|
||
|
|
"learning_rate": 4.4860365513385456e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 1.3760000467300415,
|
||
|
|
"reward_std": 0.2314550280570984,
|
||
|
|
"rewards/correctness_reward_func": 0.0,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 287
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 286.5,
|
||
|
|
"epoch": 0.03853873946206343,
|
||
|
|
"grad_norm": 0.384897381067276,
|
||
|
|
"kl": 0.019656995311379433,
|
||
|
|
"learning_rate": 4.4807241083879774e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"reward": 1.750999927520752,
|
||
|
|
"reward_std": 0.7071067094802856,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 288
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 150.875,
|
||
|
|
"epoch": 0.03867255452964004,
|
||
|
|
"grad_norm": 0.578263521194458,
|
||
|
|
"kl": 0.05649289861321449,
|
||
|
|
"learning_rate": 4.475387530939226e-06,
|
||
|
|
"loss": 0.0023,
|
||
|
|
"reward": 3.434999942779541,
|
||
|
|
"reward_std": 0.42524483799934387,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0625,
|
||
|
|
"rewards/xmlcount_reward_func": 0.872499942779541,
|
||
|
|
"step": 289
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 278.875,
|
||
|
|
"epoch": 0.038806369597216644,
|
||
|
|
"grad_norm": 0.3382101356983185,
|
||
|
|
"kl": 0.031233415007591248,
|
||
|
|
"learning_rate": 4.470026884016805e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"reward": 2.250999927520752,
|
||
|
|
"reward_std": 1.0350983142852783,
|
||
|
|
"rewards/correctness_reward_func": 0.75,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 226.0,
|
||
|
|
"epoch": 0.03894018466479326,
|
||
|
|
"grad_norm": 0.613914430141449,
|
||
|
|
"kl": 0.027662649750709534,
|
||
|
|
"learning_rate": 4.464642232938505e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"reward": 2.847249984741211,
|
||
|
|
"reward_std": 1.0878411531448364,
|
||
|
|
"rewards/correctness_reward_func": 1.5,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9097499847412109,
|
||
|
|
"step": 291
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 262.75,
|
||
|
|
"epoch": 0.039073999732369864,
|
||
|
|
"grad_norm": 0.39136871695518494,
|
||
|
|
"kl": 0.042321134358644485,
|
||
|
|
"learning_rate": 4.4592336433146e-06,
|
||
|
|
"loss": 0.0017,
|
||
|
|
"reward": 1.937999963760376,
|
||
|
|
"reward_std": 0.9801060557365417,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.000499963760376,
|
||
|
|
"step": 292
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 197.0,
|
||
|
|
"epoch": 0.03920781479994647,
|
||
|
|
"grad_norm": 0.37938910722732544,
|
||
|
|
"kl": 0.027261145412921906,
|
||
|
|
"learning_rate": 4.453801181047047e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"reward": 3.3561248779296875,
|
||
|
|
"reward_std": 0.27151820063591003,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8561250567436218,
|
||
|
|
"step": 293
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 203.125,
|
||
|
|
"epoch": 0.039341629867523084,
|
||
|
|
"grad_norm": 0.45251545310020447,
|
||
|
|
"kl": 0.03213547542691231,
|
||
|
|
"learning_rate": 4.448344912328686e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"reward": 3.3289999961853027,
|
||
|
|
"reward_std": 0.39496004581451416,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.8289999961853027,
|
||
|
|
"step": 294
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 155.625,
|
||
|
|
"epoch": 0.03947544493509969,
|
||
|
|
"grad_norm": 0.025667138397693634,
|
||
|
|
"kl": 0.04739276319742203,
|
||
|
|
"learning_rate": 4.442864903642428e-06,
|
||
|
|
"loss": 0.0019,
|
||
|
|
"reward": 3.500999927520752,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 295
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 106.875,
|
||
|
|
"epoch": 0.039609260002676304,
|
||
|
|
"grad_norm": 0.5052531957626343,
|
||
|
|
"kl": 0.03284822404384613,
|
||
|
|
"learning_rate": 4.437361221760449e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"reward": 3.438499927520752,
|
||
|
|
"reward_std": 0.1767766922712326,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.4375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 296
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 242.375,
|
||
|
|
"epoch": 0.03974307507025291,
|
||
|
|
"grad_norm": 0.40129604935646057,
|
||
|
|
"kl": 0.03474697098135948,
|
||
|
|
"learning_rate": 4.431833933743378e-06,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"reward": 1.8422499895095825,
|
||
|
|
"reward_std": 1.063101053237915,
|
||
|
|
"rewards/correctness_reward_func": 0.5,
|
||
|
|
"rewards/int_reward_func": 0.375,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 0.9672499895095825,
|
||
|
|
"step": 297
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 298.25,
|
||
|
|
"epoch": 0.03987689013782952,
|
||
|
|
"grad_norm": 0.2920699119567871,
|
||
|
|
"kl": 0.025090906769037247,
|
||
|
|
"learning_rate": 4.426283106939474e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"reward": 1.563499927520752,
|
||
|
|
"reward_std": 0.8210402131080627,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.3125,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 298
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 236.0,
|
||
|
|
"epoch": 0.04001070520540613,
|
||
|
|
"grad_norm": 0.032971348613500595,
|
||
|
|
"kl": 0.041622765362262726,
|
||
|
|
"learning_rate": 4.420708808983809e-06,
|
||
|
|
"loss": 0.0017,
|
||
|
|
"reward": 3.500999927520752,
|
||
|
|
"reward_std": 0.0,
|
||
|
|
"rewards/correctness_reward_func": 2.0,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0010000467300415,
|
||
|
|
"step": 299
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"completion_length": 259.625,
|
||
|
|
"epoch": 0.04014452027298274,
|
||
|
|
"grad_norm": 0.312053382396698,
|
||
|
|
"kl": 0.02263636142015457,
|
||
|
|
"learning_rate": 4.415111107797445e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"reward": 1.750749945640564,
|
||
|
|
"reward_std": 0.7072080969810486,
|
||
|
|
"rewards/correctness_reward_func": 0.25,
|
||
|
|
"rewards/int_reward_func": 0.5,
|
||
|
|
"rewards/soft_format_reward_func": 0.0,
|
||
|
|
"rewards/strict_format_reward_func": 0.0,
|
||
|
|
"rewards/xmlcount_reward_func": 1.0007500648498535,
|
||
|
|
"step": 300
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 1,
|
||
|
|
"max_steps": 1000,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 1,
|
||
|
|
"save_steps": 100,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": false
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 1,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|