3206 lines
87 KiB
JSON
3206 lines
87 KiB
JSON
{
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.6736609607951408,
|
|
"eval_steps": 500,
|
|
"global_step": 2440,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"completion_length": 39.4375,
|
|
"epoch": 0.0027609055770292656,
|
|
"grad_norm": 0.384765625,
|
|
"kl": 0.0011762777163918752,
|
|
"learning_rate": 9.97239094422971e-06,
|
|
"loss": 0.0001,
|
|
"reward": 1.0288461538461537,
|
|
"reward_std": 0.09615384615384616,
|
|
"rewards/accuracy_reward": 0.057692307692307696,
|
|
"rewards/format_reward": 0.9711538461538461,
|
|
"step": 10
|
|
},
|
|
{
|
|
"completion_length": 52.55,
|
|
"epoch": 0.005521811154058531,
|
|
"grad_norm": 0.201171875,
|
|
"kl": 0.010907393763773143,
|
|
"learning_rate": 9.944781888459416e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 20
|
|
},
|
|
{
|
|
"completion_length": 55.275,
|
|
"epoch": 0.008282716731087797,
|
|
"grad_norm": 0.34765625,
|
|
"kl": 0.009688957687467336,
|
|
"learning_rate": 9.917172832689123e-06,
|
|
"loss": 0.0004,
|
|
"reward": 0.95,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.95,
|
|
"step": 30
|
|
},
|
|
{
|
|
"completion_length": 60.35,
|
|
"epoch": 0.011043622308117063,
|
|
"grad_norm": 0.125,
|
|
"kl": 0.01061137397773564,
|
|
"learning_rate": 9.889563776918831e-06,
|
|
"loss": 0.0004,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 40
|
|
},
|
|
{
|
|
"completion_length": 35.725,
|
|
"epoch": 0.013804527885146328,
|
|
"grad_norm": 0.205078125,
|
|
"kl": 0.02767656436190009,
|
|
"learning_rate": 9.861954721148538e-06,
|
|
"loss": 0.0011,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 50
|
|
},
|
|
{
|
|
"completion_length": 63.925,
|
|
"epoch": 0.016565433462175594,
|
|
"grad_norm": 0.1923828125,
|
|
"kl": 0.018191399239003658,
|
|
"learning_rate": 9.834345665378246e-06,
|
|
"loss": 0.0007,
|
|
"reward": 0.975,
|
|
"reward_std": 0.16547005176544188,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 0.925,
|
|
"step": 60
|
|
},
|
|
{
|
|
"completion_length": 62.65,
|
|
"epoch": 0.01932633903920486,
|
|
"grad_norm": 0.408203125,
|
|
"kl": 0.014432385563850403,
|
|
"learning_rate": 9.806736609607953e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 70
|
|
},
|
|
{
|
|
"completion_length": 51.15,
|
|
"epoch": 0.022087244616234125,
|
|
"grad_norm": 0.3671875,
|
|
"kl": 0.01872744457796216,
|
|
"learning_rate": 9.77912755383766e-06,
|
|
"loss": 0.0007,
|
|
"reward": 1.025,
|
|
"reward_std": 0.15,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 80
|
|
},
|
|
{
|
|
"completion_length": 40.325,
|
|
"epoch": 0.02484815019326339,
|
|
"grad_norm": 0.734375,
|
|
"kl": 0.015993439219892026,
|
|
"learning_rate": 9.751518498067368e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 90
|
|
},
|
|
{
|
|
"completion_length": 40.85,
|
|
"epoch": 0.027609055770292656,
|
|
"grad_norm": 0.66015625,
|
|
"kl": 0.015149969141930342,
|
|
"learning_rate": 9.723909442297074e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 100
|
|
},
|
|
{
|
|
"completion_length": 73.125,
|
|
"epoch": 0.030369961347321922,
|
|
"grad_norm": 19.25,
|
|
"kl": 0.011897594993934035,
|
|
"learning_rate": 9.696300386526781e-06,
|
|
"loss": 0.0005,
|
|
"reward": 0.925,
|
|
"reward_std": 0.18938468694686889,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 0.9,
|
|
"step": 110
|
|
},
|
|
{
|
|
"completion_length": 60.65,
|
|
"epoch": 0.03313086692435119,
|
|
"grad_norm": 0.1494140625,
|
|
"kl": 0.01287351930513978,
|
|
"learning_rate": 9.66869133075649e-06,
|
|
"loss": 0.0005,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 120
|
|
},
|
|
{
|
|
"completion_length": 50.525,
|
|
"epoch": 0.03589177250138045,
|
|
"grad_norm": 0.37890625,
|
|
"kl": 0.013646931713446974,
|
|
"learning_rate": 9.641082274986196e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 130
|
|
},
|
|
{
|
|
"completion_length": 45.275,
|
|
"epoch": 0.03865267807840972,
|
|
"grad_norm": 0.37109375,
|
|
"kl": 0.011697723204270006,
|
|
"learning_rate": 9.613473219215904e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 140
|
|
},
|
|
{
|
|
"completion_length": 44.725,
|
|
"epoch": 0.041413583655438985,
|
|
"grad_norm": 0.25,
|
|
"kl": 0.008990359771996737,
|
|
"learning_rate": 9.585864163445611e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 150
|
|
},
|
|
{
|
|
"completion_length": 45.575,
|
|
"epoch": 0.04417448923246825,
|
|
"grad_norm": 0.3125,
|
|
"kl": 0.008469818299636245,
|
|
"learning_rate": 9.558255107675318e-06,
|
|
"loss": 0.0003,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 160
|
|
},
|
|
{
|
|
"completion_length": 57.625,
|
|
"epoch": 0.046935394809497516,
|
|
"grad_norm": 0.5625,
|
|
"kl": 0.0104887580499053,
|
|
"learning_rate": 9.530646051905026e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 170
|
|
},
|
|
{
|
|
"completion_length": 50.475,
|
|
"epoch": 0.04969630038652678,
|
|
"grad_norm": 0.53515625,
|
|
"kl": 0.01001789947040379,
|
|
"learning_rate": 9.503036996134733e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 180
|
|
},
|
|
{
|
|
"completion_length": 48.725,
|
|
"epoch": 0.05245720596355605,
|
|
"grad_norm": 0.224609375,
|
|
"kl": 0.006566637242212891,
|
|
"learning_rate": 9.47542794036444e-06,
|
|
"loss": 0.0003,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 190
|
|
},
|
|
{
|
|
"completion_length": 50.125,
|
|
"epoch": 0.05521811154058531,
|
|
"grad_norm": 0.271484375,
|
|
"kl": 0.0062376999529078605,
|
|
"learning_rate": 9.447818884594148e-06,
|
|
"loss": 0.0002,
|
|
"reward": 1.0,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 200
|
|
},
|
|
{
|
|
"completion_length": 63.85,
|
|
"epoch": 0.05797901711761458,
|
|
"grad_norm": 0.2734375,
|
|
"kl": 0.01781983794644475,
|
|
"learning_rate": 9.420209828823854e-06,
|
|
"loss": 0.0007,
|
|
"reward": 1.025,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 210
|
|
},
|
|
{
|
|
"completion_length": 46.025,
|
|
"epoch": 0.060739922694643844,
|
|
"grad_norm": 0.54296875,
|
|
"kl": 0.009719261107966304,
|
|
"learning_rate": 9.392600773053563e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 220
|
|
},
|
|
{
|
|
"completion_length": 50.175,
|
|
"epoch": 0.06350082827167311,
|
|
"grad_norm": 0.2490234375,
|
|
"kl": 0.006523344293236732,
|
|
"learning_rate": 9.36499171728327e-06,
|
|
"loss": 0.0003,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 230
|
|
},
|
|
{
|
|
"completion_length": 52.375,
|
|
"epoch": 0.06626173384870238,
|
|
"grad_norm": 0.5546875,
|
|
"kl": 0.008632810413837433,
|
|
"learning_rate": 9.337382661512978e-06,
|
|
"loss": 0.0003,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 240
|
|
},
|
|
{
|
|
"completion_length": 38.75,
|
|
"epoch": 0.06902263942573164,
|
|
"grad_norm": 0.126953125,
|
|
"kl": 0.007552449638023972,
|
|
"learning_rate": 9.309773605742684e-06,
|
|
"loss": 0.0003,
|
|
"reward": 1.05,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 250
|
|
},
|
|
{
|
|
"completion_length": 48.375,
|
|
"epoch": 0.0717835450027609,
|
|
"grad_norm": 0.142578125,
|
|
"kl": 0.01255230070091784,
|
|
"learning_rate": 9.282164549972393e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 260
|
|
},
|
|
{
|
|
"completion_length": 63.85,
|
|
"epoch": 0.07454445057979017,
|
|
"grad_norm": 0.1513671875,
|
|
"kl": 0.008702394971624016,
|
|
"learning_rate": 9.2545554942021e-06,
|
|
"loss": 0.0003,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 270
|
|
},
|
|
{
|
|
"completion_length": 55.275,
|
|
"epoch": 0.07730535615681944,
|
|
"grad_norm": 0.09716796875,
|
|
"kl": 0.010681362450122833,
|
|
"learning_rate": 9.226946438431806e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 280
|
|
},
|
|
{
|
|
"completion_length": 62.7,
|
|
"epoch": 0.0800662617338487,
|
|
"grad_norm": 9.5625,
|
|
"kl": 0.007968542701564729,
|
|
"learning_rate": 9.199337382661515e-06,
|
|
"loss": 0.0003,
|
|
"reward": 1.0,
|
|
"reward_std": 0.2,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 0.95,
|
|
"step": 290
|
|
},
|
|
{
|
|
"completion_length": 49.3,
|
|
"epoch": 0.08282716731087797,
|
|
"grad_norm": 8.75,
|
|
"kl": 0.019733481900766493,
|
|
"learning_rate": 9.171728326891221e-06,
|
|
"loss": 0.0008,
|
|
"reward": 1.05,
|
|
"reward_std": 0.15773502588272095,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 300
|
|
},
|
|
{
|
|
"completion_length": 58.925,
|
|
"epoch": 0.08558807288790723,
|
|
"grad_norm": 0.267578125,
|
|
"kl": 0.01582272765226662,
|
|
"learning_rate": 9.144119271120928e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.075,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 310
|
|
},
|
|
{
|
|
"completion_length": 36.35,
|
|
"epoch": 0.0883489784649365,
|
|
"grad_norm": 10.6875,
|
|
"kl": 0.01768846157938242,
|
|
"learning_rate": 9.116510215350636e-06,
|
|
"loss": 0.0007,
|
|
"reward": 1.05,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 320
|
|
},
|
|
{
|
|
"completion_length": 36.1,
|
|
"epoch": 0.09110988404196577,
|
|
"grad_norm": 0.2373046875,
|
|
"kl": 0.024144751764833928,
|
|
"learning_rate": 9.088901159580343e-06,
|
|
"loss": 0.001,
|
|
"reward": 1.05,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 330
|
|
},
|
|
{
|
|
"completion_length": 35.225,
|
|
"epoch": 0.09387078961899503,
|
|
"grad_norm": 0.19140625,
|
|
"kl": 0.021723289880901574,
|
|
"learning_rate": 9.061292103810051e-06,
|
|
"loss": 0.0009,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 340
|
|
},
|
|
{
|
|
"completion_length": 41.35,
|
|
"epoch": 0.0966316951960243,
|
|
"grad_norm": 0.1962890625,
|
|
"kl": 0.020848556840792298,
|
|
"learning_rate": 9.033683048039758e-06,
|
|
"loss": 0.0008,
|
|
"reward": 1.075,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 350
|
|
},
|
|
{
|
|
"completion_length": 42.825,
|
|
"epoch": 0.09939260077305356,
|
|
"grad_norm": 0.26171875,
|
|
"kl": 0.01832999554462731,
|
|
"learning_rate": 9.006073992269465e-06,
|
|
"loss": 0.0007,
|
|
"reward": 1.05,
|
|
"reward_std": 0.18164966106414795,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 360
|
|
},
|
|
{
|
|
"completion_length": 50.575,
|
|
"epoch": 0.10215350635008283,
|
|
"grad_norm": 0.57421875,
|
|
"kl": 0.017559885466471314,
|
|
"learning_rate": 8.978464936499173e-06,
|
|
"loss": 0.0007,
|
|
"reward": 1.05,
|
|
"reward_std": 0.05773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 370
|
|
},
|
|
{
|
|
"completion_length": 40.825,
|
|
"epoch": 0.1049144119271121,
|
|
"grad_norm": 0.2041015625,
|
|
"kl": 0.017281288420781492,
|
|
"learning_rate": 8.95085588072888e-06,
|
|
"loss": 0.0007,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 380
|
|
},
|
|
{
|
|
"completion_length": 52.85,
|
|
"epoch": 0.10767531750414136,
|
|
"grad_norm": 16.875,
|
|
"kl": 0.021487308479845525,
|
|
"learning_rate": 8.923246824958586e-06,
|
|
"loss": 0.0009,
|
|
"reward": 1.125,
|
|
"reward_std": 0.15,
|
|
"rewards/accuracy_reward": 0.175,
|
|
"rewards/format_reward": 0.95,
|
|
"step": 390
|
|
},
|
|
{
|
|
"completion_length": 40.625,
|
|
"epoch": 0.11043622308117063,
|
|
"grad_norm": 0.388671875,
|
|
"kl": 0.017221461795270444,
|
|
"learning_rate": 8.895637769188295e-06,
|
|
"loss": 0.0007,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 400
|
|
},
|
|
{
|
|
"completion_length": 38.725,
|
|
"epoch": 0.11319712865819989,
|
|
"grad_norm": 29.125,
|
|
"kl": 0.017092665936797858,
|
|
"learning_rate": 8.868028713418001e-06,
|
|
"loss": 0.0007,
|
|
"reward": 0.95,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.95,
|
|
"step": 410
|
|
},
|
|
{
|
|
"completion_length": 36.15,
|
|
"epoch": 0.11595803423522916,
|
|
"grad_norm": 0.3671875,
|
|
"kl": 0.012849510973319411,
|
|
"learning_rate": 8.84041965764771e-06,
|
|
"loss": 0.0005,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 420
|
|
},
|
|
{
|
|
"completion_length": 44.075,
|
|
"epoch": 0.11871893981225842,
|
|
"grad_norm": 0.34375,
|
|
"kl": 0.01729746237397194,
|
|
"learning_rate": 8.812810601877416e-06,
|
|
"loss": 0.0007,
|
|
"reward": 1.0,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 430
|
|
},
|
|
{
|
|
"completion_length": 47.05,
|
|
"epoch": 0.12147984538928769,
|
|
"grad_norm": 0.232421875,
|
|
"kl": 0.012639089813455939,
|
|
"learning_rate": 8.785201546107123e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.05,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 440
|
|
},
|
|
{
|
|
"completion_length": 62.525,
|
|
"epoch": 0.12424075096631695,
|
|
"grad_norm": 0.5234375,
|
|
"kl": 0.014472203236073256,
|
|
"learning_rate": 8.757592490336831e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 450
|
|
},
|
|
{
|
|
"completion_length": 69.5,
|
|
"epoch": 0.12700165654334622,
|
|
"grad_norm": 0.267578125,
|
|
"kl": 0.011864464683458208,
|
|
"learning_rate": 8.72998343456654e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.1,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.1,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 460
|
|
},
|
|
{
|
|
"completion_length": 47.625,
|
|
"epoch": 0.12976256212037549,
|
|
"grad_norm": 0.2041015625,
|
|
"kl": 0.01089612185023725,
|
|
"learning_rate": 8.702374378796245e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.05,
|
|
"reward_std": 0.05773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 470
|
|
},
|
|
{
|
|
"completion_length": 48.025,
|
|
"epoch": 0.13252346769740475,
|
|
"grad_norm": 0.1865234375,
|
|
"kl": 0.011578870052471758,
|
|
"learning_rate": 8.674765323025953e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.075,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 480
|
|
},
|
|
{
|
|
"completion_length": 48.675,
|
|
"epoch": 0.13528437327443402,
|
|
"grad_norm": 0.15234375,
|
|
"kl": 0.015454969368875027,
|
|
"learning_rate": 8.647156267255661e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.0,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 490
|
|
},
|
|
{
|
|
"completion_length": 51.525,
|
|
"epoch": 0.13804527885146328,
|
|
"grad_norm": 0.341796875,
|
|
"kl": 0.011392440786585212,
|
|
"learning_rate": 8.619547211485368e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.15,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.15,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 500
|
|
},
|
|
{
|
|
"completion_length": 51.85,
|
|
"epoch": 0.14080618442849255,
|
|
"grad_norm": 0.302734375,
|
|
"kl": 0.010139884939417244,
|
|
"learning_rate": 8.591938155715075e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.075,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 510
|
|
},
|
|
{
|
|
"completion_length": 49.175,
|
|
"epoch": 0.1435670900055218,
|
|
"grad_norm": 0.486328125,
|
|
"kl": 0.013384641613811255,
|
|
"learning_rate": 8.564329099944783e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.075,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 520
|
|
},
|
|
{
|
|
"completion_length": 58.45,
|
|
"epoch": 0.14632799558255108,
|
|
"grad_norm": 0.3046875,
|
|
"kl": 0.014279201999306679,
|
|
"learning_rate": 8.53672004417449e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.05,
|
|
"reward_std": 0.2,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 530
|
|
},
|
|
{
|
|
"completion_length": 46.65,
|
|
"epoch": 0.14908890115958034,
|
|
"grad_norm": 0.462890625,
|
|
"kl": 0.017445506248623132,
|
|
"learning_rate": 8.509110988404198e-06,
|
|
"loss": 0.0007,
|
|
"reward": 1.05,
|
|
"reward_std": 0.05773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 540
|
|
},
|
|
{
|
|
"completion_length": 43.45,
|
|
"epoch": 0.1518498067366096,
|
|
"grad_norm": 0.6875,
|
|
"kl": 0.029544192645698786,
|
|
"learning_rate": 8.481501932633905e-06,
|
|
"loss": 0.0012,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 550
|
|
},
|
|
{
|
|
"completion_length": 52.625,
|
|
"epoch": 0.15461071231363888,
|
|
"grad_norm": 0.052001953125,
|
|
"kl": 0.0027885228249942885,
|
|
"learning_rate": 8.453892876863612e-06,
|
|
"loss": 0.0001,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 560
|
|
},
|
|
{
|
|
"completion_length": 69.475,
|
|
"epoch": 0.15737161789066814,
|
|
"grad_norm": 0.126953125,
|
|
"kl": 0.0009036411240231246,
|
|
"learning_rate": 8.42628382109332e-06,
|
|
"loss": 0.0,
|
|
"reward": 1.0,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 570
|
|
},
|
|
{
|
|
"completion_length": 46.75,
|
|
"epoch": 0.1601325234676974,
|
|
"grad_norm": 0.1201171875,
|
|
"kl": 0.0012649911106564105,
|
|
"learning_rate": 8.398674765323027e-06,
|
|
"loss": 0.0001,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 580
|
|
},
|
|
{
|
|
"completion_length": 57.325,
|
|
"epoch": 0.16289342904472667,
|
|
"grad_norm": 0.11669921875,
|
|
"kl": 0.0012186030071461573,
|
|
"learning_rate": 8.371065709552733e-06,
|
|
"loss": 0.0,
|
|
"reward": 1.1,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.1,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 590
|
|
},
|
|
{
|
|
"completion_length": 54.925,
|
|
"epoch": 0.16565433462175594,
|
|
"grad_norm": 0.10595703125,
|
|
"kl": 0.0020359954505693167,
|
|
"learning_rate": 8.343456653782442e-06,
|
|
"loss": 0.0001,
|
|
"reward": 1.075,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 600
|
|
},
|
|
{
|
|
"completion_length": 45.775,
|
|
"epoch": 0.1684152401987852,
|
|
"grad_norm": 0.1396484375,
|
|
"kl": 0.0013287237728945912,
|
|
"learning_rate": 8.315847598012148e-06,
|
|
"loss": 0.0001,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 610
|
|
},
|
|
{
|
|
"completion_length": 57.35,
|
|
"epoch": 0.17117614577581447,
|
|
"grad_norm": 13.3125,
|
|
"kl": 0.0009392780877533369,
|
|
"learning_rate": 8.288238542241857e-06,
|
|
"loss": 0.0,
|
|
"reward": 1.05,
|
|
"reward_std": 0.05773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 620
|
|
},
|
|
{
|
|
"completion_length": 61.025,
|
|
"epoch": 0.17393705135284374,
|
|
"grad_norm": 0.1376953125,
|
|
"kl": 0.0012591792619787157,
|
|
"learning_rate": 8.260629486471563e-06,
|
|
"loss": 0.0001,
|
|
"reward": 1.075,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 630
|
|
},
|
|
{
|
|
"completion_length": 42.275,
|
|
"epoch": 0.176697956929873,
|
|
"grad_norm": 0.1533203125,
|
|
"kl": 0.0016204892890527844,
|
|
"learning_rate": 8.23302043070127e-06,
|
|
"loss": 0.0001,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 640
|
|
},
|
|
{
|
|
"completion_length": 57.25,
|
|
"epoch": 0.17945886250690227,
|
|
"grad_norm": 0.19140625,
|
|
"kl": 0.0028853721683844923,
|
|
"learning_rate": 8.205411374930978e-06,
|
|
"loss": 0.0001,
|
|
"reward": 0.975,
|
|
"reward_std": 0.15,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 0.95,
|
|
"step": 650
|
|
},
|
|
{
|
|
"completion_length": 50.125,
|
|
"epoch": 0.18221976808393153,
|
|
"grad_norm": 0.189453125,
|
|
"kl": 0.002419785177335143,
|
|
"learning_rate": 8.177802319160687e-06,
|
|
"loss": 0.0001,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 660
|
|
},
|
|
{
|
|
"completion_length": 55.85,
|
|
"epoch": 0.1849806736609608,
|
|
"grad_norm": 0.1474609375,
|
|
"kl": 0.0022337425500154494,
|
|
"learning_rate": 8.150193263390392e-06,
|
|
"loss": 0.0001,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 670
|
|
},
|
|
{
|
|
"completion_length": 51.05,
|
|
"epoch": 0.18774157923799006,
|
|
"grad_norm": 0.2099609375,
|
|
"kl": 0.0033827337785623966,
|
|
"learning_rate": 8.1225842076201e-06,
|
|
"loss": 0.0001,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 680
|
|
},
|
|
{
|
|
"completion_length": 43.275,
|
|
"epoch": 0.19050248481501933,
|
|
"grad_norm": 0.5078125,
|
|
"kl": 0.01497054547071457,
|
|
"learning_rate": 8.094975151849808e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 690
|
|
},
|
|
{
|
|
"completion_length": 42.1,
|
|
"epoch": 0.1932633903920486,
|
|
"grad_norm": 12.0625,
|
|
"kl": 0.013852290622889996,
|
|
"learning_rate": 8.067366096079515e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.05,
|
|
"reward_std": 0.15773502588272095,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 700
|
|
},
|
|
{
|
|
"completion_length": 53.15,
|
|
"epoch": 0.19602429596907786,
|
|
"grad_norm": 19.625,
|
|
"kl": 0.013076470326632261,
|
|
"learning_rate": 8.039757040309222e-06,
|
|
"loss": 0.0005,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 710
|
|
},
|
|
{
|
|
"completion_length": 56.225,
|
|
"epoch": 0.19878520154610713,
|
|
"grad_norm": 0.1669921875,
|
|
"kl": 0.014432823657989502,
|
|
"learning_rate": 8.01214798453893e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.0,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 720
|
|
},
|
|
{
|
|
"completion_length": 46.8,
|
|
"epoch": 0.2015461071231364,
|
|
"grad_norm": 16.5,
|
|
"kl": 0.013125162106007338,
|
|
"learning_rate": 7.984538928768637e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.05,
|
|
"reward_std": 0.05773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 730
|
|
},
|
|
{
|
|
"completion_length": 47.425,
|
|
"epoch": 0.20430701270016566,
|
|
"grad_norm": 17.125,
|
|
"kl": 0.010656389640644193,
|
|
"learning_rate": 7.956929872998345e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.0,
|
|
"reward_std": 0.2,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 0.95,
|
|
"step": 740
|
|
},
|
|
{
|
|
"completion_length": 44.45,
|
|
"epoch": 0.20706791827719492,
|
|
"grad_norm": 0.2734375,
|
|
"kl": 0.014079915639013053,
|
|
"learning_rate": 7.929320817228052e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.1,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.1,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 750
|
|
},
|
|
{
|
|
"completion_length": 58.25,
|
|
"epoch": 0.2098288238542242,
|
|
"grad_norm": 0.1123046875,
|
|
"kl": 0.013165965024381877,
|
|
"learning_rate": 7.901711761457758e-06,
|
|
"loss": 0.0005,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 760
|
|
},
|
|
{
|
|
"completion_length": 50.35,
|
|
"epoch": 0.21258972943125345,
|
|
"grad_norm": 0.53125,
|
|
"kl": 0.02708628848195076,
|
|
"learning_rate": 7.874102705687467e-06,
|
|
"loss": 0.0011,
|
|
"reward": 1.075,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.1,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 770
|
|
},
|
|
{
|
|
"completion_length": 49.125,
|
|
"epoch": 0.21535063500828272,
|
|
"grad_norm": 0.302734375,
|
|
"kl": 0.006959504983387888,
|
|
"learning_rate": 7.846493649917174e-06,
|
|
"loss": 0.0003,
|
|
"reward": 1.05,
|
|
"reward_std": 0.05773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 780
|
|
},
|
|
{
|
|
"completion_length": 53.575,
|
|
"epoch": 0.21811154058531199,
|
|
"grad_norm": 0.1923828125,
|
|
"kl": 0.008690130175091326,
|
|
"learning_rate": 7.81888459414688e-06,
|
|
"loss": 0.0003,
|
|
"reward": 1.0,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 790
|
|
},
|
|
{
|
|
"completion_length": 48.7,
|
|
"epoch": 0.22087244616234125,
|
|
"grad_norm": 0.234375,
|
|
"kl": 0.01659679235890508,
|
|
"learning_rate": 7.791275538376589e-06,
|
|
"loss": 0.0007,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 800
|
|
},
|
|
{
|
|
"completion_length": 39.625,
|
|
"epoch": 0.22363335173937052,
|
|
"grad_norm": 0.244140625,
|
|
"kl": 0.010390093945898116,
|
|
"learning_rate": 7.763666482606295e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 810
|
|
},
|
|
{
|
|
"completion_length": 62.025,
|
|
"epoch": 0.22639425731639978,
|
|
"grad_norm": 0.154296875,
|
|
"kl": 0.009211322665214539,
|
|
"learning_rate": 7.736057426836004e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 820
|
|
},
|
|
{
|
|
"completion_length": 53.2,
|
|
"epoch": 0.22915516289342905,
|
|
"grad_norm": 0.263671875,
|
|
"kl": 0.011441088351421057,
|
|
"learning_rate": 7.70844837106571e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 830
|
|
},
|
|
{
|
|
"completion_length": 51.25,
|
|
"epoch": 0.23191606847045831,
|
|
"grad_norm": 0.2578125,
|
|
"kl": 0.00715010860003531,
|
|
"learning_rate": 7.680839315295417e-06,
|
|
"loss": 0.0003,
|
|
"reward": 1.075,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 840
|
|
},
|
|
{
|
|
"completion_length": 52.925,
|
|
"epoch": 0.23467697404748758,
|
|
"grad_norm": 0.34765625,
|
|
"kl": 0.010182457882910967,
|
|
"learning_rate": 7.653230259525125e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.05,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 850
|
|
},
|
|
{
|
|
"completion_length": 51.9,
|
|
"epoch": 0.23743787962451685,
|
|
"grad_norm": 0.279296875,
|
|
"kl": 0.009262909227982164,
|
|
"learning_rate": 7.625621203754833e-06,
|
|
"loss": 0.0004,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 860
|
|
},
|
|
{
|
|
"completion_length": 47.1,
|
|
"epoch": 0.2401987852015461,
|
|
"grad_norm": 0.1201171875,
|
|
"kl": 0.014673679834231734,
|
|
"learning_rate": 7.5980121479845395e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.075,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 870
|
|
},
|
|
{
|
|
"completion_length": 43.425,
|
|
"epoch": 0.24295969077857538,
|
|
"grad_norm": 0.291015625,
|
|
"kl": 0.019366781413555145,
|
|
"learning_rate": 7.570403092214247e-06,
|
|
"loss": 0.0008,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 880
|
|
},
|
|
{
|
|
"completion_length": 59.625,
|
|
"epoch": 0.24572059635560464,
|
|
"grad_norm": 0.5625,
|
|
"kl": 0.011405015597119927,
|
|
"learning_rate": 7.5427940364439545e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.0,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 890
|
|
},
|
|
{
|
|
"completion_length": 56.85,
|
|
"epoch": 0.2484815019326339,
|
|
"grad_norm": 0.3515625,
|
|
"kl": 0.012598360935226083,
|
|
"learning_rate": 7.515184980673662e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 900
|
|
},
|
|
{
|
|
"completion_length": 42.475,
|
|
"epoch": 0.25124240750966315,
|
|
"grad_norm": 19.125,
|
|
"kl": 0.013188354019075633,
|
|
"learning_rate": 7.487575924903369e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.15,
|
|
"reward_std": 0.15773502588272095,
|
|
"rewards/accuracy_reward": 0.15,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 910
|
|
},
|
|
{
|
|
"completion_length": 58.5,
|
|
"epoch": 0.25400331308669244,
|
|
"grad_norm": 10.625,
|
|
"kl": 0.024910295475274326,
|
|
"learning_rate": 7.459966869133076e-06,
|
|
"loss": 0.001,
|
|
"reward": 1.0,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 920
|
|
},
|
|
{
|
|
"completion_length": 47.925,
|
|
"epoch": 0.2567642186637217,
|
|
"grad_norm": 9.125,
|
|
"kl": 0.013725518900901078,
|
|
"learning_rate": 7.432357813362784e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.05,
|
|
"reward_std": 0.05773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 930
|
|
},
|
|
{
|
|
"completion_length": 42.125,
|
|
"epoch": 0.25952512424075097,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.01066298601217568,
|
|
"learning_rate": 7.404748757592491e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.0,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 940
|
|
},
|
|
{
|
|
"completion_length": 44.525,
|
|
"epoch": 0.2622860298177802,
|
|
"grad_norm": 0.2578125,
|
|
"kl": 0.010468068346381188,
|
|
"learning_rate": 7.377139701822198e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 950
|
|
},
|
|
{
|
|
"completion_length": 53.225,
|
|
"epoch": 0.2650469353948095,
|
|
"grad_norm": 0.09521484375,
|
|
"kl": 0.009685284015722573,
|
|
"learning_rate": 7.3495306460519054e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.0,
|
|
"reward_std": 0.08164966106414795,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 960
|
|
},
|
|
{
|
|
"completion_length": 45.075,
|
|
"epoch": 0.26780784097183874,
|
|
"grad_norm": 0.3046875,
|
|
"kl": 0.01133408104069531,
|
|
"learning_rate": 7.321921590281613e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.05,
|
|
"reward_std": 0.05773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 970
|
|
},
|
|
{
|
|
"completion_length": 53.85,
|
|
"epoch": 0.27056874654886803,
|
|
"grad_norm": 8.4375,
|
|
"kl": 0.009008155239280313,
|
|
"learning_rate": 7.2943125345113205e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.0,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 980
|
|
},
|
|
{
|
|
"completion_length": 55.875,
|
|
"epoch": 0.27332965212589727,
|
|
"grad_norm": 0.2734375,
|
|
"kl": 0.01266736425459385,
|
|
"learning_rate": 7.266703478741027e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 990
|
|
},
|
|
{
|
|
"completion_length": 60.4,
|
|
"epoch": 0.27609055770292656,
|
|
"grad_norm": 0.33984375,
|
|
"kl": 0.009570520836859941,
|
|
"learning_rate": 7.239094422970735e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.0,
|
|
"reward_std": 0.2,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 0.95,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"completion_length": 41.35,
|
|
"epoch": 0.2788514632799558,
|
|
"grad_norm": 0.21484375,
|
|
"kl": 0.017147039994597435,
|
|
"learning_rate": 7.211485367200442e-06,
|
|
"loss": 0.0007,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1010
|
|
},
|
|
{
|
|
"completion_length": 56.6,
|
|
"epoch": 0.2816123688569851,
|
|
"grad_norm": 0.248046875,
|
|
"kl": 0.010259191761724651,
|
|
"learning_rate": 7.1838763114301505e-06,
|
|
"loss": 0.0004,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"completion_length": 46.375,
|
|
"epoch": 0.28437327443401433,
|
|
"grad_norm": 14.25,
|
|
"kl": 0.012054991768673062,
|
|
"learning_rate": 7.156267255659856e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.125,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.15,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1030
|
|
},
|
|
{
|
|
"completion_length": 59.975,
|
|
"epoch": 0.2871341800110436,
|
|
"grad_norm": 0.1884765625,
|
|
"kl": 0.00799824194982648,
|
|
"learning_rate": 7.128658199889565e-06,
|
|
"loss": 0.0003,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"completion_length": 48.4,
|
|
"epoch": 0.28989508558807286,
|
|
"grad_norm": 0.12451171875,
|
|
"kl": 0.00863449857570231,
|
|
"learning_rate": 7.101049144119272e-06,
|
|
"loss": 0.0003,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1050
|
|
},
|
|
{
|
|
"completion_length": 46.525,
|
|
"epoch": 0.29265599116510216,
|
|
"grad_norm": 0.443359375,
|
|
"kl": 0.015027077682316304,
|
|
"learning_rate": 7.07344008834898e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"completion_length": 52.5,
|
|
"epoch": 0.2954168967421314,
|
|
"grad_norm": 0.216796875,
|
|
"kl": 0.023942758096382022,
|
|
"learning_rate": 9.97239094422971e-06,
|
|
"loss": 0.001,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1070
|
|
},
|
|
{
|
|
"completion_length": 43.475,
|
|
"epoch": 0.2981778023191607,
|
|
"grad_norm": 2.5625,
|
|
"kl": 0.022519207373261452,
|
|
"learning_rate": 9.944781888459416e-06,
|
|
"loss": 0.0009,
|
|
"reward": 1.125,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.125,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"completion_length": 39.875,
|
|
"epoch": 0.3009387078961899,
|
|
"grad_norm": 0.173828125,
|
|
"kl": 0.015989596396684645,
|
|
"learning_rate": 9.917172832689123e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1090
|
|
},
|
|
{
|
|
"completion_length": 53.975,
|
|
"epoch": 0.3036996134732192,
|
|
"grad_norm": 25.125,
|
|
"kl": 0.013253817660734057,
|
|
"learning_rate": 9.889563776918831e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.075,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"completion_length": 46.475,
|
|
"epoch": 0.30646051905024846,
|
|
"grad_norm": 0.193359375,
|
|
"kl": 0.015551709942519665,
|
|
"learning_rate": 9.861954721148538e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.05,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1110
|
|
},
|
|
{
|
|
"completion_length": 41.15,
|
|
"epoch": 0.30922142462727775,
|
|
"grad_norm": 0.6796875,
|
|
"kl": 0.01951191038824618,
|
|
"learning_rate": 9.834345665378246e-06,
|
|
"loss": 0.0008,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"completion_length": 54.9,
|
|
"epoch": 0.311982330204307,
|
|
"grad_norm": 14.4375,
|
|
"kl": 0.020035648718476295,
|
|
"learning_rate": 9.806736609607953e-06,
|
|
"loss": 0.0008,
|
|
"reward": 1.15,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.15,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1130
|
|
},
|
|
{
|
|
"completion_length": 47.95,
|
|
"epoch": 0.3147432357813363,
|
|
"grad_norm": 11.375,
|
|
"kl": 0.014760071225464345,
|
|
"learning_rate": 9.77912755383766e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"completion_length": 51.55,
|
|
"epoch": 0.3175041413583655,
|
|
"grad_norm": 0.30859375,
|
|
"kl": 0.013024270394816994,
|
|
"learning_rate": 9.751518498067368e-06,
|
|
"loss": 0.0005,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1150
|
|
},
|
|
{
|
|
"completion_length": 42.075,
|
|
"epoch": 0.3202650469353948,
|
|
"grad_norm": 0.08837890625,
|
|
"kl": 0.009475133288651705,
|
|
"learning_rate": 9.723909442297074e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"completion_length": 54.35,
|
|
"epoch": 0.32302595251242405,
|
|
"grad_norm": 14.0625,
|
|
"kl": 0.012639770423993469,
|
|
"learning_rate": 9.696300386526781e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.0,
|
|
"reward_std": 0.1154700517654419,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 0.95,
|
|
"step": 1170
|
|
},
|
|
{
|
|
"completion_length": 43.6,
|
|
"epoch": 0.32578685808945335,
|
|
"grad_norm": 0.1875,
|
|
"kl": 0.02179276868700981,
|
|
"learning_rate": 9.66869133075649e-06,
|
|
"loss": 0.0009,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"completion_length": 43.725,
|
|
"epoch": 0.3285477636664826,
|
|
"grad_norm": 0.1689453125,
|
|
"kl": 0.014609583420678973,
|
|
"learning_rate": 9.641082274986196e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.075,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1190
|
|
},
|
|
{
|
|
"completion_length": 56.45,
|
|
"epoch": 0.3313086692435119,
|
|
"grad_norm": 0.205078125,
|
|
"kl": 0.01842534001916647,
|
|
"learning_rate": 9.613473219215904e-06,
|
|
"loss": 0.0007,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"completion_length": 42.975,
|
|
"epoch": 0.3340695748205411,
|
|
"grad_norm": 0.330078125,
|
|
"kl": 0.02300509326159954,
|
|
"learning_rate": 9.585864163445611e-06,
|
|
"loss": 0.0009,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1210
|
|
},
|
|
{
|
|
"completion_length": 41.375,
|
|
"epoch": 0.3368304803975704,
|
|
"grad_norm": 0.2421875,
|
|
"kl": 0.011269495077431201,
|
|
"learning_rate": 9.558255107675318e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.05,
|
|
"reward_std": 0.05773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"completion_length": 47.8,
|
|
"epoch": 0.33959138597459965,
|
|
"grad_norm": 0.34375,
|
|
"kl": 0.019025292107835413,
|
|
"learning_rate": 9.530646051905026e-06,
|
|
"loss": 0.0008,
|
|
"reward": 0.95,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.95,
|
|
"step": 1230
|
|
},
|
|
{
|
|
"completion_length": 39.875,
|
|
"epoch": 0.34235229155162894,
|
|
"grad_norm": 0.60546875,
|
|
"kl": 0.016004760656505822,
|
|
"learning_rate": 9.503036996134733e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"completion_length": 47.925,
|
|
"epoch": 0.3451131971286582,
|
|
"grad_norm": 19.375,
|
|
"kl": 0.021373574901372195,
|
|
"learning_rate": 9.47542794036444e-06,
|
|
"loss": 0.0009,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1250
|
|
},
|
|
{
|
|
"completion_length": 53.9,
|
|
"epoch": 0.34787410270568747,
|
|
"grad_norm": 0.16015625,
|
|
"kl": 0.013111930852755905,
|
|
"learning_rate": 9.447818884594148e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.075,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.1,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"completion_length": 53.8,
|
|
"epoch": 0.3506350082827167,
|
|
"grad_norm": 0.2236328125,
|
|
"kl": 0.017734084371477367,
|
|
"learning_rate": 9.420209828823854e-06,
|
|
"loss": 0.0007,
|
|
"reward": 1.05,
|
|
"reward_std": 0.05773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1270
|
|
},
|
|
{
|
|
"completion_length": 45.125,
|
|
"epoch": 0.353395913859746,
|
|
"grad_norm": 0.326171875,
|
|
"kl": 0.018635424505919217,
|
|
"learning_rate": 9.392600773053563e-06,
|
|
"loss": 0.0007,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"completion_length": 37.35,
|
|
"epoch": 0.35615681943677524,
|
|
"grad_norm": 0.330078125,
|
|
"kl": 0.009210627479478716,
|
|
"learning_rate": 9.36499171728327e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.05,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1290
|
|
},
|
|
{
|
|
"completion_length": 50.85,
|
|
"epoch": 0.35891772501380453,
|
|
"grad_norm": 0.470703125,
|
|
"kl": 0.028195956349372865,
|
|
"learning_rate": 9.337382661512978e-06,
|
|
"loss": 0.0011,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"completion_length": 64.925,
|
|
"epoch": 0.36167863059083377,
|
|
"grad_norm": 0.173828125,
|
|
"kl": 0.01299036997370422,
|
|
"learning_rate": 9.309773605742684e-06,
|
|
"loss": 0.0005,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1310
|
|
},
|
|
{
|
|
"completion_length": 48.3,
|
|
"epoch": 0.36443953616786307,
|
|
"grad_norm": 0.2314453125,
|
|
"kl": 0.013318869331851602,
|
|
"learning_rate": 9.282164549972393e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"completion_length": 43.675,
|
|
"epoch": 0.3672004417448923,
|
|
"grad_norm": 12.375,
|
|
"kl": 0.021023075561970472,
|
|
"learning_rate": 9.2545554942021e-06,
|
|
"loss": 0.0008,
|
|
"reward": 1.05,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1330
|
|
},
|
|
{
|
|
"completion_length": 50.0,
|
|
"epoch": 0.3699613473219216,
|
|
"grad_norm": 0.1865234375,
|
|
"kl": 0.018457670323550703,
|
|
"learning_rate": 9.226946438431806e-06,
|
|
"loss": 0.0007,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"completion_length": 38.625,
|
|
"epoch": 0.37272225289895083,
|
|
"grad_norm": 0.265625,
|
|
"kl": 0.03328995956107974,
|
|
"learning_rate": 9.199337382661515e-06,
|
|
"loss": 0.0013,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1350
|
|
},
|
|
{
|
|
"completion_length": 55.825,
|
|
"epoch": 0.3754831584759801,
|
|
"grad_norm": 0.29296875,
|
|
"kl": 0.013746723486110567,
|
|
"learning_rate": 9.171728326891221e-06,
|
|
"loss": 0.0005,
|
|
"reward": 0.975,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 0.95,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"completion_length": 47.65,
|
|
"epoch": 0.37824406405300937,
|
|
"grad_norm": 0.21484375,
|
|
"kl": 0.01800147187896073,
|
|
"learning_rate": 9.144119271120928e-06,
|
|
"loss": 0.0007,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1370
|
|
},
|
|
{
|
|
"completion_length": 44.6,
|
|
"epoch": 0.38100496963003866,
|
|
"grad_norm": 0.275390625,
|
|
"kl": 0.01803777916356921,
|
|
"learning_rate": 9.116510215350636e-06,
|
|
"loss": 0.0007,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"completion_length": 54.575,
|
|
"epoch": 0.3837658752070679,
|
|
"grad_norm": 0.46484375,
|
|
"kl": 0.010938380844891071,
|
|
"learning_rate": 9.088901159580343e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1390
|
|
},
|
|
{
|
|
"completion_length": 47.55,
|
|
"epoch": 0.3865267807840972,
|
|
"grad_norm": 0.1748046875,
|
|
"kl": 0.01793263861909509,
|
|
"learning_rate": 9.061292103810051e-06,
|
|
"loss": 0.0007,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"completion_length": 42.2,
|
|
"epoch": 0.38928768636112643,
|
|
"grad_norm": 0.27734375,
|
|
"kl": 0.015857995301485062,
|
|
"learning_rate": 9.033683048039758e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1410
|
|
},
|
|
{
|
|
"completion_length": 50.9,
|
|
"epoch": 0.3920485919381557,
|
|
"grad_norm": 0.1875,
|
|
"kl": 0.01180995129980147,
|
|
"learning_rate": 9.006073992269465e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.05,
|
|
"reward_std": 0.05773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"completion_length": 45.775,
|
|
"epoch": 0.39480949751518496,
|
|
"grad_norm": 15.75,
|
|
"kl": 0.012839672528207302,
|
|
"learning_rate": 8.978464936499173e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1430
|
|
},
|
|
{
|
|
"completion_length": 60.45,
|
|
"epoch": 0.39757040309221425,
|
|
"grad_norm": 0.50390625,
|
|
"kl": 0.014541534706950188,
|
|
"learning_rate": 8.95085588072888e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"completion_length": 48.25,
|
|
"epoch": 0.4003313086692435,
|
|
"grad_norm": 0.435546875,
|
|
"kl": 0.013137860433198511,
|
|
"learning_rate": 8.923246824958586e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1450
|
|
},
|
|
{
|
|
"completion_length": 55.175,
|
|
"epoch": 0.4030922142462728,
|
|
"grad_norm": 18.875,
|
|
"kl": 0.018580901995301248,
|
|
"learning_rate": 8.895637769188295e-06,
|
|
"loss": 0.0007,
|
|
"reward": 1.125,
|
|
"reward_std": 0.20773502588272094,
|
|
"rewards/accuracy_reward": 0.15,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"completion_length": 45.825,
|
|
"epoch": 0.405853119823302,
|
|
"grad_norm": 0.203125,
|
|
"kl": 0.019587689847685396,
|
|
"learning_rate": 8.868028713418001e-06,
|
|
"loss": 0.0008,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1470
|
|
},
|
|
{
|
|
"completion_length": 44.4,
|
|
"epoch": 0.4086140254003313,
|
|
"grad_norm": 0.28515625,
|
|
"kl": 0.015912714693695307,
|
|
"learning_rate": 8.84041965764771e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"completion_length": 61.825,
|
|
"epoch": 0.41137493097736055,
|
|
"grad_norm": 0.38671875,
|
|
"kl": 0.015045941760763525,
|
|
"learning_rate": 8.812810601877416e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.05,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1490
|
|
},
|
|
{
|
|
"completion_length": 40.4,
|
|
"epoch": 0.41413583655438985,
|
|
"grad_norm": 14.25,
|
|
"kl": 0.015176839521154761,
|
|
"learning_rate": 8.785201546107123e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.025,
|
|
"reward_std": 0.15,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"completion_length": 66.775,
|
|
"epoch": 0.4168967421314191,
|
|
"grad_norm": 0.373046875,
|
|
"kl": 0.013549512764438987,
|
|
"learning_rate": 8.757592490336831e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.0,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1510
|
|
},
|
|
{
|
|
"completion_length": 58.625,
|
|
"epoch": 0.4196576477084484,
|
|
"grad_norm": 0.224609375,
|
|
"kl": 0.019057274051010608,
|
|
"learning_rate": 8.72998343456654e-06,
|
|
"loss": 0.0008,
|
|
"reward": 1.075,
|
|
"reward_std": 0.15,
|
|
"rewards/accuracy_reward": 0.1,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"completion_length": 61.15,
|
|
"epoch": 0.4224185532854776,
|
|
"grad_norm": 0.7265625,
|
|
"kl": 0.02888321438804269,
|
|
"learning_rate": 8.702374378796245e-06,
|
|
"loss": 0.0012,
|
|
"reward": 1.05,
|
|
"reward_std": 0.1393846869468689,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1530
|
|
},
|
|
{
|
|
"completion_length": 62.375,
|
|
"epoch": 0.4251794588625069,
|
|
"grad_norm": 0.1630859375,
|
|
"kl": 0.014266025042161345,
|
|
"learning_rate": 8.674765323025953e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"completion_length": 69.3,
|
|
"epoch": 0.42794036443953615,
|
|
"grad_norm": 0.28125,
|
|
"kl": 0.015173655841499567,
|
|
"learning_rate": 8.647156267255661e-06,
|
|
"loss": 0.0006,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1550
|
|
},
|
|
{
|
|
"completion_length": 49.8,
|
|
"epoch": 0.43070127001656544,
|
|
"grad_norm": 0.189453125,
|
|
"kl": 0.015002421289682388,
|
|
"learning_rate": 8.619547211485368e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"completion_length": 53.725,
|
|
"epoch": 0.4334621755935947,
|
|
"grad_norm": 14.875,
|
|
"kl": 0.0250665825791657,
|
|
"learning_rate": 8.591938155715075e-06,
|
|
"loss": 0.001,
|
|
"reward": 1.0,
|
|
"reward_std": 0.15773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 0.95,
|
|
"step": 1570
|
|
},
|
|
{
|
|
"completion_length": 37.4,
|
|
"epoch": 0.43622308117062397,
|
|
"grad_norm": 0.44140625,
|
|
"kl": 0.020354483788833023,
|
|
"learning_rate": 8.564329099944783e-06,
|
|
"loss": 0.0008,
|
|
"reward": 1.1,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.1,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"completion_length": 62.6,
|
|
"epoch": 0.4389839867476532,
|
|
"grad_norm": 14.6875,
|
|
"kl": 0.014426431572064757,
|
|
"learning_rate": 8.53672004417449e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.075,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1590
|
|
},
|
|
{
|
|
"completion_length": 41.65,
|
|
"epoch": 0.4417448923246825,
|
|
"grad_norm": 0.4609375,
|
|
"kl": 0.013243564590811729,
|
|
"learning_rate": 8.509110988404198e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.225,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.225,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"completion_length": 52.45,
|
|
"epoch": 0.44450579790171174,
|
|
"grad_norm": 1.03125,
|
|
"kl": 0.01361188031733036,
|
|
"learning_rate": 8.481501932633905e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1610
|
|
},
|
|
{
|
|
"completion_length": 50.15,
|
|
"epoch": 0.44726670347874103,
|
|
"grad_norm": 17.375,
|
|
"kl": 0.018944200593978167,
|
|
"learning_rate": 8.453892876863612e-06,
|
|
"loss": 0.0008,
|
|
"reward": 1.075,
|
|
"reward_std": 0.15,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"completion_length": 49.65,
|
|
"epoch": 0.4500276090557703,
|
|
"grad_norm": 16.75,
|
|
"kl": 0.015870027616620062,
|
|
"learning_rate": 8.42628382109332e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.075,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1630
|
|
},
|
|
{
|
|
"completion_length": 78.3,
|
|
"epoch": 0.45278851463279957,
|
|
"grad_norm": 0.06494140625,
|
|
"kl": 0.017166434228420256,
|
|
"learning_rate": 8.398674765323027e-06,
|
|
"loss": 0.0007,
|
|
"reward": 1.025,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"completion_length": 58.325,
|
|
"epoch": 0.4555494202098288,
|
|
"grad_norm": 0.318359375,
|
|
"kl": 0.012359794043004513,
|
|
"learning_rate": 8.371065709552733e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.05,
|
|
"reward_std": 0.05773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1650
|
|
},
|
|
{
|
|
"completion_length": 56.5,
|
|
"epoch": 0.4583103257868581,
|
|
"grad_norm": 0.244140625,
|
|
"kl": 0.011522294348105788,
|
|
"learning_rate": 8.343456653782442e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.1,
|
|
"reward_std": 0.1154700517654419,
|
|
"rewards/accuracy_reward": 0.1,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"completion_length": 59.55,
|
|
"epoch": 0.46107123136388733,
|
|
"grad_norm": 0.3125,
|
|
"kl": 0.013884695107117295,
|
|
"learning_rate": 8.315847598012148e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.05,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.1,
|
|
"rewards/format_reward": 0.95,
|
|
"step": 1670
|
|
},
|
|
{
|
|
"completion_length": 44.025,
|
|
"epoch": 0.46383213694091663,
|
|
"grad_norm": 0.380859375,
|
|
"kl": 0.012195908557623624,
|
|
"learning_rate": 8.288238542241857e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"completion_length": 54.825,
|
|
"epoch": 0.46659304251794587,
|
|
"grad_norm": 0.54296875,
|
|
"kl": 0.01730495747178793,
|
|
"learning_rate": 8.260629486471563e-06,
|
|
"loss": 0.0007,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1690
|
|
},
|
|
{
|
|
"completion_length": 47.55,
|
|
"epoch": 0.46935394809497516,
|
|
"grad_norm": 0.283203125,
|
|
"kl": 0.015025637298822402,
|
|
"learning_rate": 8.23302043070127e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.0,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"completion_length": 52.875,
|
|
"epoch": 0.4721148536720044,
|
|
"grad_norm": 7.78125,
|
|
"kl": 0.012533561326563358,
|
|
"learning_rate": 8.205411374930978e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.075,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1710
|
|
},
|
|
{
|
|
"completion_length": 57.725,
|
|
"epoch": 0.4748757592490337,
|
|
"grad_norm": 0.2294921875,
|
|
"kl": 0.015987796289846302,
|
|
"learning_rate": 8.177802319160687e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"completion_length": 35.375,
|
|
"epoch": 0.47763666482606293,
|
|
"grad_norm": 0.2041015625,
|
|
"kl": 0.01583628305234015,
|
|
"learning_rate": 8.150193263390392e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.075,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1730
|
|
},
|
|
{
|
|
"completion_length": 57.15,
|
|
"epoch": 0.4803975704030922,
|
|
"grad_norm": 0.349609375,
|
|
"kl": 0.013509547524154186,
|
|
"learning_rate": 8.1225842076201e-06,
|
|
"loss": 0.0005,
|
|
"reward": 0.925,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.925,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"completion_length": 43.875,
|
|
"epoch": 0.48315847598012146,
|
|
"grad_norm": 0.38671875,
|
|
"kl": 0.01573203131556511,
|
|
"learning_rate": 8.094975151849808e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1750
|
|
},
|
|
{
|
|
"completion_length": 38.725,
|
|
"epoch": 0.48591938155715075,
|
|
"grad_norm": 0.3984375,
|
|
"kl": 0.015004007937386632,
|
|
"learning_rate": 8.067366096079515e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"completion_length": 51.575,
|
|
"epoch": 0.48868028713418,
|
|
"grad_norm": 0.3359375,
|
|
"kl": 0.012508704885840416,
|
|
"learning_rate": 8.039757040309222e-06,
|
|
"loss": 0.0005,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1770
|
|
},
|
|
{
|
|
"completion_length": 39.95,
|
|
"epoch": 0.4914411927112093,
|
|
"grad_norm": 0.294921875,
|
|
"kl": 0.011702212109230458,
|
|
"learning_rate": 8.01214798453893e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"completion_length": 52.575,
|
|
"epoch": 0.4942020982882385,
|
|
"grad_norm": 0.265625,
|
|
"kl": 0.011571823060512543,
|
|
"learning_rate": 7.984538928768637e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1790
|
|
},
|
|
{
|
|
"completion_length": 65.675,
|
|
"epoch": 0.4969630038652678,
|
|
"grad_norm": 9.1875,
|
|
"kl": 0.009807331394404173,
|
|
"learning_rate": 7.956929872998345e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.0,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"completion_length": 41.25,
|
|
"epoch": 0.49972390944229705,
|
|
"grad_norm": 10.125,
|
|
"kl": 0.012691410770639778,
|
|
"learning_rate": 7.929320817228052e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.05,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1810
|
|
},
|
|
{
|
|
"completion_length": 39.975,
|
|
"epoch": 0.5024848150193263,
|
|
"grad_norm": 0.388671875,
|
|
"kl": 0.012763157673180103,
|
|
"learning_rate": 7.901711761457758e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.1,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.1,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"completion_length": 59.4,
|
|
"epoch": 0.5052457205963556,
|
|
"grad_norm": 0.2890625,
|
|
"kl": 0.012727226875722409,
|
|
"learning_rate": 7.874102705687467e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.075,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1830
|
|
},
|
|
{
|
|
"completion_length": 59.375,
|
|
"epoch": 0.5080066261733849,
|
|
"grad_norm": 9.1875,
|
|
"kl": 0.011757147964090109,
|
|
"learning_rate": 7.846493649917174e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.075,
|
|
"reward_std": 0.25,
|
|
"rewards/accuracy_reward": 0.1,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"completion_length": 42.075,
|
|
"epoch": 0.5107675317504141,
|
|
"grad_norm": 0.1552734375,
|
|
"kl": 0.011294585885480046,
|
|
"learning_rate": 7.81888459414688e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1850
|
|
},
|
|
{
|
|
"completion_length": 51.15,
|
|
"epoch": 0.5135284373274434,
|
|
"grad_norm": 11.9375,
|
|
"kl": 0.011790570057928561,
|
|
"learning_rate": 7.791275538376589e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.025,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"completion_length": 44.25,
|
|
"epoch": 0.5162893429044727,
|
|
"grad_norm": 1.234375,
|
|
"kl": 0.023779562208801508,
|
|
"learning_rate": 7.763666482606295e-06,
|
|
"loss": 0.001,
|
|
"reward": 1.075,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.1,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1870
|
|
},
|
|
{
|
|
"completion_length": 44.45,
|
|
"epoch": 0.5190502484815019,
|
|
"grad_norm": 0.265625,
|
|
"kl": 0.018847981560975315,
|
|
"learning_rate": 7.736057426836004e-06,
|
|
"loss": 0.0008,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"completion_length": 67.5,
|
|
"epoch": 0.5218111540585312,
|
|
"grad_norm": 0.423828125,
|
|
"kl": 0.010858609480783344,
|
|
"learning_rate": 7.70844837106571e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.025,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1890
|
|
},
|
|
{
|
|
"completion_length": 57.375,
|
|
"epoch": 0.5245720596355604,
|
|
"grad_norm": 0.359375,
|
|
"kl": 0.012408159440383315,
|
|
"learning_rate": 7.680839315295417e-06,
|
|
"loss": 0.0005,
|
|
"reward": 0.9,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 0.875,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"completion_length": 41.075,
|
|
"epoch": 0.5273329652125898,
|
|
"grad_norm": 11.375,
|
|
"kl": 0.01915076021105051,
|
|
"learning_rate": 7.653230259525125e-06,
|
|
"loss": 0.0008,
|
|
"reward": 1.1,
|
|
"reward_std": 0.15773502588272095,
|
|
"rewards/accuracy_reward": 0.1,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1910
|
|
},
|
|
{
|
|
"completion_length": 44.75,
|
|
"epoch": 0.530093870789619,
|
|
"grad_norm": 0.134765625,
|
|
"kl": 0.013074626727029682,
|
|
"learning_rate": 7.625621203754833e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"completion_length": 47.875,
|
|
"epoch": 0.5328547763666482,
|
|
"grad_norm": 0.1728515625,
|
|
"kl": 0.022233243193477394,
|
|
"learning_rate": 7.5980121479845395e-06,
|
|
"loss": 0.0009,
|
|
"reward": 1.05,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1930
|
|
},
|
|
{
|
|
"completion_length": 40.375,
|
|
"epoch": 0.5356156819436775,
|
|
"grad_norm": 0.33203125,
|
|
"kl": 0.013019495364278554,
|
|
"learning_rate": 7.570403092214247e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.1,
|
|
"reward_std": 0.2,
|
|
"rewards/accuracy_reward": 0.1,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"completion_length": 50.2,
|
|
"epoch": 0.5383765875207068,
|
|
"grad_norm": 0.265625,
|
|
"kl": 0.013900231663137674,
|
|
"learning_rate": 7.5427940364439545e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.025,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1950
|
|
},
|
|
{
|
|
"completion_length": 44.625,
|
|
"epoch": 0.5411374930977361,
|
|
"grad_norm": 14.125,
|
|
"kl": 0.015454199723899364,
|
|
"learning_rate": 7.515184980673662e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.05,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"completion_length": 52.725,
|
|
"epoch": 0.5438983986747653,
|
|
"grad_norm": 0.1259765625,
|
|
"kl": 0.012598886806517839,
|
|
"learning_rate": 7.487575924903369e-06,
|
|
"loss": 0.0005,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 1970
|
|
},
|
|
{
|
|
"completion_length": 42.975,
|
|
"epoch": 0.5466593042517945,
|
|
"grad_norm": 0.1806640625,
|
|
"kl": 0.01452862734440714,
|
|
"learning_rate": 7.459966869133076e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.2,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.2,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"completion_length": 46.725,
|
|
"epoch": 0.5494202098288239,
|
|
"grad_norm": 0.2236328125,
|
|
"kl": 0.011979705933481456,
|
|
"learning_rate": 7.432357813362784e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 1990
|
|
},
|
|
{
|
|
"completion_length": 62.225,
|
|
"epoch": 0.5521811154058531,
|
|
"grad_norm": 13.8125,
|
|
"kl": 0.009113723039627075,
|
|
"learning_rate": 7.404748757592491e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.05,
|
|
"reward_std": 0.05773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"completion_length": 50.425,
|
|
"epoch": 0.5549420209828824,
|
|
"grad_norm": 0.2333984375,
|
|
"kl": 0.012480269372463226,
|
|
"learning_rate": 7.377139701822198e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.05,
|
|
"reward_std": 0.05773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2010
|
|
},
|
|
{
|
|
"completion_length": 45.65,
|
|
"epoch": 0.5577029265599116,
|
|
"grad_norm": 0.24609375,
|
|
"kl": 0.010359317273832857,
|
|
"learning_rate": 7.3495306460519054e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"completion_length": 54.025,
|
|
"epoch": 0.560463832136941,
|
|
"grad_norm": 0.068359375,
|
|
"kl": 0.010447596153244375,
|
|
"learning_rate": 7.321921590281613e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.1,
|
|
"reward_std": 0.15773502588272095,
|
|
"rewards/accuracy_reward": 0.125,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 2030
|
|
},
|
|
{
|
|
"completion_length": 51.575,
|
|
"epoch": 0.5632247377139702,
|
|
"grad_norm": 0.296875,
|
|
"kl": 0.01125028906390071,
|
|
"learning_rate": 7.2943125345113205e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.075,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.1,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"completion_length": 57.05,
|
|
"epoch": 0.5659856432909994,
|
|
"grad_norm": 5.34375,
|
|
"kl": 0.014221622655168176,
|
|
"learning_rate": 7.266703478741027e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.075,
|
|
"reward_std": 0.13164966106414794,
|
|
"rewards/accuracy_reward": 0.1,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 2050
|
|
},
|
|
{
|
|
"completion_length": 47.65,
|
|
"epoch": 0.5687465488680287,
|
|
"grad_norm": 0.2099609375,
|
|
"kl": 0.011002461006864906,
|
|
"learning_rate": 7.239094422970735e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.125,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.125,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"completion_length": 40.75,
|
|
"epoch": 0.571507454445058,
|
|
"grad_norm": 3.65625,
|
|
"kl": 0.02065484719350934,
|
|
"learning_rate": 7.211485367200442e-06,
|
|
"loss": 0.0008,
|
|
"reward": 1.125,
|
|
"reward_std": 0.16547005176544188,
|
|
"rewards/accuracy_reward": 0.125,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2070
|
|
},
|
|
{
|
|
"completion_length": 54.775,
|
|
"epoch": 0.5742683600220873,
|
|
"grad_norm": 9.75,
|
|
"kl": 0.012140319403260946,
|
|
"learning_rate": 7.1838763114301505e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.1,
|
|
"reward_std": 0.2,
|
|
"rewards/accuracy_reward": 0.1,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"completion_length": 47.2,
|
|
"epoch": 0.5770292655991165,
|
|
"grad_norm": 15.3125,
|
|
"kl": 0.015356668597087264,
|
|
"learning_rate": 7.156267255659856e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.1,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.1,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2090
|
|
},
|
|
{
|
|
"completion_length": 56.45,
|
|
"epoch": 0.5797901711761457,
|
|
"grad_norm": 0.4609375,
|
|
"kl": 0.014882310247048735,
|
|
"learning_rate": 7.128658199889565e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.0,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"completion_length": 51.45,
|
|
"epoch": 0.5825510767531751,
|
|
"grad_norm": 12.1875,
|
|
"kl": 0.013292353972792626,
|
|
"learning_rate": 7.101049144119272e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.05,
|
|
"reward_std": 0.05773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2110
|
|
},
|
|
{
|
|
"completion_length": 34.5,
|
|
"epoch": 0.5853119823302043,
|
|
"grad_norm": 0.2373046875,
|
|
"kl": 0.013515590690076351,
|
|
"learning_rate": 7.07344008834898e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.05,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"completion_length": 45.725,
|
|
"epoch": 0.5880728879072336,
|
|
"grad_norm": 0.1611328125,
|
|
"kl": 0.01667518252506852,
|
|
"learning_rate": 7.045831032578686e-06,
|
|
"loss": 0.0007,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2130
|
|
},
|
|
{
|
|
"completion_length": 55.575,
|
|
"epoch": 0.5908337934842628,
|
|
"grad_norm": 0.75,
|
|
"kl": 0.016457650857046247,
|
|
"learning_rate": 7.018221976808394e-06,
|
|
"loss": 0.0007,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"completion_length": 43.8,
|
|
"epoch": 0.5935946990612921,
|
|
"grad_norm": 0.2255859375,
|
|
"kl": 0.012675747368484735,
|
|
"learning_rate": 6.9906129210381015e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.1,
|
|
"reward_std": 0.1154700517654419,
|
|
"rewards/accuracy_reward": 0.1,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2150
|
|
},
|
|
{
|
|
"completion_length": 50.85,
|
|
"epoch": 0.5963556046383214,
|
|
"grad_norm": 0.4296875,
|
|
"kl": 0.013116476451978087,
|
|
"learning_rate": 6.963003865267809e-06,
|
|
"loss": 0.0005,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"completion_length": 67.65,
|
|
"epoch": 0.5991165102153506,
|
|
"grad_norm": 0.197265625,
|
|
"kl": 0.011186759034171701,
|
|
"learning_rate": 6.935394809497516e-06,
|
|
"loss": 0.0004,
|
|
"reward": 0.95,
|
|
"reward_std": 0.14574271440505981,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 0.925,
|
|
"step": 2170
|
|
},
|
|
{
|
|
"completion_length": 66.6,
|
|
"epoch": 0.6018774157923799,
|
|
"grad_norm": 0.259765625,
|
|
"kl": 0.014577974379062653,
|
|
"learning_rate": 6.907785753727223e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"completion_length": 47.9,
|
|
"epoch": 0.6046383213694092,
|
|
"grad_norm": 0.2021484375,
|
|
"kl": 0.012043908704072237,
|
|
"learning_rate": 6.880176697956931e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2190
|
|
},
|
|
{
|
|
"completion_length": 52.125,
|
|
"epoch": 0.6073992269464384,
|
|
"grad_norm": 0.1630859375,
|
|
"kl": 0.008721905946731567,
|
|
"learning_rate": 6.852567642186638e-06,
|
|
"loss": 0.0003,
|
|
"reward": 1.075,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"completion_length": 46.925,
|
|
"epoch": 0.6101601325234677,
|
|
"grad_norm": 0.31640625,
|
|
"kl": 0.010289201140403747,
|
|
"learning_rate": 6.824958586416345e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.075,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2210
|
|
},
|
|
{
|
|
"completion_length": 48.95,
|
|
"epoch": 0.6129210381004969,
|
|
"grad_norm": 0.2421875,
|
|
"kl": 0.01192173978779465,
|
|
"learning_rate": 6.797349530646052e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.05,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"completion_length": 58.65,
|
|
"epoch": 0.6156819436775263,
|
|
"grad_norm": 12.125,
|
|
"kl": 0.009246612852439284,
|
|
"learning_rate": 6.76974047487576e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.025,
|
|
"reward_std": 0.15,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 2230
|
|
},
|
|
{
|
|
"completion_length": 42.225,
|
|
"epoch": 0.6184428492545555,
|
|
"grad_norm": 0.44140625,
|
|
"kl": 0.016198016284033657,
|
|
"learning_rate": 6.742131419105467e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.075,
|
|
"reward_std": 0.10773502588272095,
|
|
"rewards/accuracy_reward": 0.075,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"completion_length": 64.425,
|
|
"epoch": 0.6212037548315847,
|
|
"grad_norm": 0.1640625,
|
|
"kl": 0.008573147933930159,
|
|
"learning_rate": 6.714522363335174e-06,
|
|
"loss": 0.0003,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 2250
|
|
},
|
|
{
|
|
"completion_length": 42.85,
|
|
"epoch": 0.623964660408614,
|
|
"grad_norm": 0.2431640625,
|
|
"kl": 0.018018354568630456,
|
|
"learning_rate": 6.686913307564882e-06,
|
|
"loss": 0.0007,
|
|
"reward": 1.05,
|
|
"reward_std": 0.05773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"completion_length": 44.275,
|
|
"epoch": 0.6267255659856433,
|
|
"grad_norm": 0.2470703125,
|
|
"kl": 0.01138524515554309,
|
|
"learning_rate": 6.659304251794589e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2270
|
|
},
|
|
{
|
|
"completion_length": 56.425,
|
|
"epoch": 0.6294864715626726,
|
|
"grad_norm": 0.166015625,
|
|
"kl": 0.007961477618664503,
|
|
"learning_rate": 6.631695196024297e-06,
|
|
"loss": 0.0003,
|
|
"reward": 1.05,
|
|
"reward_std": 0.05773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"completion_length": 40.1,
|
|
"epoch": 0.6322473771397018,
|
|
"grad_norm": 0.1474609375,
|
|
"kl": 0.011783124506473541,
|
|
"learning_rate": 6.604086140254003e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2290
|
|
},
|
|
{
|
|
"completion_length": 40.425,
|
|
"epoch": 0.635008282716731,
|
|
"grad_norm": 12.0,
|
|
"kl": 0.010805513197556137,
|
|
"learning_rate": 6.576477084483711e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.05,
|
|
"reward_std": 0.05773502588272095,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"completion_length": 50.275,
|
|
"epoch": 0.6377691882937604,
|
|
"grad_norm": 0.484375,
|
|
"kl": 0.012207349389791488,
|
|
"learning_rate": 6.548868028713419e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.025,
|
|
"reward_std": 0.15,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 2310
|
|
},
|
|
{
|
|
"completion_length": 49.575,
|
|
"epoch": 0.6405300938707896,
|
|
"grad_norm": 0.11767578125,
|
|
"kl": 0.014282464934512972,
|
|
"learning_rate": 6.521258972943127e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2320
|
|
},
|
|
{
|
|
"completion_length": 50.975,
|
|
"epoch": 0.6432909994478189,
|
|
"grad_norm": 0.23828125,
|
|
"kl": 0.010307052405551076,
|
|
"learning_rate": 6.4936499171728325e-06,
|
|
"loss": 0.0004,
|
|
"reward": 0.975,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 2330
|
|
},
|
|
{
|
|
"completion_length": 43.45,
|
|
"epoch": 0.6460519050248481,
|
|
"grad_norm": 0.283203125,
|
|
"kl": 0.009153263340704144,
|
|
"learning_rate": 6.466040861402541e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.05,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2340
|
|
},
|
|
{
|
|
"completion_length": 42.45,
|
|
"epoch": 0.6488128106018775,
|
|
"grad_norm": 0.1669921875,
|
|
"kl": 0.012787223979830741,
|
|
"learning_rate": 6.438431805632248e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2350
|
|
},
|
|
{
|
|
"completion_length": 49.15,
|
|
"epoch": 0.6515737161789067,
|
|
"grad_norm": 0.306640625,
|
|
"kl": 0.011894195433706045,
|
|
"learning_rate": 6.410822749861956e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.05,
|
|
"reward_std": 0.1,
|
|
"rewards/accuracy_reward": 0.05,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2360
|
|
},
|
|
{
|
|
"completion_length": 40.175,
|
|
"epoch": 0.6543346217559359,
|
|
"grad_norm": 0.2177734375,
|
|
"kl": 0.014734916668385268,
|
|
"learning_rate": 6.3832136940916626e-06,
|
|
"loss": 0.0006,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2370
|
|
},
|
|
{
|
|
"completion_length": 45.1,
|
|
"epoch": 0.6570955273329652,
|
|
"grad_norm": 0.27734375,
|
|
"kl": 0.01835952061228454,
|
|
"learning_rate": 6.35560463832137e-06,
|
|
"loss": 0.0007,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2380
|
|
},
|
|
{
|
|
"completion_length": 49.15,
|
|
"epoch": 0.6598564329099945,
|
|
"grad_norm": 16.625,
|
|
"kl": 0.011911587277427315,
|
|
"learning_rate": 6.327995582551078e-06,
|
|
"loss": 0.0005,
|
|
"reward": 1.25,
|
|
"reward_std": 0.25773502588272096,
|
|
"rewards/accuracy_reward": 0.275,
|
|
"rewards/format_reward": 0.975,
|
|
"step": 2390
|
|
},
|
|
{
|
|
"completion_length": 56.825,
|
|
"epoch": 0.6626173384870238,
|
|
"grad_norm": 0.1357421875,
|
|
"kl": 0.006976808630861342,
|
|
"learning_rate": 6.300386526780785e-06,
|
|
"loss": 0.0003,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2400
|
|
},
|
|
{
|
|
"completion_length": 60.475,
|
|
"epoch": 0.665378244064053,
|
|
"grad_norm": 0.2353515625,
|
|
"kl": 0.010746273957192898,
|
|
"learning_rate": 6.272777471010492e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2410
|
|
},
|
|
{
|
|
"completion_length": 39.75,
|
|
"epoch": 0.6681391496410822,
|
|
"grad_norm": 0.37109375,
|
|
"kl": 0.01077571688219905,
|
|
"learning_rate": 6.245168415240199e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2420
|
|
},
|
|
{
|
|
"completion_length": 49.2,
|
|
"epoch": 0.6709000552181116,
|
|
"grad_norm": 0.27734375,
|
|
"kl": 0.010110868490301073,
|
|
"learning_rate": 6.217559359469907e-06,
|
|
"loss": 0.0004,
|
|
"reward": 1.0,
|
|
"reward_std": 0.0,
|
|
"rewards/accuracy_reward": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2430
|
|
},
|
|
{
|
|
"completion_length": 58.1,
|
|
"epoch": 0.6736609607951408,
|
|
"grad_norm": 11.125,
|
|
"kl": 0.007148551405407488,
|
|
"learning_rate": 6.189950303699614e-06,
|
|
"loss": 0.0003,
|
|
"reward": 1.025,
|
|
"reward_std": 0.05,
|
|
"rewards/accuracy_reward": 0.025,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 2440
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 3622,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 10,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 4,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|