4243 lines
132 KiB
JSON
4243 lines
132 KiB
JSON
{
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.2570694087403599,
|
|
"eval_steps": 500,
|
|
"global_step": 300,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3156.8612060546875,
|
|
"epoch": 0.000856898029134533,
|
|
"grad_norm": 0.08138668537139893,
|
|
"kl": 0.0,
|
|
"learning_rate": 3.3333333333333334e-08,
|
|
"loss": 0.0383,
|
|
"reward": -2.583333373069763,
|
|
"reward_std": 0.09128709882497787,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.4166666716337204,
|
|
"step": 1
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3294.638916015625,
|
|
"epoch": 0.001713796058269066,
|
|
"grad_norm": 0.3450618088245392,
|
|
"kl": 0.0,
|
|
"learning_rate": 6.666666666666667e-08,
|
|
"loss": 0.1933,
|
|
"reward": -2.6666667461395264,
|
|
"reward_std": 0.2221490517258644,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3333333432674408,
|
|
"step": 2
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3111.7222900390625,
|
|
"epoch": 0.002570694087403599,
|
|
"grad_norm": 0.4438176453113556,
|
|
"kl": 4.208087921142578e-05,
|
|
"learning_rate": 1e-07,
|
|
"loss": 0.1213,
|
|
"reward": -2.435185194015503,
|
|
"reward_std": 0.46697859466075897,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.4722222238779068,
|
|
"step": 3
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3262.8612060546875,
|
|
"epoch": 0.003427592116538132,
|
|
"grad_norm": 0.1571539044380188,
|
|
"kl": 3.17692756652832e-05,
|
|
"learning_rate": 1.3333333333333334e-07,
|
|
"loss": 0.1405,
|
|
"reward": -2.6944445371627808,
|
|
"reward_std": 0.15410767495632172,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3055555671453476,
|
|
"step": 4
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3076.166748046875,
|
|
"epoch": 0.004284490145672665,
|
|
"grad_norm": 0.0744488388299942,
|
|
"kl": 4.2378902435302734e-05,
|
|
"learning_rate": 1.6666666666666665e-07,
|
|
"loss": 0.0441,
|
|
"reward": -2.4722222089767456,
|
|
"reward_std": 0.15410767495632172,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.5277777910232544,
|
|
"step": 5
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3411.5277099609375,
|
|
"epoch": 0.005141388174807198,
|
|
"grad_norm": 0.1373002976179123,
|
|
"kl": 2.950429916381836e-05,
|
|
"learning_rate": 2e-07,
|
|
"loss": 0.0469,
|
|
"reward": -2.583333373069763,
|
|
"reward_std": 0.33668186515569687,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.361111119389534,
|
|
"step": 6
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3255.5556640625,
|
|
"epoch": 0.005998286203941731,
|
|
"grad_norm": 0.1415766477584839,
|
|
"kl": 3.135204315185547e-05,
|
|
"learning_rate": 2.3333333333333333e-07,
|
|
"loss": 0.0729,
|
|
"reward": -2.5,
|
|
"reward_std": 0.2453947737812996,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.5000000149011612,
|
|
"step": 7
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2561.388916015625,
|
|
"epoch": 0.006855184233076264,
|
|
"grad_norm": 0.3355005085468292,
|
|
"kl": 3.647804260253906e-05,
|
|
"learning_rate": 2.6666666666666667e-07,
|
|
"loss": 0.0923,
|
|
"reward": -2.170351266860962,
|
|
"reward_std": 0.6446175873279572,
|
|
"rewards/confidence_reward": -2.809240221977234,
|
|
"rewards/format_reward": 0.6388888955116272,
|
|
"step": 8
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3178.75,
|
|
"epoch": 0.007712082262210797,
|
|
"grad_norm": 0.2505446970462799,
|
|
"kl": 2.8371810913085938e-05,
|
|
"learning_rate": 3e-07,
|
|
"loss": 0.2008,
|
|
"reward": -2.4722222089767456,
|
|
"reward_std": 0.409944087266922,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.4722222238779068,
|
|
"step": 9
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2501.1944580078125,
|
|
"epoch": 0.00856898029134533,
|
|
"grad_norm": 0.2691773772239685,
|
|
"kl": 2.0295381546020508e-05,
|
|
"learning_rate": 3.333333333333333e-07,
|
|
"loss": 0.1208,
|
|
"reward": -2.4166667461395264,
|
|
"reward_std": 0.15410767495632172,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.5833333283662796,
|
|
"step": 10
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3531.8333740234375,
|
|
"epoch": 0.009425878320479864,
|
|
"grad_norm": 0.1814609318971634,
|
|
"kl": 3.612041473388672e-05,
|
|
"learning_rate": 3.666666666666666e-07,
|
|
"loss": 0.0473,
|
|
"reward": -2.555555582046509,
|
|
"reward_std": 0.3082153648138046,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.4444444552063942,
|
|
"step": 11
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3414.6666259765625,
|
|
"epoch": 0.010282776349614395,
|
|
"grad_norm": 0.14940866827964783,
|
|
"kl": 4.470348358154297e-05,
|
|
"learning_rate": 4e-07,
|
|
"loss": 0.1065,
|
|
"reward": -2.6388888359069824,
|
|
"reward_std": 0.25061557441949844,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3611111268401146,
|
|
"step": 12
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2698.2222900390625,
|
|
"epoch": 0.011139674378748929,
|
|
"grad_norm": 0.193411186337471,
|
|
"kl": 3.445148468017578e-05,
|
|
"learning_rate": 4.3333333333333335e-07,
|
|
"loss": 0.0232,
|
|
"reward": -2.2592592239379883,
|
|
"reward_std": 0.37295398116111755,
|
|
"rewards/confidence_reward": -2.814814805984497,
|
|
"rewards/format_reward": 0.5555555671453476,
|
|
"step": 13
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 1606.9166870117188,
|
|
"epoch": 0.011996572407883462,
|
|
"grad_norm": 0.1921149641275406,
|
|
"kl": 2.7477741241455078e-05,
|
|
"learning_rate": 4.6666666666666666e-07,
|
|
"loss": 0.0873,
|
|
"reward": -1.929610550403595,
|
|
"reward_std": 0.4556204080581665,
|
|
"rewards/confidence_reward": -2.7907216548919678,
|
|
"rewards/format_reward": 0.8611111044883728,
|
|
"step": 14
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2907.9444580078125,
|
|
"epoch": 0.012853470437017995,
|
|
"grad_norm": 0.1692344844341278,
|
|
"kl": 2.485513687133789e-05,
|
|
"learning_rate": 5e-07,
|
|
"loss": 0.0894,
|
|
"reward": -2.305555582046509,
|
|
"reward_std": 0.22736986726522446,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.6944444477558136,
|
|
"step": 15
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2834.5833740234375,
|
|
"epoch": 0.013710368466152529,
|
|
"grad_norm": 0.20961543917655945,
|
|
"kl": 2.6166439056396484e-05,
|
|
"learning_rate": 5.333333333333333e-07,
|
|
"loss": 0.1212,
|
|
"reward": -2.3611111640930176,
|
|
"reward_std": 0.4495189040899277,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.5833333283662796,
|
|
"step": 16
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 1908.2777709960938,
|
|
"epoch": 0.01456726649528706,
|
|
"grad_norm": 0.13441088795661926,
|
|
"kl": 1.5820842236280441e-06,
|
|
"learning_rate": 5.666666666666666e-07,
|
|
"loss": 0.0448,
|
|
"reward": -2.25,
|
|
"reward_std": 0.15410767495632172,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.75,
|
|
"step": 17
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3585.5001220703125,
|
|
"epoch": 0.015424164524421594,
|
|
"grad_norm": 0.23104918003082275,
|
|
"kl": 3.343820571899414e-05,
|
|
"learning_rate": 6e-07,
|
|
"loss": 0.1208,
|
|
"reward": -2.6666667461395264,
|
|
"reward_std": 0.3314610570669174,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3333333432674408,
|
|
"step": 18
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2827.1943969726562,
|
|
"epoch": 0.016281062553556127,
|
|
"grad_norm": 0.24534867703914642,
|
|
"kl": 2.0720064640045166e-05,
|
|
"learning_rate": 6.333333333333332e-07,
|
|
"loss": 0.0672,
|
|
"reward": -2.555555582046509,
|
|
"reward_std": 0.245394766330719,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.4444444477558136,
|
|
"step": 19
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 4031.5556640625,
|
|
"epoch": 0.01713796058269066,
|
|
"grad_norm": 0.14566344022750854,
|
|
"kl": 3.6835670471191406e-05,
|
|
"learning_rate": 6.666666666666666e-07,
|
|
"loss": 0.0251,
|
|
"reward": -2.8333332538604736,
|
|
"reward_std": 0.30821534991264343,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.1666666716337204,
|
|
"step": 20
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2811.638916015625,
|
|
"epoch": 0.017994858611825194,
|
|
"grad_norm": 0.2733408510684967,
|
|
"kl": 1.6957521438598633e-05,
|
|
"learning_rate": 7e-07,
|
|
"loss": 0.1159,
|
|
"reward": -2.3611111640930176,
|
|
"reward_std": 0.48556874692440033,
|
|
"rewards/confidence_reward": -2.888888955116272,
|
|
"rewards/format_reward": 0.5277777910232544,
|
|
"step": 21
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3465.8056640625,
|
|
"epoch": 0.018851756640959727,
|
|
"grad_norm": 0.197701096534729,
|
|
"kl": 2.8759241104125977e-05,
|
|
"learning_rate": 7.333333333333332e-07,
|
|
"loss": 0.0588,
|
|
"reward": -2.694444417953491,
|
|
"reward_std": 0.2901904359459877,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.25000000186264515,
|
|
"step": 22
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3026.4722900390625,
|
|
"epoch": 0.01970865467009426,
|
|
"grad_norm": 0.3089195787906647,
|
|
"kl": 1.558661460876465e-05,
|
|
"learning_rate": 7.666666666666667e-07,
|
|
"loss": 0.2173,
|
|
"reward": -2.555555582046509,
|
|
"reward_std": 0.2221490517258644,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.4444444552063942,
|
|
"step": 23
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3507.5555419921875,
|
|
"epoch": 0.02056555269922879,
|
|
"grad_norm": 0.40053591132164,
|
|
"kl": 5.6743621826171875e-05,
|
|
"learning_rate": 8e-07,
|
|
"loss": 0.1994,
|
|
"reward": -2.6944445371627808,
|
|
"reward_std": 0.3995024412870407,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3055555671453476,
|
|
"step": 24
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3597.361083984375,
|
|
"epoch": 0.021422450728363324,
|
|
"grad_norm": 0.3277747929096222,
|
|
"kl": 2.2545456886291504e-05,
|
|
"learning_rate": 8.333333333333333e-07,
|
|
"loss": 0.1546,
|
|
"reward": -2.583333373069763,
|
|
"reward_std": 0.49601033329963684,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.4166666716337204,
|
|
"step": 25
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2460.694580078125,
|
|
"epoch": 0.022279348757497857,
|
|
"grad_norm": 0.24351875483989716,
|
|
"kl": 3.9458274841308594e-05,
|
|
"learning_rate": 8.666666666666667e-07,
|
|
"loss": 0.1574,
|
|
"reward": -2.305555582046509,
|
|
"reward_std": 0.22736985981464386,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.6944444477558136,
|
|
"step": 26
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3784.3056640625,
|
|
"epoch": 0.02313624678663239,
|
|
"grad_norm": 0.14974988996982574,
|
|
"kl": 0.0001430511474609375,
|
|
"learning_rate": 9e-07,
|
|
"loss": 0.0993,
|
|
"reward": -2.7777777910232544,
|
|
"reward_std": 0.2453947588801384,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.22222222574055195,
|
|
"step": 27
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2524.4444580078125,
|
|
"epoch": 0.023993144815766924,
|
|
"grad_norm": 0.2757478356361389,
|
|
"kl": 0.00017023086547851562,
|
|
"learning_rate": 9.333333333333333e-07,
|
|
"loss": 0.0924,
|
|
"reward": -2.305555582046509,
|
|
"reward_std": 0.3995024487376213,
|
|
"rewards/confidence_reward": -2.888888955116272,
|
|
"rewards/format_reward": 0.5833333432674408,
|
|
"step": 28
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3612.0556640625,
|
|
"epoch": 0.024850042844901457,
|
|
"grad_norm": 0.31738027930259705,
|
|
"kl": 0.00018072128295898438,
|
|
"learning_rate": 9.666666666666666e-07,
|
|
"loss": 0.1543,
|
|
"reward": -2.6666667461395264,
|
|
"reward_std": 0.4803479313850403,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3333333358168602,
|
|
"step": 29
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3717.0555419921875,
|
|
"epoch": 0.02570694087403599,
|
|
"grad_norm": 0.223888099193573,
|
|
"kl": 0.00014829635620117188,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.1178,
|
|
"reward": -2.583333373069763,
|
|
"reward_std": 0.49601036310195923,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.4166666716337204,
|
|
"step": 30
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2863.02783203125,
|
|
"epoch": 0.026563838903170524,
|
|
"grad_norm": 0.15052570402622223,
|
|
"kl": 0.0007700920104980469,
|
|
"learning_rate": 9.99969538601693e-07,
|
|
"loss": 0.1475,
|
|
"reward": -2.472222328186035,
|
|
"reward_std": 0.25061558187007904,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.5277777761220932,
|
|
"step": 31
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2220.388916015625,
|
|
"epoch": 0.027420736932305057,
|
|
"grad_norm": 0.1104821264743805,
|
|
"kl": 0.0006427764892578125,
|
|
"learning_rate": 9.998781585307575e-07,
|
|
"loss": 0.0251,
|
|
"reward": -2.1944445371627808,
|
|
"reward_std": 0.2901904359459877,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.75,
|
|
"step": 32
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2779.8055419921875,
|
|
"epoch": 0.028277634961439587,
|
|
"grad_norm": 0.2474696934223175,
|
|
"kl": 0.00033664703369140625,
|
|
"learning_rate": 9.997258721585931e-07,
|
|
"loss": 0.0417,
|
|
"reward": -2.2870370149612427,
|
|
"reward_std": 0.4910118281841278,
|
|
"rewards/confidence_reward": -2.8148149251937866,
|
|
"rewards/format_reward": 0.5277777910232544,
|
|
"step": 33
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3405.7777099609375,
|
|
"epoch": 0.02913453299057412,
|
|
"grad_norm": 0.11548831313848495,
|
|
"kl": 0.0003676414489746094,
|
|
"learning_rate": 9.99512700102336e-07,
|
|
"loss": 0.0451,
|
|
"reward": -2.7222222089767456,
|
|
"reward_std": 0.15932847559452057,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.2777777835726738,
|
|
"step": 34
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3638.1666259765625,
|
|
"epoch": 0.029991431019708654,
|
|
"grad_norm": 0.22043903172016144,
|
|
"kl": 0.0002741813659667969,
|
|
"learning_rate": 9.992386712220707e-07,
|
|
"loss": 0.1144,
|
|
"reward": -2.6666667461395264,
|
|
"reward_std": 0.31865697354078293,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3333333432674408,
|
|
"step": 35
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3693.27783203125,
|
|
"epoch": 0.030848329048843187,
|
|
"grad_norm": 0.008012472651898861,
|
|
"kl": 0.0003859996795654297,
|
|
"learning_rate": 9.989038226169207e-07,
|
|
"loss": 0.0,
|
|
"reward": -2.833333373069763,
|
|
"reward_std": 0.0,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.1666666716337204,
|
|
"step": 36
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3796.9444580078125,
|
|
"epoch": 0.031705227077977724,
|
|
"grad_norm": 0.18162299692630768,
|
|
"kl": 0.001140594482421875,
|
|
"learning_rate": 9.985081996200277e-07,
|
|
"loss": 0.056,
|
|
"reward": -2.75,
|
|
"reward_std": 0.22736985236406326,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.25000000186264515,
|
|
"step": 37
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3224.5001220703125,
|
|
"epoch": 0.032562125107112254,
|
|
"grad_norm": 0.33491846919059753,
|
|
"kl": 0.00153350830078125,
|
|
"learning_rate": 9.98051855792412e-07,
|
|
"loss": 0.222,
|
|
"reward": -2.4722222089767456,
|
|
"reward_std": 0.4983728677034378,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.5277777910232544,
|
|
"step": 38
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3594.0,
|
|
"epoch": 0.033419023136246784,
|
|
"grad_norm": 0.17942766845226288,
|
|
"kl": 0.0010013580322265625,
|
|
"learning_rate": 9.975348529157229e-07,
|
|
"loss": 0.0885,
|
|
"reward": -2.6111111640930176,
|
|
"reward_std": 0.36798951029777527,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.3333333432674408,
|
|
"step": 39
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3537.7222900390625,
|
|
"epoch": 0.03427592116538132,
|
|
"grad_norm": 0.20143228769302368,
|
|
"kl": 0.001697540283203125,
|
|
"learning_rate": 9.969572609838744e-07,
|
|
"loss": 0.1538,
|
|
"reward": -2.555555582046509,
|
|
"reward_std": 0.3814775347709656,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.4444444626569748,
|
|
"step": 40
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3175.02783203125,
|
|
"epoch": 0.03513281919451585,
|
|
"grad_norm": 0.16529610753059387,
|
|
"kl": 0.000911712646484375,
|
|
"learning_rate": 9.963191581935677e-07,
|
|
"loss": 0.1,
|
|
"reward": -2.555555582046509,
|
|
"reward_std": 0.15932848304510117,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.4444444477558136,
|
|
"step": 41
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3616.7777099609375,
|
|
"epoch": 0.03598971722365039,
|
|
"grad_norm": 0.08064991980791092,
|
|
"kl": 0.001148223876953125,
|
|
"learning_rate": 9.956206309337066e-07,
|
|
"loss": 0.0023,
|
|
"reward": -2.7777777910232544,
|
|
"reward_std": 0.08606629818677902,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.2222222276031971,
|
|
"step": 42
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3938.3055419921875,
|
|
"epoch": 0.03684661525278492,
|
|
"grad_norm": 0.12845046818256378,
|
|
"kl": 0.00202178955078125,
|
|
"learning_rate": 9.948617737737001e-07,
|
|
"loss": 0.0375,
|
|
"reward": -2.805555582046509,
|
|
"reward_std": 0.1773533970117569,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.1944444477558136,
|
|
"step": 43
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3115.9722900390625,
|
|
"epoch": 0.037703513281919454,
|
|
"grad_norm": 0.1982233226299286,
|
|
"kl": 0.0035247802734375,
|
|
"learning_rate": 9.940426894506606e-07,
|
|
"loss": 0.1154,
|
|
"reward": -2.3240740299224854,
|
|
"reward_std": 0.49022431671619415,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.5833333432674408,
|
|
"step": 44
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3329.9444580078125,
|
|
"epoch": 0.038560411311053984,
|
|
"grad_norm": 0.2199009209871292,
|
|
"kl": 0.00379180908203125,
|
|
"learning_rate": 9.931634888554935e-07,
|
|
"loss": 0.0548,
|
|
"reward": -2.6388888359069824,
|
|
"reward_std": 0.36345263570547104,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.30555556900799274,
|
|
"step": 45
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3532.666748046875,
|
|
"epoch": 0.03941730934018852,
|
|
"grad_norm": 0.1083880066871643,
|
|
"kl": 0.0015277862548828125,
|
|
"learning_rate": 9.922242910178859e-07,
|
|
"loss": 0.0935,
|
|
"reward": -2.638888955116272,
|
|
"reward_std": 0.29313045740127563,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.3055555671453476,
|
|
"step": 46
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2992.1944580078125,
|
|
"epoch": 0.04027420736932305,
|
|
"grad_norm": 1.405205488204956,
|
|
"kl": 0.0645751953125,
|
|
"learning_rate": 9.912252230901906e-07,
|
|
"loss": 0.0511,
|
|
"reward": -2.5,
|
|
"reward_std": 0.3582318127155304,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.4444444477558136,
|
|
"step": 47
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3138.666748046875,
|
|
"epoch": 0.04113110539845758,
|
|
"grad_norm": 0.19926148653030396,
|
|
"kl": 0.02202606201171875,
|
|
"learning_rate": 9.901664203302124e-07,
|
|
"loss": 0.0487,
|
|
"reward": -2.5,
|
|
"reward_std": 0.13608276098966599,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.5,
|
|
"step": 48
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2550.6944580078125,
|
|
"epoch": 0.04198800342759212,
|
|
"grad_norm": 0.2365693897008896,
|
|
"kl": 0.0066070556640625,
|
|
"learning_rate": 9.890480260828965e-07,
|
|
"loss": 0.1002,
|
|
"reward": -2.138888955116272,
|
|
"reward_std": 0.49953536689281464,
|
|
"rewards/confidence_reward": -2.8888888359069824,
|
|
"rewards/format_reward": 0.75,
|
|
"step": 49
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2925.8612060546875,
|
|
"epoch": 0.04284490145672665,
|
|
"grad_norm": 0.2880806028842926,
|
|
"kl": 0.0074005126953125,
|
|
"learning_rate": 9.878701917609207e-07,
|
|
"loss": 0.2009,
|
|
"reward": -2.4166667461395264,
|
|
"reward_std": 0.49203380942344666,
|
|
"rewards/confidence_reward": -2.833333373069763,
|
|
"rewards/format_reward": 0.4166666716337204,
|
|
"step": 50
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3871.27783203125,
|
|
"epoch": 0.043701799485861184,
|
|
"grad_norm": 0.1224374920129776,
|
|
"kl": 0.00347900390625,
|
|
"learning_rate": 9.866330768241983e-07,
|
|
"loss": 0.067,
|
|
"reward": -2.75,
|
|
"reward_std": 0.2634196802973747,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.2500000037252903,
|
|
"step": 51
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3002.5833740234375,
|
|
"epoch": 0.044558697514995714,
|
|
"grad_norm": 0.3084653913974762,
|
|
"kl": 0.010650634765625,
|
|
"learning_rate": 9.853368487582886e-07,
|
|
"loss": 0.1609,
|
|
"reward": -2.472222328186035,
|
|
"reward_std": 0.22736985981464386,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.5277777761220932,
|
|
"step": 52
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3275.4444580078125,
|
|
"epoch": 0.04541559554413025,
|
|
"grad_norm": 0.1611953228712082,
|
|
"kl": 0.00553131103515625,
|
|
"learning_rate": 9.839816830517225e-07,
|
|
"loss": 0.0874,
|
|
"reward": -2.5277777910232544,
|
|
"reward_std": 0.34948598593473434,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.4722222238779068,
|
|
"step": 53
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3493.4444580078125,
|
|
"epoch": 0.04627249357326478,
|
|
"grad_norm": 0.05148787423968315,
|
|
"kl": 0.00240325927734375,
|
|
"learning_rate": 9.825677631722435e-07,
|
|
"loss": 0.0171,
|
|
"reward": -2.4166667461395264,
|
|
"reward_std": 0.09128709882497787,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.5833333283662796,
|
|
"step": 54
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2541.5,
|
|
"epoch": 0.04712939160239932,
|
|
"grad_norm": 0.2660021483898163,
|
|
"kl": 0.0017642974853515625,
|
|
"learning_rate": 9.8109528054197e-07,
|
|
"loss": 0.0367,
|
|
"reward": -2.2685184478759766,
|
|
"reward_std": 0.5399755388498306,
|
|
"rewards/confidence_reward": -2.7962963581085205,
|
|
"rewards/format_reward": 0.5277777761220932,
|
|
"step": 55
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2758.02783203125,
|
|
"epoch": 0.04798628963153385,
|
|
"grad_norm": 0.1080583855509758,
|
|
"kl": 0.002521514892578125,
|
|
"learning_rate": 9.795644345114794e-07,
|
|
"loss": 0.0841,
|
|
"reward": -2.277777910232544,
|
|
"reward_std": 0.2221490517258644,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.7222222089767456,
|
|
"step": 56
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3181.0001220703125,
|
|
"epoch": 0.04884318766066838,
|
|
"grad_norm": 0.3695676922798157,
|
|
"kl": 0.0043849945068359375,
|
|
"learning_rate": 9.779754323328192e-07,
|
|
"loss": 0.2031,
|
|
"reward": -2.4166667461395264,
|
|
"reward_std": 0.34020692110061646,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.5833333283662796,
|
|
"step": 57
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3258.4722900390625,
|
|
"epoch": 0.049700085689802914,
|
|
"grad_norm": 0.45151278376579285,
|
|
"kl": 0.002864837646484375,
|
|
"learning_rate": 9.76328489131448e-07,
|
|
"loss": 0.2348,
|
|
"reward": -2.5833332538604736,
|
|
"reward_std": 0.3995024561882019,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.416666679084301,
|
|
"step": 58
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3137.638916015625,
|
|
"epoch": 0.050556983718937444,
|
|
"grad_norm": 0.24236924946308136,
|
|
"kl": 0.007476806640625,
|
|
"learning_rate": 9.746238278771125e-07,
|
|
"loss": 0.0592,
|
|
"reward": -2.3888888359069824,
|
|
"reward_std": 0.245394766330719,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.6111111044883728,
|
|
"step": 59
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2517.861083984375,
|
|
"epoch": 0.05141388174807198,
|
|
"grad_norm": 0.17099355161190033,
|
|
"kl": 0.00344085693359375,
|
|
"learning_rate": 9.728616793536587e-07,
|
|
"loss": 0.0848,
|
|
"reward": -2.1944445371627808,
|
|
"reward_std": 0.36345260590314865,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.75,
|
|
"step": 60
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3005.75,
|
|
"epoch": 0.05227077977720651,
|
|
"grad_norm": 0.17847158014774323,
|
|
"kl": 0.0050048828125,
|
|
"learning_rate": 9.71042282127789e-07,
|
|
"loss": 0.0104,
|
|
"reward": -2.379629611968994,
|
|
"reward_std": 0.2948459982872009,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.5277777761220932,
|
|
"step": 61
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2963.4722900390625,
|
|
"epoch": 0.05312767780634105,
|
|
"grad_norm": 0.22850683331489563,
|
|
"kl": 0.00327301025390625,
|
|
"learning_rate": 9.69165882516764e-07,
|
|
"loss": -0.0198,
|
|
"reward": -2.6111111640930176,
|
|
"reward_std": 0.1360827535390854,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.3333333432674408,
|
|
"step": 62
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2869.3056640625,
|
|
"epoch": 0.05398457583547558,
|
|
"grad_norm": 0.1911962777376175,
|
|
"kl": 0.001628875732421875,
|
|
"learning_rate": 9.672327345550543e-07,
|
|
"loss": 0.0823,
|
|
"reward": -2.1574073433876038,
|
|
"reward_std": 0.4669785648584366,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.75,
|
|
"step": 63
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3161.9444580078125,
|
|
"epoch": 0.054841473864610114,
|
|
"grad_norm": 0.349700391292572,
|
|
"kl": 0.00498199462890625,
|
|
"learning_rate": 9.65243099959949e-07,
|
|
"loss": 0.2121,
|
|
"reward": -2.5,
|
|
"reward_std": 0.4047232121229172,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.5000000149011612,
|
|
"step": 64
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2009.166748046875,
|
|
"epoch": 0.055698371893744644,
|
|
"grad_norm": 0.03013899177312851,
|
|
"kl": 0.00531768798828125,
|
|
"learning_rate": 9.631972480961233e-07,
|
|
"loss": 0.0002,
|
|
"reward": -2.3333332538604736,
|
|
"reward_std": 0.0,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.6666666865348816,
|
|
"step": 65
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3121.138916015625,
|
|
"epoch": 0.056555269922879174,
|
|
"grad_norm": 0.1924014538526535,
|
|
"kl": 0.003353118896484375,
|
|
"learning_rate": 9.610954559391704e-07,
|
|
"loss": 0.0912,
|
|
"reward": -2.333333373069763,
|
|
"reward_std": 0.5140352547168732,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.6111111044883728,
|
|
"step": 66
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2877.4722900390625,
|
|
"epoch": 0.05741216795201371,
|
|
"grad_norm": 0.34053537249565125,
|
|
"kl": 0.005859375,
|
|
"learning_rate": 9.589380080381038e-07,
|
|
"loss": -0.0137,
|
|
"reward": -2.3425925970077515,
|
|
"reward_std": 0.5270616561174393,
|
|
"rewards/confidence_reward": -2.7037038803100586,
|
|
"rewards/format_reward": 0.361111119389534,
|
|
"step": 67
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3536.3333740234375,
|
|
"epoch": 0.05826906598114824,
|
|
"grad_norm": 0.24978837370872498,
|
|
"kl": 0.00376129150390625,
|
|
"learning_rate": 9.567251964768342e-07,
|
|
"loss": 0.1593,
|
|
"reward": -2.555555582046509,
|
|
"reward_std": 0.4675438404083252,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.4444444477558136,
|
|
"step": 68
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3617.8612060546875,
|
|
"epoch": 0.05912596401028278,
|
|
"grad_norm": 0.23766648769378662,
|
|
"kl": 0.00360870361328125,
|
|
"learning_rate": 9.54457320834625e-07,
|
|
"loss": 0.1549,
|
|
"reward": -2.722222328186035,
|
|
"reward_std": 0.3814775347709656,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.2777777910232544,
|
|
"step": 69
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3170.4444580078125,
|
|
"epoch": 0.05998286203941731,
|
|
"grad_norm": 0.11585590243339539,
|
|
"kl": 0.0055084228515625,
|
|
"learning_rate": 9.521346881455354e-07,
|
|
"loss": 0.0739,
|
|
"reward": -2.5,
|
|
"reward_std": 0.2221490666270256,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.4444444626569748,
|
|
"step": 70
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3495.6666259765625,
|
|
"epoch": 0.060839760068551844,
|
|
"grad_norm": 0.25010788440704346,
|
|
"kl": 0.0036468505859375,
|
|
"learning_rate": 9.497576128568518e-07,
|
|
"loss": 0.0395,
|
|
"reward": -2.7041274309158325,
|
|
"reward_std": 0.316488653421402,
|
|
"rewards/confidence_reward": -2.8985718488693237,
|
|
"rewards/format_reward": 0.1944444477558136,
|
|
"step": 71
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2589.1111450195312,
|
|
"epoch": 0.061696658097686374,
|
|
"grad_norm": 0.08638693392276764,
|
|
"kl": 0.0056915283203125,
|
|
"learning_rate": 9.473264167865171e-07,
|
|
"loss": 0.0357,
|
|
"reward": -2.4444445371627808,
|
|
"reward_std": 0.13608276844024658,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.5000000149011612,
|
|
"step": 72
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3842.9444580078125,
|
|
"epoch": 0.06255355612682091,
|
|
"grad_norm": 0.2187531441450119,
|
|
"kl": 0.003448486328125,
|
|
"learning_rate": 9.448414290795618e-07,
|
|
"loss": 0.1007,
|
|
"reward": -2.833333373069763,
|
|
"reward_std": 0.30821535736322403,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.16666667349636555,
|
|
"step": 73
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3221.8055419921875,
|
|
"epoch": 0.06341045415595545,
|
|
"grad_norm": 0.21624591946601868,
|
|
"kl": 0.00295257568359375,
|
|
"learning_rate": 9.42302986163543e-07,
|
|
"loss": 0.073,
|
|
"reward": -2.4444445371627808,
|
|
"reward_std": 0.3082153648138046,
|
|
"rewards/confidence_reward": -2.7777777910232544,
|
|
"rewards/format_reward": 0.3333333432674408,
|
|
"step": 74
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3737.4722900390625,
|
|
"epoch": 0.06426735218508997,
|
|
"grad_norm": 0.18189558386802673,
|
|
"kl": 0.007110595703125,
|
|
"learning_rate": 9.397114317029974e-07,
|
|
"loss": 0.0809,
|
|
"reward": -2.6944445371627808,
|
|
"reward_std": 0.3134361654520035,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.305555559694767,
|
|
"step": 75
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3442.9722900390625,
|
|
"epoch": 0.06512425021422451,
|
|
"grad_norm": 0.11082806438207626,
|
|
"kl": 0.005889892578125,
|
|
"learning_rate": 9.370671165529144e-07,
|
|
"loss": 0.0364,
|
|
"reward": -2.5277777910232544,
|
|
"reward_std": 0.0680413767695427,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.472222238779068,
|
|
"step": 76
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3204.166748046875,
|
|
"epoch": 0.06598114824335904,
|
|
"grad_norm": 0.1811019480228424,
|
|
"kl": 0.007965087890625,
|
|
"learning_rate": 9.343703987112365e-07,
|
|
"loss": 0.1172,
|
|
"reward": -2.5277777910232544,
|
|
"reward_std": 0.37919675558805466,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.4166666865348816,
|
|
"step": 77
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3627.47216796875,
|
|
"epoch": 0.06683804627249357,
|
|
"grad_norm": 0.15231575071811676,
|
|
"kl": 0.00232696533203125,
|
|
"learning_rate": 9.316216432703916e-07,
|
|
"loss": 0.0252,
|
|
"reward": -2.694444417953491,
|
|
"reward_std": 0.24017397314310074,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3055555559694767,
|
|
"step": 78
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2848.416748046875,
|
|
"epoch": 0.0676949443016281,
|
|
"grad_norm": 0.2135927677154541,
|
|
"kl": 0.0075836181640625,
|
|
"learning_rate": 9.288212223678658e-07,
|
|
"loss": 0.0625,
|
|
"reward": -2.407407283782959,
|
|
"reward_std": 0.44895367324352264,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.5,
|
|
"step": 79
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3405.77783203125,
|
|
"epoch": 0.06855184233076264,
|
|
"grad_norm": 0.12736141681671143,
|
|
"kl": 0.00734710693359375,
|
|
"learning_rate": 9.259695151358214e-07,
|
|
"loss": 0.0282,
|
|
"reward": -2.638888955116272,
|
|
"reward_std": 0.20412413775920868,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.361111119389534,
|
|
"step": 80
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3254.0555419921875,
|
|
"epoch": 0.06940874035989718,
|
|
"grad_norm": 0.20807288587093353,
|
|
"kl": 0.008697509765625,
|
|
"learning_rate": 9.230669076497687e-07,
|
|
"loss": 0.1148,
|
|
"reward": -2.638888955116272,
|
|
"reward_std": 0.20412413775920868,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.3055555522441864,
|
|
"step": 81
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3111.166748046875,
|
|
"epoch": 0.0702656383890317,
|
|
"grad_norm": 0.2130519151687622,
|
|
"kl": 0.006744384765625,
|
|
"learning_rate": 9.20113792876298e-07,
|
|
"loss": 0.1215,
|
|
"reward": -2.5,
|
|
"reward_std": 0.18257419764995575,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.5000000149011612,
|
|
"step": 82
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2739.3333740234375,
|
|
"epoch": 0.07112253641816624,
|
|
"grad_norm": 0.2380674034357071,
|
|
"kl": 0.007965087890625,
|
|
"learning_rate": 9.171105706198774e-07,
|
|
"loss": 0.0922,
|
|
"reward": -2.2222222685813904,
|
|
"reward_std": 0.31865695118904114,
|
|
"rewards/confidence_reward": -2.833333373069763,
|
|
"rewards/format_reward": 0.6111111342906952,
|
|
"step": 83
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2850.4166259765625,
|
|
"epoch": 0.07197943444730077,
|
|
"grad_norm": 0.19383125007152557,
|
|
"kl": 0.00946044921875,
|
|
"learning_rate": 9.140576474687263e-07,
|
|
"loss": 0.0087,
|
|
"reward": -1.9166666865348816,
|
|
"reward_std": 0.6418167352676392,
|
|
"rewards/confidence_reward": -2.4444446563720703,
|
|
"rewards/format_reward": 0.5277777910232544,
|
|
"step": 84
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2711.52783203125,
|
|
"epoch": 0.0728363324764353,
|
|
"grad_norm": 0.23535026609897614,
|
|
"kl": 0.004180908203125,
|
|
"learning_rate": 9.109554367397697e-07,
|
|
"loss": 0.1209,
|
|
"reward": -2.333333373069763,
|
|
"reward_std": 0.3412187397480011,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.6111111044883728,
|
|
"step": 85
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3125.9722900390625,
|
|
"epoch": 0.07369323050556983,
|
|
"grad_norm": 0.07005209475755692,
|
|
"kl": 0.010040283203125,
|
|
"learning_rate": 9.078043584226815e-07,
|
|
"loss": 0.0131,
|
|
"reward": -2.583333373069763,
|
|
"reward_std": 0.09128709882497787,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.4166666567325592,
|
|
"step": 86
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3320.0555419921875,
|
|
"epoch": 0.07455012853470437,
|
|
"grad_norm": 0.1450621485710144,
|
|
"kl": 0.010986328125,
|
|
"learning_rate": 9.046048391230247e-07,
|
|
"loss": 0.0719,
|
|
"reward": -2.5277777910232544,
|
|
"reward_std": 0.20412413030862808,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.4722222238779068,
|
|
"step": 87
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3126.52783203125,
|
|
"epoch": 0.07540702656383891,
|
|
"grad_norm": 0.14595212042331696,
|
|
"kl": 0.004852294921875,
|
|
"learning_rate": 9.013573120044966e-07,
|
|
"loss": 0.0592,
|
|
"reward": -2.3518518209457397,
|
|
"reward_std": 0.3861331045627594,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.5555555671453476,
|
|
"step": 88
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3651.5833740234375,
|
|
"epoch": 0.07626392459297343,
|
|
"grad_norm": 0.21023660898208618,
|
|
"kl": 0.006317138671875,
|
|
"learning_rate": 8.980622167302837e-07,
|
|
"loss": 0.0916,
|
|
"reward": -2.5740740299224854,
|
|
"reward_std": 0.5071772634983063,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.3333333358168602,
|
|
"step": 89
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3397.9722900390625,
|
|
"epoch": 0.07712082262210797,
|
|
"grad_norm": 0.188828244805336,
|
|
"kl": 0.0087890625,
|
|
"learning_rate": 8.9471999940354e-07,
|
|
"loss": 0.0082,
|
|
"reward": -2.2870370149612427,
|
|
"reward_std": 0.40421056002378464,
|
|
"rewards/confidence_reward": -2.7037036418914795,
|
|
"rewards/format_reward": 0.4166666716337204,
|
|
"step": 90
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2552.52783203125,
|
|
"epoch": 0.0779777206512425,
|
|
"grad_norm": 0.40647614002227783,
|
|
"kl": 0.01544189453125,
|
|
"learning_rate": 8.91331112506991e-07,
|
|
"loss": 0.1232,
|
|
"reward": -1.8888888955116272,
|
|
"reward_std": 0.5773965641856194,
|
|
"rewards/confidence_reward": -2.444444477558136,
|
|
"rewards/format_reward": 0.5555555522441864,
|
|
"step": 91
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2823.6944580078125,
|
|
"epoch": 0.07883461868037704,
|
|
"grad_norm": 0.12339317798614502,
|
|
"kl": 0.0080413818359375,
|
|
"learning_rate": 8.878960148416747e-07,
|
|
"loss": 0.0168,
|
|
"reward": -2.3611111640930176,
|
|
"reward_std": 0.22736985236406326,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.638888880610466,
|
|
"step": 92
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3384.52783203125,
|
|
"epoch": 0.07969151670951156,
|
|
"grad_norm": 0.29699084162712097,
|
|
"kl": 0.00677490234375,
|
|
"learning_rate": 8.844151714648274e-07,
|
|
"loss": 0.1672,
|
|
"reward": -2.5,
|
|
"reward_std": 0.3814775347709656,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.5000000149011612,
|
|
"step": 93
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3287.25,
|
|
"epoch": 0.0805484147386461,
|
|
"grad_norm": 0.39855116605758667,
|
|
"kl": 0.0213623046875,
|
|
"learning_rate": 8.808890536269229e-07,
|
|
"loss": 0.1707,
|
|
"reward": -2.416666626930237,
|
|
"reward_std": 0.3995024412870407,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.5833333432674408,
|
|
"step": 94
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2682.75,
|
|
"epoch": 0.08140531276778064,
|
|
"grad_norm": 0.391999751329422,
|
|
"kl": 0.015533447265625,
|
|
"learning_rate": 8.773181387078719e-07,
|
|
"loss": 0.1891,
|
|
"reward": -2.3611111640930176,
|
|
"reward_std": 0.49601034820079803,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.5833333432674408,
|
|
"step": 95
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3296.9722900390625,
|
|
"epoch": 0.08226221079691516,
|
|
"grad_norm": 0.22848033905029297,
|
|
"kl": 0.0120391845703125,
|
|
"learning_rate": 8.737029101523929e-07,
|
|
"loss": 0.1268,
|
|
"reward": -2.3240740299224854,
|
|
"reward_std": 0.5216506272554398,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.5833333432674408,
|
|
"step": 96
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2389.4166870117188,
|
|
"epoch": 0.0831191088260497,
|
|
"grad_norm": 0.2833496034145355,
|
|
"kl": 0.0255126953125,
|
|
"learning_rate": 8.700438574045617e-07,
|
|
"loss": 0.239,
|
|
"reward": -2.0092591643333435,
|
|
"reward_std": 0.788471519947052,
|
|
"rewards/confidence_reward": -2.759259343147278,
|
|
"rewards/format_reward": 0.75,
|
|
"step": 97
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2865.9722900390625,
|
|
"epoch": 0.08397600685518423,
|
|
"grad_norm": 0.18639419972896576,
|
|
"kl": 0.014617919921875,
|
|
"learning_rate": 8.663414758415478e-07,
|
|
"loss": 0.0585,
|
|
"reward": -2.333333373069763,
|
|
"reward_std": 0.3314610570669174,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.6666666567325592,
|
|
"step": 98
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2753.9444580078125,
|
|
"epoch": 0.08483290488431877,
|
|
"grad_norm": 0.26271870732307434,
|
|
"kl": 0.021240234375,
|
|
"learning_rate": 8.625962667065487e-07,
|
|
"loss": 0.1419,
|
|
"reward": -2.25,
|
|
"reward_std": 0.41698990762233734,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.6944444477558136,
|
|
"step": 99
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3196.52783203125,
|
|
"epoch": 0.0856898029134533,
|
|
"grad_norm": 0.2569756805896759,
|
|
"kl": 0.027587890625,
|
|
"learning_rate": 8.588087370409302e-07,
|
|
"loss": 0.0869,
|
|
"reward": -2.2129629254341125,
|
|
"reward_std": 0.642838716506958,
|
|
"rewards/confidence_reward": -2.574074149131775,
|
|
"rewards/format_reward": 0.361111119389534,
|
|
"step": 100
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3108.25,
|
|
"epoch": 0.08654670094258783,
|
|
"grad_norm": 0.221235990524292,
|
|
"kl": 0.0069427490234375,
|
|
"learning_rate": 8.549793996155795e-07,
|
|
"loss": 0.1286,
|
|
"reward": -2.3518518209457397,
|
|
"reward_std": 0.5122157335281372,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.5555555820465088,
|
|
"step": 101
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3036.388916015625,
|
|
"epoch": 0.08740359897172237,
|
|
"grad_norm": 0.19860585033893585,
|
|
"kl": 0.01312255859375,
|
|
"learning_rate": 8.511087728614862e-07,
|
|
"loss": 0.1027,
|
|
"reward": -2.472222328186035,
|
|
"reward_std": 0.4092601239681244,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.4722222238779068,
|
|
"step": 102
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3937.0833740234375,
|
|
"epoch": 0.08826049700085689,
|
|
"grad_norm": 0.18265724182128906,
|
|
"kl": 0.0120849609375,
|
|
"learning_rate": 8.471973807995534e-07,
|
|
"loss": 0.0519,
|
|
"reward": -2.805555582046509,
|
|
"reward_std": 0.3134361505508423,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.19444444961845875,
|
|
"step": 103
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2821.6944580078125,
|
|
"epoch": 0.08911739502999143,
|
|
"grad_norm": 0.1616680771112442,
|
|
"kl": 0.013641357421875,
|
|
"learning_rate": 8.432457529696548e-07,
|
|
"loss": 0.0466,
|
|
"reward": -2.240740656852722,
|
|
"reward_std": 0.409378819167614,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.6666666567325592,
|
|
"step": 104
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3072.27783203125,
|
|
"epoch": 0.08997429305912596,
|
|
"grad_norm": 0.27748724818229675,
|
|
"kl": 0.008331298828125,
|
|
"learning_rate": 8.392544243589427e-07,
|
|
"loss": 0.1194,
|
|
"reward": -2.305555582046509,
|
|
"reward_std": 0.6821095794439316,
|
|
"rewards/confidence_reward": -2.833333373069763,
|
|
"rewards/format_reward": 0.5277777910232544,
|
|
"step": 105
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3736.3333740234375,
|
|
"epoch": 0.0908311910882605,
|
|
"grad_norm": 0.19383865594863892,
|
|
"kl": 0.014678955078125,
|
|
"learning_rate": 8.352239353294194e-07,
|
|
"loss": 0.0574,
|
|
"reward": -2.5,
|
|
"reward_std": 0.37219424545764923,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.4444444477558136,
|
|
"step": 106
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3856.583251953125,
|
|
"epoch": 0.09168808911739502,
|
|
"grad_norm": 0.17508208751678467,
|
|
"kl": 0.0128173828125,
|
|
"learning_rate": 8.31154831544782e-07,
|
|
"loss": 0.0391,
|
|
"reward": -2.6944445371627808,
|
|
"reward_std": 0.24017397314310074,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.305555559694767,
|
|
"step": 107
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3584.5555419921875,
|
|
"epoch": 0.09254498714652956,
|
|
"grad_norm": 0.15831252932548523,
|
|
"kl": 0.017791748046875,
|
|
"learning_rate": 8.270476638965461e-07,
|
|
"loss": 0.0035,
|
|
"reward": -2.518518567085266,
|
|
"reward_std": 0.47219936549663544,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.3888889029622078,
|
|
"step": 108
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3814.5555419921875,
|
|
"epoch": 0.0934018851756641,
|
|
"grad_norm": 0.2039646953344345,
|
|
"kl": 0.012939453125,
|
|
"learning_rate": 8.229029884294662e-07,
|
|
"loss": 0.0669,
|
|
"reward": -2.666666626930237,
|
|
"reward_std": 0.3547067865729332,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3333333358168602,
|
|
"step": 109
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3702.0,
|
|
"epoch": 0.09425878320479864,
|
|
"grad_norm": 0.19251351058483124,
|
|
"kl": 0.00970458984375,
|
|
"learning_rate": 8.187213662662538e-07,
|
|
"loss": 0.0819,
|
|
"reward": -2.5833332538604736,
|
|
"reward_std": 0.3995024412870407,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.416666679084301,
|
|
"step": 110
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3664.5833740234375,
|
|
"epoch": 0.09511568123393316,
|
|
"grad_norm": 0.10979479551315308,
|
|
"kl": 0.013031005859375,
|
|
"learning_rate": 8.145033635316128e-07,
|
|
"loss": 0.0533,
|
|
"reward": -2.75,
|
|
"reward_std": 0.15410767495632172,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.25,
|
|
"step": 111
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3430.5556640625,
|
|
"epoch": 0.0959725792630677,
|
|
"grad_norm": 0.21925541758537292,
|
|
"kl": 0.012115478515625,
|
|
"learning_rate": 8.102495512755938e-07,
|
|
"loss": 0.0942,
|
|
"reward": -2.5833332538604736,
|
|
"reward_std": 0.2738612964749336,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.416666679084301,
|
|
"step": 112
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3218.166748046875,
|
|
"epoch": 0.09682947729220223,
|
|
"grad_norm": 0.2610246241092682,
|
|
"kl": 0.014556884765625,
|
|
"learning_rate": 8.059605053962833e-07,
|
|
"loss": 0.1001,
|
|
"reward": -2.435185194015503,
|
|
"reward_std": 0.39413511008024216,
|
|
"rewards/confidence_reward": -2.8518519401550293,
|
|
"rewards/format_reward": 0.4166666865348816,
|
|
"step": 113
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3630.8612060546875,
|
|
"epoch": 0.09768637532133675,
|
|
"grad_norm": 0.14595678448677063,
|
|
"kl": 0.011627197265625,
|
|
"learning_rate": 8.01636806561836e-07,
|
|
"loss": 0.0598,
|
|
"reward": -2.555555582046509,
|
|
"reward_std": 0.1360827535390854,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.4444444477558136,
|
|
"step": 114
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3384.3055419921875,
|
|
"epoch": 0.09854327335047129,
|
|
"grad_norm": 0.1648566722869873,
|
|
"kl": 0.02392578125,
|
|
"learning_rate": 7.972790401318627e-07,
|
|
"loss": 0.103,
|
|
"reward": -2.6388888359069824,
|
|
"reward_std": 0.2901904359459877,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.30555556900799274,
|
|
"step": 115
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3751.388916015625,
|
|
"epoch": 0.09940017137960583,
|
|
"grad_norm": 0.21332553029060364,
|
|
"kl": 0.02044677734375,
|
|
"learning_rate": 7.928877960781808e-07,
|
|
"loss": 0.1117,
|
|
"reward": -2.75,
|
|
"reward_std": 0.3134361356496811,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.2500000037252903,
|
|
"step": 116
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3072.2501220703125,
|
|
"epoch": 0.10025706940874037,
|
|
"grad_norm": 0.1085459515452385,
|
|
"kl": 0.02020263671875,
|
|
"learning_rate": 7.884636689049422e-07,
|
|
"loss": 0.0629,
|
|
"reward": -2.2685184478759766,
|
|
"reward_std": 0.2494850754737854,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.6388888955116272,
|
|
"step": 117
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3123.9722900390625,
|
|
"epoch": 0.10111396743787489,
|
|
"grad_norm": 0.16286125779151917,
|
|
"kl": 0.0162353515625,
|
|
"learning_rate": 7.840072575681468e-07,
|
|
"loss": 0.0787,
|
|
"reward": -2.4452918767929077,
|
|
"reward_std": 0.4294184446334839,
|
|
"rewards/confidence_reward": -2.889736294746399,
|
|
"rewards/format_reward": 0.4444444626569748,
|
|
"step": 118
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3960.3333740234375,
|
|
"epoch": 0.10197086546700942,
|
|
"grad_norm": 0.15160124003887177,
|
|
"kl": 0.01416015625,
|
|
"learning_rate": 7.795191653945538e-07,
|
|
"loss": 0.0512,
|
|
"reward": -2.805555582046509,
|
|
"reward_std": 0.32624027132987976,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.1944444477558136,
|
|
"step": 119
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3328.9722900390625,
|
|
"epoch": 0.10282776349614396,
|
|
"grad_norm": 0.1655929535627365,
|
|
"kl": 0.01312255859375,
|
|
"learning_rate": 7.75e-07,
|
|
"loss": 0.0925,
|
|
"reward": -2.4722222089767456,
|
|
"reward_std": 0.3134361580014229,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.5277777910232544,
|
|
"step": 120
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3596.25,
|
|
"epoch": 0.1036846615252785,
|
|
"grad_norm": 0.08225465565919876,
|
|
"kl": 0.011199951171875,
|
|
"learning_rate": 7.704503732071391e-07,
|
|
"loss": 0.0074,
|
|
"reward": -2.6111111640930176,
|
|
"reward_std": 0.08606629818677902,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3888888955116272,
|
|
"step": 121
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3386.77783203125,
|
|
"epoch": 0.10454155955441302,
|
|
"grad_norm": 0.14785455167293549,
|
|
"kl": 0.02099609375,
|
|
"learning_rate": 7.658709009626109e-07,
|
|
"loss": 0.0373,
|
|
"reward": -2.6944445371627808,
|
|
"reward_std": 0.24017397314310074,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3055555671453476,
|
|
"step": 122
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3158.27783203125,
|
|
"epoch": 0.10539845758354756,
|
|
"grad_norm": 0.22167643904685974,
|
|
"kl": 0.02313232421875,
|
|
"learning_rate": 7.612622032536507e-07,
|
|
"loss": 0.0777,
|
|
"reward": -2.444444417953491,
|
|
"reward_std": 0.3582318276166916,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.5000000149011612,
|
|
"step": 123
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2636.861083984375,
|
|
"epoch": 0.1062553556126821,
|
|
"grad_norm": 0.2819192707538605,
|
|
"kl": 0.01617431640625,
|
|
"learning_rate": 7.566249040241553e-07,
|
|
"loss": -0.0007,
|
|
"reward": -1.9629629850387573,
|
|
"reward_std": 0.656702309846878,
|
|
"rewards/confidence_reward": -2.574074149131775,
|
|
"rewards/format_reward": 0.6111111044883728,
|
|
"step": 124
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3987.638916015625,
|
|
"epoch": 0.10711225364181662,
|
|
"grad_norm": 0.15799356997013092,
|
|
"kl": 0.0233154296875,
|
|
"learning_rate": 7.51959631090208e-07,
|
|
"loss": 0.0367,
|
|
"reward": -2.833333373069763,
|
|
"reward_std": 0.245394766330719,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.1666666716337204,
|
|
"step": 125
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3636.0001220703125,
|
|
"epoch": 0.10796915167095116,
|
|
"grad_norm": 0.20148231089115143,
|
|
"kl": 0.012664794921875,
|
|
"learning_rate": 7.472670160550848e-07,
|
|
"loss": 0.1046,
|
|
"reward": -2.555555582046509,
|
|
"reward_std": 0.3814775347709656,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.4444444477558136,
|
|
"step": 126
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3470.388916015625,
|
|
"epoch": 0.10882604970008569,
|
|
"grad_norm": 0.29233935475349426,
|
|
"kl": 0.018310546875,
|
|
"learning_rate": 7.425476942237444e-07,
|
|
"loss": 0.0973,
|
|
"reward": -2.5407216548919678,
|
|
"reward_std": 0.5416866987943649,
|
|
"rewards/confidence_reward": -2.8462772369384766,
|
|
"rewards/format_reward": 0.305555559694767,
|
|
"step": 127
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3344.111083984375,
|
|
"epoch": 0.10968294772922023,
|
|
"grad_norm": 0.12020324170589447,
|
|
"kl": 0.02020263671875,
|
|
"learning_rate": 7.37802304516818e-07,
|
|
"loss": 0.025,
|
|
"reward": -2.583333373069763,
|
|
"reward_std": 0.15410767495632172,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.4166666716337204,
|
|
"step": 128
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3881.1944580078125,
|
|
"epoch": 0.11053984575835475,
|
|
"grad_norm": 0.17918746173381805,
|
|
"kl": 0.01629638671875,
|
|
"learning_rate": 7.330314893841101e-07,
|
|
"loss": 0.0508,
|
|
"reward": -2.6944445371627808,
|
|
"reward_std": 0.3995024263858795,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.305555559694767,
|
|
"step": 129
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2966.388916015625,
|
|
"epoch": 0.11139674378748929,
|
|
"grad_norm": 0.2491733282804489,
|
|
"kl": 0.02984619140625,
|
|
"learning_rate": 7.282358947176205e-07,
|
|
"loss": 0.075,
|
|
"reward": -2.3425925970077515,
|
|
"reward_std": 0.514257550239563,
|
|
"rewards/confidence_reward": -2.814814805984497,
|
|
"rewards/format_reward": 0.472222238779068,
|
|
"step": 130
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3056.888916015625,
|
|
"epoch": 0.11225364181662383,
|
|
"grad_norm": 0.1091020330786705,
|
|
"kl": 0.018524169921875,
|
|
"learning_rate": 7.234161697641017e-07,
|
|
"loss": 0.0298,
|
|
"reward": -2.5008474588394165,
|
|
"reward_std": 0.35615625977516174,
|
|
"rewards/confidence_reward": -2.889736294746399,
|
|
"rewards/format_reward": 0.3888888955116272,
|
|
"step": 131
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3280.1944580078125,
|
|
"epoch": 0.11311053984575835,
|
|
"grad_norm": 0.22998450696468353,
|
|
"kl": 0.016815185546875,
|
|
"learning_rate": 7.185729670371604e-07,
|
|
"loss": 0.0906,
|
|
"reward": -2.555555582046509,
|
|
"reward_std": 0.3082153648138046,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.3888888955116272,
|
|
"step": 132
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3573.75,
|
|
"epoch": 0.11396743787489289,
|
|
"grad_norm": 0.17413660883903503,
|
|
"kl": 0.018310546875,
|
|
"learning_rate": 7.137069422289181e-07,
|
|
"loss": 0.089,
|
|
"reward": -2.638888955116272,
|
|
"reward_std": 0.3134361431002617,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.361111119389534,
|
|
"step": 133
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3063.0555419921875,
|
|
"epoch": 0.11482433590402742,
|
|
"grad_norm": 0.214571014046669,
|
|
"kl": 0.03424072265625,
|
|
"learning_rate": 7.08818754121241e-07,
|
|
"loss": 0.039,
|
|
"reward": -2.2314815521240234,
|
|
"reward_std": 0.4910118132829666,
|
|
"rewards/confidence_reward": -2.7592592239379883,
|
|
"rewards/format_reward": 0.5277777910232544,
|
|
"step": 134
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3406.6112060546875,
|
|
"epoch": 0.11568123393316196,
|
|
"grad_norm": 0.20651200413703918,
|
|
"kl": 0.026092529296875,
|
|
"learning_rate": 7.039090644965509e-07,
|
|
"loss": 0.0338,
|
|
"reward": -2.296296238899231,
|
|
"reward_std": 0.5582656562328339,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.6111111044883728,
|
|
"step": 135
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3469.6944580078125,
|
|
"epoch": 0.11653813196229648,
|
|
"grad_norm": 0.12644833326339722,
|
|
"kl": 0.01922607421875,
|
|
"learning_rate": 6.989785380482312e-07,
|
|
"loss": 0.0342,
|
|
"reward": -2.6944445371627808,
|
|
"reward_std": 0.1541076824069023,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.305555559694767,
|
|
"step": 136
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3475.4166259765625,
|
|
"epoch": 0.11739502999143102,
|
|
"grad_norm": 0.2094379961490631,
|
|
"kl": 0.02838134765625,
|
|
"learning_rate": 6.940278422906372e-07,
|
|
"loss": 0.1098,
|
|
"reward": -2.5,
|
|
"reward_std": 0.5836373865604401,
|
|
"rewards/confidence_reward": -2.888888955116272,
|
|
"rewards/format_reward": 0.3888888955116272,
|
|
"step": 137
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3328.0,
|
|
"epoch": 0.11825192802056556,
|
|
"grad_norm": 0.26488515734672546,
|
|
"kl": 0.019317626953125,
|
|
"learning_rate": 6.890576474687263e-07,
|
|
"loss": 0.0928,
|
|
"reward": -2.260106682777405,
|
|
"reward_std": 0.5154568552970886,
|
|
"rewards/confidence_reward": -2.704551100730896,
|
|
"rewards/format_reward": 0.4444444626569748,
|
|
"step": 138
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2500.5833740234375,
|
|
"epoch": 0.11910882604970009,
|
|
"grad_norm": 0.22615168988704681,
|
|
"kl": 0.0201416015625,
|
|
"learning_rate": 6.840686264673168e-07,
|
|
"loss": 0.0361,
|
|
"reward": -2.000847280025482,
|
|
"reward_std": 0.6050563156604767,
|
|
"rewards/confidence_reward": -2.5564029216766357,
|
|
"rewards/format_reward": 0.5555555671453476,
|
|
"step": 139
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3513.1112060546875,
|
|
"epoch": 0.11996572407883462,
|
|
"grad_norm": 0.3215450644493103,
|
|
"kl": 0.02825927734375,
|
|
"learning_rate": 6.790614547199906e-07,
|
|
"loss": 0.1226,
|
|
"reward": -2.555555582046509,
|
|
"reward_std": 0.41752736270427704,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.4444444552063942,
|
|
"step": 140
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3207.638916015625,
|
|
"epoch": 0.12082262210796915,
|
|
"grad_norm": 0.2798071503639221,
|
|
"kl": 0.02069091796875,
|
|
"learning_rate": 6.740368101176495e-07,
|
|
"loss": 0.0961,
|
|
"reward": -2.2870370149612427,
|
|
"reward_std": 0.6775233149528503,
|
|
"rewards/confidence_reward": -2.6481481790542603,
|
|
"rewards/format_reward": 0.3611111119389534,
|
|
"step": 141
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3339.1666259765625,
|
|
"epoch": 0.12167952013710369,
|
|
"grad_norm": 0.26966410875320435,
|
|
"kl": 0.02276611328125,
|
|
"learning_rate": 6.68995372916741e-07,
|
|
"loss": 0.0981,
|
|
"reward": -2.3518519401550293,
|
|
"reward_std": 0.44895367324352264,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.5555555522441864,
|
|
"step": 142
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3242.3055419921875,
|
|
"epoch": 0.12253641816623821,
|
|
"grad_norm": 0.2671264111995697,
|
|
"kl": 0.02325439453125,
|
|
"learning_rate": 6.639378256471608e-07,
|
|
"loss": 0.0918,
|
|
"reward": -2.3611111640930176,
|
|
"reward_std": 0.4583979994058609,
|
|
"rewards/confidence_reward": -2.722222328186035,
|
|
"rewards/format_reward": 0.361111119389534,
|
|
"step": 143
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2081.5,
|
|
"epoch": 0.12339331619537275,
|
|
"grad_norm": 0.25654128193855286,
|
|
"kl": 0.02740478515625,
|
|
"learning_rate": 6.588648530198504e-07,
|
|
"loss": 0.0977,
|
|
"reward": -1.8148147463798523,
|
|
"reward_std": 0.7522204518318176,
|
|
"rewards/confidence_reward": -2.64814829826355,
|
|
"rewards/format_reward": 0.8333333432674408,
|
|
"step": 144
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3736.9166259765625,
|
|
"epoch": 0.12425021422450729,
|
|
"grad_norm": 0.12350251525640488,
|
|
"kl": 0.02294921875,
|
|
"learning_rate": 6.537771418340981e-07,
|
|
"loss": 0.0331,
|
|
"reward": -2.7222222089767456,
|
|
"reward_std": 0.2221490517258644,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.27777777798473835,
|
|
"step": 145
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2392.7222900390625,
|
|
"epoch": 0.12510711225364182,
|
|
"grad_norm": 0.5596352815628052,
|
|
"kl": 0.03955078125,
|
|
"learning_rate": 6.486753808845564e-07,
|
|
"loss": 0.2211,
|
|
"reward": -1.8425925374031067,
|
|
"reward_std": 0.754194438457489,
|
|
"rewards/confidence_reward": -2.6481481790542603,
|
|
"rewards/format_reward": 0.8055555820465088,
|
|
"step": 146
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2694.5001220703125,
|
|
"epoch": 0.12596401028277635,
|
|
"grad_norm": 0.0685691088438034,
|
|
"kl": 0.02557373046875,
|
|
"learning_rate": 6.435602608679916e-07,
|
|
"loss": 0.0014,
|
|
"reward": -2.305555582046509,
|
|
"reward_std": 0.0680413767695427,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.6944444477558136,
|
|
"step": 147
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2063.6111450195312,
|
|
"epoch": 0.1268209083119109,
|
|
"grad_norm": 0.34025242924690247,
|
|
"kl": 0.03564453125,
|
|
"learning_rate": 6.384324742897735e-07,
|
|
"loss": 0.1248,
|
|
"reward": -2.0092591047286987,
|
|
"reward_std": 0.7437996864318848,
|
|
"rewards/confidence_reward": -2.759259343147278,
|
|
"rewards/format_reward": 0.75,
|
|
"step": 148
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3310.27783203125,
|
|
"epoch": 0.12767780634104542,
|
|
"grad_norm": 0.17515520751476288,
|
|
"kl": 0.02813720703125,
|
|
"learning_rate": 6.332927153701215e-07,
|
|
"loss": 0.105,
|
|
"reward": -2.3518518805503845,
|
|
"reward_std": 0.44590386003255844,
|
|
"rewards/confidence_reward": -2.8518519401550293,
|
|
"rewards/format_reward": 0.5000000074505806,
|
|
"step": 149
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3576.0833740234375,
|
|
"epoch": 0.12853470437017994,
|
|
"grad_norm": 0.540116012096405,
|
|
"kl": 0.0565185546875,
|
|
"learning_rate": 6.281416799501187e-07,
|
|
"loss": 0.0537,
|
|
"reward": -2.638888955116272,
|
|
"reward_std": 0.3134361505508423,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3611111119389534,
|
|
"step": 150
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2819.3056640625,
|
|
"epoch": 0.1293916023993145,
|
|
"grad_norm": 0.3199455440044403,
|
|
"kl": 0.032958984375,
|
|
"learning_rate": 6.229800653975054e-07,
|
|
"loss": 0.1061,
|
|
"reward": -2.2314813137054443,
|
|
"reward_std": 0.6937831938266754,
|
|
"rewards/confidence_reward": -2.814814805984497,
|
|
"rewards/format_reward": 0.5833333432674408,
|
|
"step": 151
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2036.6388854980469,
|
|
"epoch": 0.13024850042844902,
|
|
"grad_norm": 0.25735169649124146,
|
|
"kl": 0.02972412109375,
|
|
"learning_rate": 6.178085705122674e-07,
|
|
"loss": 0.023,
|
|
"reward": -1.5749215185642242,
|
|
"reward_std": 1.027258962392807,
|
|
"rewards/confidence_reward": -2.3526992201805115,
|
|
"rewards/format_reward": 0.7777777910232544,
|
|
"step": 152
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3299.5833740234375,
|
|
"epoch": 0.13110539845758354,
|
|
"grad_norm": 0.14086535573005676,
|
|
"kl": 0.0323486328125,
|
|
"learning_rate": 6.126278954320294e-07,
|
|
"loss": 0.0307,
|
|
"reward": -2.42592591047287,
|
|
"reward_std": 0.4229704663157463,
|
|
"rewards/confidence_reward": -2.8148149251937866,
|
|
"rewards/format_reward": 0.38888888247311115,
|
|
"step": 153
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3406.5001220703125,
|
|
"epoch": 0.1319622964867181,
|
|
"grad_norm": 0.258766233921051,
|
|
"kl": 0.03106689453125,
|
|
"learning_rate": 6.074387415372676e-07,
|
|
"loss": 0.1326,
|
|
"reward": -2.5000001192092896,
|
|
"reward_std": 0.30821535736322403,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.5000000074505806,
|
|
"step": 154
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3476.388916015625,
|
|
"epoch": 0.1328191945158526,
|
|
"grad_norm": 0.20013348758220673,
|
|
"kl": 0.02679443359375,
|
|
"learning_rate": 6.022418113563535e-07,
|
|
"loss": 0.0258,
|
|
"reward": -2.472645878791809,
|
|
"reward_std": 0.39784130454063416,
|
|
"rewards/confidence_reward": -2.6948680877685547,
|
|
"rewards/format_reward": 0.22222222574055195,
|
|
"step": 155
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3603.3612060546875,
|
|
"epoch": 0.13367609254498714,
|
|
"grad_norm": 0.31663545966148376,
|
|
"kl": 0.027587890625,
|
|
"learning_rate": 5.97037808470444e-07,
|
|
"loss": 0.1284,
|
|
"reward": -2.4444445371627808,
|
|
"reward_std": 0.7150397002696991,
|
|
"rewards/confidence_reward": -2.888888955116272,
|
|
"rewards/format_reward": 0.4444444626569748,
|
|
"step": 156
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2918.666748046875,
|
|
"epoch": 0.13453299057412169,
|
|
"grad_norm": 0.2475612312555313,
|
|
"kl": 0.031494140625,
|
|
"learning_rate": 5.918274374182266e-07,
|
|
"loss": 0.0837,
|
|
"reward": -2.083333432674408,
|
|
"reward_std": 0.5944807976484299,
|
|
"rewards/confidence_reward": -2.666666626930237,
|
|
"rewards/format_reward": 0.5833333432674408,
|
|
"step": 157
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2515.8612060546875,
|
|
"epoch": 0.1353898886032562,
|
|
"grad_norm": 0.5117289423942566,
|
|
"kl": 0.0574951171875,
|
|
"learning_rate": 5.866114036005362e-07,
|
|
"loss": 0.2035,
|
|
"reward": -1.8888888955116272,
|
|
"reward_std": 0.8281392157077789,
|
|
"rewards/confidence_reward": -2.611111044883728,
|
|
"rewards/format_reward": 0.7222222089767456,
|
|
"step": 158
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3296.861083984375,
|
|
"epoch": 0.13624678663239073,
|
|
"grad_norm": 0.28699225187301636,
|
|
"kl": 0.02996826171875,
|
|
"learning_rate": 5.813904131848564e-07,
|
|
"loss": 0.1256,
|
|
"reward": -2.4629629850387573,
|
|
"reward_std": 0.4850034862756729,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.4444444626569748,
|
|
"step": 159
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3864.2777099609375,
|
|
"epoch": 0.13710368466152528,
|
|
"grad_norm": 0.21943415701389313,
|
|
"kl": 0.0389404296875,
|
|
"learning_rate": 5.761651730097142e-07,
|
|
"loss": 0.0667,
|
|
"reward": -2.6944445371627808,
|
|
"reward_std": 0.3995024561882019,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.305555559694767,
|
|
"step": 160
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3015.02783203125,
|
|
"epoch": 0.1379605826906598,
|
|
"grad_norm": 0.22274136543273926,
|
|
"kl": 0.03118896484375,
|
|
"learning_rate": 5.709363904889861e-07,
|
|
"loss": 0.0465,
|
|
"reward": -2.3518518209457397,
|
|
"reward_std": 0.47685495018959045,
|
|
"rewards/confidence_reward": -2.7962963581085205,
|
|
"rewards/format_reward": 0.4444444477558136,
|
|
"step": 161
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3587.5,
|
|
"epoch": 0.13881748071979436,
|
|
"grad_norm": 0.1837625652551651,
|
|
"kl": 0.042724609375,
|
|
"learning_rate": 5.657047735161255e-07,
|
|
"loss": 0.0174,
|
|
"reward": -2.5000001192092896,
|
|
"reward_std": 0.42796894907951355,
|
|
"rewards/confidence_reward": -2.833333373069763,
|
|
"rewards/format_reward": 0.3333333432674408,
|
|
"step": 162
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3590.4444580078125,
|
|
"epoch": 0.13967437874892888,
|
|
"grad_norm": 0.41009289026260376,
|
|
"kl": 0.0537109375,
|
|
"learning_rate": 5.604710303683253e-07,
|
|
"loss": 0.1195,
|
|
"reward": -2.4259259700775146,
|
|
"reward_std": 0.5152682885527611,
|
|
"rewards/confidence_reward": -2.7592592239379883,
|
|
"rewards/format_reward": 0.3333333432674408,
|
|
"step": 163
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3338.5,
|
|
"epoch": 0.1405312767780634,
|
|
"grad_norm": 0.22997497022151947,
|
|
"kl": 0.034423828125,
|
|
"learning_rate": 5.552358696106288e-07,
|
|
"loss": 0.1118,
|
|
"reward": -2.4907407760620117,
|
|
"reward_std": 0.5448963046073914,
|
|
"rewards/confidence_reward": -2.7962963581085205,
|
|
"rewards/format_reward": 0.305555559694767,
|
|
"step": 164
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3163.6944580078125,
|
|
"epoch": 0.14138817480719795,
|
|
"grad_norm": 0.29789188504219055,
|
|
"kl": 0.0311279296875,
|
|
"learning_rate": 5.5e-07,
|
|
"loss": 0.0972,
|
|
"reward": -2.2870370149612427,
|
|
"reward_std": 0.5160937160253525,
|
|
"rewards/confidence_reward": -2.7592592239379883,
|
|
"rewards/format_reward": 0.4722222238779068,
|
|
"step": 165
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2695.3333740234375,
|
|
"epoch": 0.14224507283633248,
|
|
"grad_norm": 0.25649595260620117,
|
|
"kl": 0.02960205078125,
|
|
"learning_rate": 5.447641303893714e-07,
|
|
"loss": 0.0524,
|
|
"reward": -1.7499999403953552,
|
|
"reward_std": 0.6183541268110275,
|
|
"rewards/confidence_reward": -2.3333334922790527,
|
|
"rewards/format_reward": 0.5833333432674408,
|
|
"step": 166
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2688.6944580078125,
|
|
"epoch": 0.143101970865467,
|
|
"grad_norm": 0.23350609838962555,
|
|
"kl": 0.037353515625,
|
|
"learning_rate": 5.395289696316747e-07,
|
|
"loss": 0.0783,
|
|
"reward": -2.1582546830177307,
|
|
"reward_std": 0.7079941481351852,
|
|
"rewards/confidence_reward": -2.7415881156921387,
|
|
"rewards/format_reward": 0.5833333358168602,
|
|
"step": 167
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3119.22216796875,
|
|
"epoch": 0.14395886889460155,
|
|
"grad_norm": 0.3477858006954193,
|
|
"kl": 0.0482177734375,
|
|
"learning_rate": 5.342952264838747e-07,
|
|
"loss": 0.0932,
|
|
"reward": -2.129629611968994,
|
|
"reward_std": 0.7057251632213593,
|
|
"rewards/confidence_reward": -2.7407407760620117,
|
|
"rewards/format_reward": 0.6111111044883728,
|
|
"step": 168
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2954.166748046875,
|
|
"epoch": 0.14481576692373607,
|
|
"grad_norm": 0.16720406711101532,
|
|
"kl": 0.03704833984375,
|
|
"learning_rate": 5.29063609511014e-07,
|
|
"loss": 0.0403,
|
|
"reward": -2.1481481194496155,
|
|
"reward_std": 0.20082136243581772,
|
|
"rewards/confidence_reward": -2.5925925970077515,
|
|
"rewards/format_reward": 0.4444444626569748,
|
|
"step": 169
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3575.3333740234375,
|
|
"epoch": 0.1456726649528706,
|
|
"grad_norm": 0.3258434534072876,
|
|
"kl": 0.0556640625,
|
|
"learning_rate": 5.238348269902859e-07,
|
|
"loss": 0.1092,
|
|
"reward": -2.305555522441864,
|
|
"reward_std": 0.49953543394804,
|
|
"rewards/confidence_reward": -2.7222222089767456,
|
|
"rewards/format_reward": 0.416666679084301,
|
|
"step": 170
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3646.138916015625,
|
|
"epoch": 0.14652956298200515,
|
|
"grad_norm": 0.22214293479919434,
|
|
"kl": 0.0496826171875,
|
|
"learning_rate": 5.186095868151436e-07,
|
|
"loss": 0.0377,
|
|
"reward": -2.6111111640930176,
|
|
"reward_std": 0.222149059176445,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3888888955116272,
|
|
"step": 171
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2912.25,
|
|
"epoch": 0.14738646101113967,
|
|
"grad_norm": 0.24840864539146423,
|
|
"kl": 0.0555419921875,
|
|
"learning_rate": 5.133885963994639e-07,
|
|
"loss": 0.0503,
|
|
"reward": -1.9814814329147339,
|
|
"reward_std": 0.8633435666561127,
|
|
"rewards/confidence_reward": -2.481481671333313,
|
|
"rewards/format_reward": 0.5,
|
|
"step": 172
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3765.916748046875,
|
|
"epoch": 0.14824335904027422,
|
|
"grad_norm": 0.1615857183933258,
|
|
"kl": 0.05029296875,
|
|
"learning_rate": 5.081725625817735e-07,
|
|
"loss": 0.0285,
|
|
"reward": -2.6666667461395264,
|
|
"reward_std": 0.17213259637355804,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3333333432674408,
|
|
"step": 173
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 4005.888916015625,
|
|
"epoch": 0.14910025706940874,
|
|
"grad_norm": 0.124207504093647,
|
|
"kl": 0.042724609375,
|
|
"learning_rate": 5.02962191529556e-07,
|
|
"loss": 0.0302,
|
|
"reward": -2.8611111640930176,
|
|
"reward_std": 0.15410767495632172,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.1388888955116272,
|
|
"step": 174
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3589.7222900390625,
|
|
"epoch": 0.14995715509854327,
|
|
"grad_norm": 0.38185247778892517,
|
|
"kl": 0.0694580078125,
|
|
"learning_rate": 4.977581886436462e-07,
|
|
"loss": 0.0932,
|
|
"reward": -2.5740740299224854,
|
|
"reward_std": 0.472199410200119,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.33333333022892475,
|
|
"step": 175
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3721.1666259765625,
|
|
"epoch": 0.15081405312767782,
|
|
"grad_norm": 0.35303542017936707,
|
|
"kl": 0.0589599609375,
|
|
"learning_rate": 4.925612584627324e-07,
|
|
"loss": 0.0818,
|
|
"reward": -2.722222328186035,
|
|
"reward_std": 0.3082153648138046,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.2777777910232544,
|
|
"step": 176
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3804.9444580078125,
|
|
"epoch": 0.15167095115681234,
|
|
"grad_norm": 0.37888699769973755,
|
|
"kl": 0.0810546875,
|
|
"learning_rate": 4.873721045679706e-07,
|
|
"loss": 0.0564,
|
|
"reward": -2.7777777910232544,
|
|
"reward_std": 0.2221490517258644,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.22222222574055195,
|
|
"step": 177
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2334.8056640625,
|
|
"epoch": 0.15252784918594686,
|
|
"grad_norm": 0.49078041315078735,
|
|
"kl": 0.076171875,
|
|
"learning_rate": 4.821914294877326e-07,
|
|
"loss": 0.0592,
|
|
"reward": -1.5222030878067017,
|
|
"reward_std": 0.7962049543857574,
|
|
"rewards/confidence_reward": -2.2722031474113464,
|
|
"rewards/format_reward": 0.75,
|
|
"step": 178
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3370.944580078125,
|
|
"epoch": 0.1533847472150814,
|
|
"grad_norm": 0.2731315791606903,
|
|
"kl": 0.07763671875,
|
|
"learning_rate": 4.770199346024947e-07,
|
|
"loss": 0.0635,
|
|
"reward": -2.2685184478759766,
|
|
"reward_std": 0.6701640486717224,
|
|
"rewards/confidence_reward": -2.6296297311782837,
|
|
"rewards/format_reward": 0.361111119389534,
|
|
"step": 179
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3099.888916015625,
|
|
"epoch": 0.15424164524421594,
|
|
"grad_norm": 0.24881142377853394,
|
|
"kl": 0.0810546875,
|
|
"learning_rate": 4.7185832004988133e-07,
|
|
"loss": 0.0609,
|
|
"reward": -2.3072502613067627,
|
|
"reward_std": 0.5294433236122131,
|
|
"rewards/confidence_reward": -2.6683614253997803,
|
|
"rewards/format_reward": 0.36111112125217915,
|
|
"step": 180
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3101.72216796875,
|
|
"epoch": 0.15509854327335046,
|
|
"grad_norm": 1.0561292171478271,
|
|
"kl": 0.08544921875,
|
|
"learning_rate": 4.667072846298785e-07,
|
|
"loss": 0.2226,
|
|
"reward": -2.259259283542633,
|
|
"reward_std": 0.7795112058520317,
|
|
"rewards/confidence_reward": -2.703703761100769,
|
|
"rewards/format_reward": 0.4444444514811039,
|
|
"step": 181
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3213.9444580078125,
|
|
"epoch": 0.155955441302485,
|
|
"grad_norm": 1.2878997325897217,
|
|
"kl": 0.116455078125,
|
|
"learning_rate": 4.6156752571022637e-07,
|
|
"loss": 0.1861,
|
|
"reward": -2.1574074029922485,
|
|
"reward_std": 0.556253120303154,
|
|
"rewards/confidence_reward": -2.6296297311782837,
|
|
"rewards/format_reward": 0.472222238779068,
|
|
"step": 182
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3509.138916015625,
|
|
"epoch": 0.15681233933161953,
|
|
"grad_norm": 0.22494396567344666,
|
|
"kl": 0.082763671875,
|
|
"learning_rate": 4.5643973913200837e-07,
|
|
"loss": 0.0309,
|
|
"reward": -2.453703761100769,
|
|
"reward_std": 0.44099532812833786,
|
|
"rewards/confidence_reward": -2.8148149251937866,
|
|
"rewards/format_reward": 0.361111119389534,
|
|
"step": 183
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2710.97216796875,
|
|
"epoch": 0.15766923736075408,
|
|
"grad_norm": 0.5510160326957703,
|
|
"kl": 0.1025390625,
|
|
"learning_rate": 4.513246191154434e-07,
|
|
"loss": 0.0585,
|
|
"reward": -1.4814814329147339,
|
|
"reward_std": 1.0165367126464844,
|
|
"rewards/confidence_reward": -2.2037038803100586,
|
|
"rewards/format_reward": 0.7222222089767456,
|
|
"step": 184
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2879.8333740234375,
|
|
"epoch": 0.1585261353898886,
|
|
"grad_norm": 0.36165350675582886,
|
|
"kl": 0.10498046875,
|
|
"learning_rate": 4.4622285816590186e-07,
|
|
"loss": 0.0904,
|
|
"reward": -2.0740740299224854,
|
|
"reward_std": 0.648275762796402,
|
|
"rewards/confidence_reward": -2.7407408952713013,
|
|
"rewards/format_reward": 0.6666666865348816,
|
|
"step": 185
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3437.3612060546875,
|
|
"epoch": 0.15938303341902313,
|
|
"grad_norm": 0.8590340614318848,
|
|
"kl": 0.116455078125,
|
|
"learning_rate": 4.4113514698014953e-07,
|
|
"loss": 0.1109,
|
|
"reward": -2.472222328186035,
|
|
"reward_std": 0.5155959874391556,
|
|
"rewards/confidence_reward": -2.888888955116272,
|
|
"rewards/format_reward": 0.4166666641831398,
|
|
"step": 186
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2466.5,
|
|
"epoch": 0.16023993144815768,
|
|
"grad_norm": 1.367401123046875,
|
|
"kl": 0.115234375,
|
|
"learning_rate": 4.360621743528392e-07,
|
|
"loss": 0.1976,
|
|
"reward": -2.037884473800659,
|
|
"reward_std": 0.6841873377561569,
|
|
"rewards/confidence_reward": -2.6489956378936768,
|
|
"rewards/format_reward": 0.6111111044883728,
|
|
"step": 187
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3534.27783203125,
|
|
"epoch": 0.1610968294772922,
|
|
"grad_norm": 0.3216284215450287,
|
|
"kl": 0.12939453125,
|
|
"learning_rate": 4.3100462708325914e-07,
|
|
"loss": 0.082,
|
|
"reward": -2.4722222089767456,
|
|
"reward_std": 0.3762567266821861,
|
|
"rewards/confidence_reward": -2.833333373069763,
|
|
"rewards/format_reward": 0.36111112125217915,
|
|
"step": 188
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3046.138916015625,
|
|
"epoch": 0.16195372750642673,
|
|
"grad_norm": 0.9523129463195801,
|
|
"kl": 0.142578125,
|
|
"learning_rate": 4.2596318988235037e-07,
|
|
"loss": 0.1065,
|
|
"reward": -2.120370388031006,
|
|
"reward_std": 0.5169950574636459,
|
|
"rewards/confidence_reward": -2.5370370149612427,
|
|
"rewards/format_reward": 0.4166666716337204,
|
|
"step": 189
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3720.416748046875,
|
|
"epoch": 0.16281062553556128,
|
|
"grad_norm": 3.160400629043579,
|
|
"kl": 0.734375,
|
|
"learning_rate": 4.209385452800095e-07,
|
|
"loss": 0.103,
|
|
"reward": -2.6944445371627808,
|
|
"reward_std": 0.32624026387929916,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3055555671453476,
|
|
"step": 190
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2635.75,
|
|
"epoch": 0.1636675235646958,
|
|
"grad_norm": 0.7508611679077148,
|
|
"kl": 0.119384765625,
|
|
"learning_rate": 4.1593137353268303e-07,
|
|
"loss": 0.1062,
|
|
"reward": -1.9452918767929077,
|
|
"reward_std": 0.717644564807415,
|
|
"rewards/confidence_reward": -2.7786251306533813,
|
|
"rewards/format_reward": 0.8333333432674408,
|
|
"step": 191
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3603.5277099609375,
|
|
"epoch": 0.16452442159383032,
|
|
"grad_norm": 0.4415644109249115,
|
|
"kl": 0.17431640625,
|
|
"learning_rate": 4.1094235253127374e-07,
|
|
"loss": 0.0745,
|
|
"reward": -2.5000001192092896,
|
|
"reward_std": 0.34894853830337524,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.444444440305233,
|
|
"step": 192
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3744.3612060546875,
|
|
"epoch": 0.16538131962296487,
|
|
"grad_norm": 0.7602421045303345,
|
|
"kl": 0.2197265625,
|
|
"learning_rate": 4.059721577093628e-07,
|
|
"loss": 0.1049,
|
|
"reward": -2.60185170173645,
|
|
"reward_std": 0.5628926083445549,
|
|
"rewards/confidence_reward": -2.8518519401550293,
|
|
"rewards/format_reward": 0.2500000074505806,
|
|
"step": 193
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3876.5833740234375,
|
|
"epoch": 0.1662382176520994,
|
|
"grad_norm": 0.7295367121696472,
|
|
"kl": 0.2158203125,
|
|
"learning_rate": 4.0102146195176887e-07,
|
|
"loss": 0.0704,
|
|
"reward": -2.6111111640930176,
|
|
"reward_std": 0.5082770437002182,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.3333333432674408,
|
|
"step": 194
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2431.638916015625,
|
|
"epoch": 0.16709511568123395,
|
|
"grad_norm": 0.7587151527404785,
|
|
"kl": 0.18994140625,
|
|
"learning_rate": 3.9609093550344907e-07,
|
|
"loss": 0.0699,
|
|
"reward": -1.851851761341095,
|
|
"reward_std": 0.8921651542186737,
|
|
"rewards/confidence_reward": -2.574074149131775,
|
|
"rewards/format_reward": 0.722222238779068,
|
|
"step": 195
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3446.888916015625,
|
|
"epoch": 0.16795201371036847,
|
|
"grad_norm": 1.1345640420913696,
|
|
"kl": 0.22265625,
|
|
"learning_rate": 3.911812458787591e-07,
|
|
"loss": 0.0911,
|
|
"reward": -2.407407283782959,
|
|
"reward_std": 0.4854222238063812,
|
|
"rewards/confidence_reward": -2.8518519401550293,
|
|
"rewards/format_reward": 0.4444444626569748,
|
|
"step": 196
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3630.3612060546875,
|
|
"epoch": 0.168808911739503,
|
|
"grad_norm": 1.328270673751831,
|
|
"kl": 0.26318359375,
|
|
"learning_rate": 3.86293057771082e-07,
|
|
"loss": 0.158,
|
|
"reward": -2.6018518209457397,
|
|
"reward_std": 0.5880227982997894,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.305555559694767,
|
|
"step": 197
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3384.02783203125,
|
|
"epoch": 0.16966580976863754,
|
|
"grad_norm": 0.4739760160446167,
|
|
"kl": 0.2294921875,
|
|
"learning_rate": 3.8142703296283953e-07,
|
|
"loss": 0.0634,
|
|
"reward": -2.5,
|
|
"reward_std": 0.222149059176445,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.5000000149011612,
|
|
"step": 198
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3382.8055419921875,
|
|
"epoch": 0.17052270779777207,
|
|
"grad_norm": 1.6106473207473755,
|
|
"kl": 0.306640625,
|
|
"learning_rate": 3.7658383023589833e-07,
|
|
"loss": 0.1537,
|
|
"reward": -2.3333333134651184,
|
|
"reward_std": 0.6125056594610214,
|
|
"rewards/confidence_reward": -2.7222222089767456,
|
|
"rewards/format_reward": 0.3888888880610466,
|
|
"step": 199
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3306.388916015625,
|
|
"epoch": 0.1713796058269066,
|
|
"grad_norm": 1.5265542268753052,
|
|
"kl": 0.2734375,
|
|
"learning_rate": 3.7176410528237945e-07,
|
|
"loss": 0.1782,
|
|
"reward": -2.1851852536201477,
|
|
"reward_std": 0.5692443251609802,
|
|
"rewards/confidence_reward": -2.629629611968994,
|
|
"rewards/format_reward": 0.4444444552063942,
|
|
"step": 200
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3631.8612060546875,
|
|
"epoch": 0.17223650385604114,
|
|
"grad_norm": 0.6827831864356995,
|
|
"kl": 0.2685546875,
|
|
"learning_rate": 3.6696851061588994e-07,
|
|
"loss": 0.0805,
|
|
"reward": -2.4814815521240234,
|
|
"reward_std": 0.5552610009908676,
|
|
"rewards/confidence_reward": -2.8148149251937866,
|
|
"rewards/format_reward": 0.3333333432674408,
|
|
"step": 201
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2938.4166259765625,
|
|
"epoch": 0.17309340188517566,
|
|
"grad_norm": 0.9321132898330688,
|
|
"kl": 0.30078125,
|
|
"learning_rate": 3.62197695483182e-07,
|
|
"loss": 0.0521,
|
|
"reward": -1.944444477558136,
|
|
"reward_std": 0.6303547322750092,
|
|
"rewards/confidence_reward": -2.3888888359069824,
|
|
"rewards/format_reward": 0.4444444626569748,
|
|
"step": 202
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3787.6112060546875,
|
|
"epoch": 0.17395029991431019,
|
|
"grad_norm": 0.8994116187095642,
|
|
"kl": 0.31640625,
|
|
"learning_rate": 3.5745230577625573e-07,
|
|
"loss": 0.0792,
|
|
"reward": -2.75,
|
|
"reward_std": 0.2901904284954071,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.2500000037252903,
|
|
"step": 203
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2426.9444580078125,
|
|
"epoch": 0.17480719794344474,
|
|
"grad_norm": 1.54979407787323,
|
|
"kl": 0.265625,
|
|
"learning_rate": 3.5273298394491515e-07,
|
|
"loss": 0.0357,
|
|
"reward": -1.6300533711910248,
|
|
"reward_std": 0.7251835763454437,
|
|
"rewards/confidence_reward": -2.3800532817840576,
|
|
"rewards/format_reward": 0.75,
|
|
"step": 204
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3264.6112060546875,
|
|
"epoch": 0.17566409597257926,
|
|
"grad_norm": 2.0934314727783203,
|
|
"kl": 0.22802734375,
|
|
"learning_rate": 3.4804036890979205e-07,
|
|
"loss": 0.1002,
|
|
"reward": -2.2971436977386475,
|
|
"reward_std": 0.725425198674202,
|
|
"rewards/confidence_reward": -2.7415882349014282,
|
|
"rewards/format_reward": 0.4444444477558136,
|
|
"step": 205
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3399.5001220703125,
|
|
"epoch": 0.17652099400171378,
|
|
"grad_norm": 0.9074891209602356,
|
|
"kl": 0.2763671875,
|
|
"learning_rate": 3.433750959758446e-07,
|
|
"loss": 0.0878,
|
|
"reward": -2.287036955356598,
|
|
"reward_std": 0.3180917277932167,
|
|
"rewards/confidence_reward": -2.5370370149612427,
|
|
"rewards/format_reward": 0.2500000074505806,
|
|
"step": 206
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 4017.02783203125,
|
|
"epoch": 0.17737789203084833,
|
|
"grad_norm": 0.922761857509613,
|
|
"kl": 0.2080078125,
|
|
"learning_rate": 3.387377967463493e-07,
|
|
"loss": 0.0294,
|
|
"reward": -2.638888955116272,
|
|
"reward_std": 0.3995024412870407,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.361111119389534,
|
|
"step": 207
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3856.416748046875,
|
|
"epoch": 0.17823479005998286,
|
|
"grad_norm": 1.1995811462402344,
|
|
"kl": 0.3154296875,
|
|
"learning_rate": 3.3412909903738936e-07,
|
|
"loss": 0.078,
|
|
"reward": -2.7777777910232544,
|
|
"reward_std": 0.30821534991264343,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.2222222313284874,
|
|
"step": 208
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2836.25,
|
|
"epoch": 0.1790916880891174,
|
|
"grad_norm": 1.958234429359436,
|
|
"kl": 0.138671875,
|
|
"learning_rate": 3.295496267928609e-07,
|
|
"loss": 0.078,
|
|
"reward": -1.870370328426361,
|
|
"reward_std": 0.5447776764631271,
|
|
"rewards/confidence_reward": -2.4814815521240234,
|
|
"rewards/format_reward": 0.6111111342906952,
|
|
"step": 209
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3240.2777099609375,
|
|
"epoch": 0.17994858611825193,
|
|
"grad_norm": 6.24537992477417,
|
|
"kl": 0.2060546875,
|
|
"learning_rate": 3.250000000000001e-07,
|
|
"loss": 0.1677,
|
|
"reward": -2.046296238899231,
|
|
"reward_std": 0.6235695481300354,
|
|
"rewards/confidence_reward": -2.6296297311782837,
|
|
"rewards/format_reward": 0.5833333432674408,
|
|
"step": 210
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3614.166748046875,
|
|
"epoch": 0.18080548414738645,
|
|
"grad_norm": 0.7317115068435669,
|
|
"kl": 0.1962890625,
|
|
"learning_rate": 3.204808346054461e-07,
|
|
"loss": 0.0303,
|
|
"reward": -2.694444417953491,
|
|
"reward_std": 0.22736986726522446,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.2500000074505806,
|
|
"step": 211
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3610.52783203125,
|
|
"epoch": 0.181662382176521,
|
|
"grad_norm": 1.1794308423995972,
|
|
"kl": 0.22119140625,
|
|
"learning_rate": 3.159927424318531e-07,
|
|
"loss": 0.0538,
|
|
"reward": -2.666666626930237,
|
|
"reward_std": 0.2453947737812996,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3333333358168602,
|
|
"step": 212
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3340.8333740234375,
|
|
"epoch": 0.18251928020565553,
|
|
"grad_norm": 1.3902448415756226,
|
|
"kl": 0.216796875,
|
|
"learning_rate": 3.115363310950578e-07,
|
|
"loss": 0.0582,
|
|
"reward": -2.5277777910232544,
|
|
"reward_std": 0.35992757976055145,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.4166666865348816,
|
|
"step": 213
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3454.6666259765625,
|
|
"epoch": 0.18337617823479005,
|
|
"grad_norm": 0.36289045214653015,
|
|
"kl": 0.1962890625,
|
|
"learning_rate": 3.0711220392181934e-07,
|
|
"loss": 0.0291,
|
|
"reward": -2.3055556416511536,
|
|
"reward_std": 0.4583980068564415,
|
|
"rewards/confidence_reward": -2.7222222089767456,
|
|
"rewards/format_reward": 0.4166666641831398,
|
|
"step": 214
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3966.5555419921875,
|
|
"epoch": 0.1842330762639246,
|
|
"grad_norm": 1.2311605215072632,
|
|
"kl": 0.2685546875,
|
|
"learning_rate": 3.027209598681373e-07,
|
|
"loss": 0.0363,
|
|
"reward": -2.638888955116272,
|
|
"reward_std": 0.4123065620660782,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.361111119389534,
|
|
"step": 215
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3175.3333740234375,
|
|
"epoch": 0.18508997429305912,
|
|
"grad_norm": 4.940278053283691,
|
|
"kl": 0.24658203125,
|
|
"learning_rate": 2.9836319343816397e-07,
|
|
"loss": 0.1492,
|
|
"reward": -2.037036955356598,
|
|
"reward_std": 0.7587747424840927,
|
|
"rewards/confidence_reward": -2.7592592239379883,
|
|
"rewards/format_reward": 0.722222238779068,
|
|
"step": 216
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3212.5277099609375,
|
|
"epoch": 0.18594687232219365,
|
|
"grad_norm": 7.882691383361816,
|
|
"kl": 0.310546875,
|
|
"learning_rate": 2.9403949460371677e-07,
|
|
"loss": 0.2234,
|
|
"reward": -2.2685185074806213,
|
|
"reward_std": 0.5562530681490898,
|
|
"rewards/confidence_reward": -2.6296297311782837,
|
|
"rewards/format_reward": 0.3611111082136631,
|
|
"step": 217
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3286.27783203125,
|
|
"epoch": 0.1868037703513282,
|
|
"grad_norm": 2.209719657897949,
|
|
"kl": 0.3505859375,
|
|
"learning_rate": 2.897504487244061e-07,
|
|
"loss": 0.0156,
|
|
"reward": -2.370370388031006,
|
|
"reward_std": 0.5552610009908676,
|
|
"rewards/confidence_reward": -2.8148149251937866,
|
|
"rewards/format_reward": 0.444444440305233,
|
|
"step": 218
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3131.8333740234375,
|
|
"epoch": 0.18766066838046272,
|
|
"grad_norm": 6.650064945220947,
|
|
"kl": 0.3505859375,
|
|
"learning_rate": 2.854966364683872e-07,
|
|
"loss": 0.2357,
|
|
"reward": -1.9518136978149414,
|
|
"reward_std": 0.8104376196861267,
|
|
"rewards/confidence_reward": -2.4518136978149414,
|
|
"rewards/format_reward": 0.5000000149011612,
|
|
"step": 219
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3573.47216796875,
|
|
"epoch": 0.18851756640959727,
|
|
"grad_norm": 1.0854192972183228,
|
|
"kl": 0.427734375,
|
|
"learning_rate": 2.812786337337463e-07,
|
|
"loss": 0.1052,
|
|
"reward": -2.5185184478759766,
|
|
"reward_std": 0.5389864593744278,
|
|
"rewards/confidence_reward": -2.8518518209457397,
|
|
"rewards/format_reward": 0.3333333432674408,
|
|
"step": 220
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3518.25,
|
|
"epoch": 0.1893744644387318,
|
|
"grad_norm": 2.382499933242798,
|
|
"kl": 0.46484375,
|
|
"learning_rate": 2.770970115705341e-07,
|
|
"loss": 0.0109,
|
|
"reward": -2.740740656852722,
|
|
"reward_std": 0.22680462896823883,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.1666666716337204,
|
|
"step": 221
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3254.4166259765625,
|
|
"epoch": 0.19023136246786632,
|
|
"grad_norm": 1.3864353895187378,
|
|
"kl": 0.4208984375,
|
|
"learning_rate": 2.729523361034538e-07,
|
|
"loss": 0.0677,
|
|
"reward": -2.388888955116272,
|
|
"reward_std": 0.3582318052649498,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.5555555522441864,
|
|
"step": 222
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3346.9444580078125,
|
|
"epoch": 0.19108826049700087,
|
|
"grad_norm": 1.5457087755203247,
|
|
"kl": 0.4716796875,
|
|
"learning_rate": 2.68845168455218e-07,
|
|
"loss": 0.0491,
|
|
"reward": -2.388888955116272,
|
|
"reward_std": 0.41752736270427704,
|
|
"rewards/confidence_reward": -2.888888955116272,
|
|
"rewards/format_reward": 0.5,
|
|
"step": 223
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2871.02783203125,
|
|
"epoch": 0.1919451585261354,
|
|
"grad_norm": 2.262847661972046,
|
|
"kl": 0.3876953125,
|
|
"learning_rate": 2.6477606467058035e-07,
|
|
"loss": 0.1352,
|
|
"reward": -1.7870370149612427,
|
|
"reward_std": 0.7662279903888702,
|
|
"rewards/confidence_reward": -2.3703705072402954,
|
|
"rewards/format_reward": 0.5833333432674408,
|
|
"step": 224
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3113.6944580078125,
|
|
"epoch": 0.1928020565552699,
|
|
"grad_norm": 1.4508579969406128,
|
|
"kl": 0.3828125,
|
|
"learning_rate": 2.6074557564105724e-07,
|
|
"loss": 0.1048,
|
|
"reward": -1.7592592239379883,
|
|
"reward_std": 0.5638266205787659,
|
|
"rewards/confidence_reward": -2.3703704476356506,
|
|
"rewards/format_reward": 0.6111111044883728,
|
|
"step": 225
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3247.4444580078125,
|
|
"epoch": 0.19365895458440446,
|
|
"grad_norm": 0.9147429466247559,
|
|
"kl": 0.513671875,
|
|
"learning_rate": 2.567542470303452e-07,
|
|
"loss": 0.1006,
|
|
"reward": -2.1296297311782837,
|
|
"reward_std": 0.44895367324352264,
|
|
"rewards/confidence_reward": -2.574074149131775,
|
|
"rewards/format_reward": 0.4444444477558136,
|
|
"step": 226
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2793.9722290039062,
|
|
"epoch": 0.194515852613539,
|
|
"grad_norm": 1.462504506111145,
|
|
"kl": 0.3876953125,
|
|
"learning_rate": 2.528026192004466e-07,
|
|
"loss": 0.0483,
|
|
"reward": -1.8333333134651184,
|
|
"reward_std": 0.6497750803828239,
|
|
"rewards/confidence_reward": -2.3888890147209167,
|
|
"rewards/format_reward": 0.5555555522441864,
|
|
"step": 227
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3342.916748046875,
|
|
"epoch": 0.1953727506426735,
|
|
"grad_norm": 1.5023483037948608,
|
|
"kl": 0.494140625,
|
|
"learning_rate": 2.488912271385139e-07,
|
|
"loss": 0.0784,
|
|
"reward": -2.4907407760620117,
|
|
"reward_std": 0.4541744738817215,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.4166666641831398,
|
|
"step": 228
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3135.138916015625,
|
|
"epoch": 0.19622964867180806,
|
|
"grad_norm": 2.827260971069336,
|
|
"kl": 0.53125,
|
|
"learning_rate": 2.450206003844205e-07,
|
|
"loss": 0.1925,
|
|
"reward": -2.130053400993347,
|
|
"reward_std": 0.9212456345558167,
|
|
"rewards/confidence_reward": -2.713386654853821,
|
|
"rewards/format_reward": 0.5833333432674408,
|
|
"step": 229
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2381.2500610351562,
|
|
"epoch": 0.19708654670094258,
|
|
"grad_norm": 4.513103008270264,
|
|
"kl": 0.28125,
|
|
"learning_rate": 2.411912629590699e-07,
|
|
"loss": 0.1358,
|
|
"reward": -1.7693658471107483,
|
|
"reward_std": 1.0916875302791595,
|
|
"rewards/confidence_reward": -2.4638103246688843,
|
|
"rewards/format_reward": 0.6944444477558136,
|
|
"step": 230
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3553.3055419921875,
|
|
"epoch": 0.19794344473007713,
|
|
"grad_norm": 2.0678796768188477,
|
|
"kl": 0.4443359375,
|
|
"learning_rate": 2.374037332934512e-07,
|
|
"loss": 0.0392,
|
|
"reward": -2.5,
|
|
"reward_std": 0.27888670563697815,
|
|
"rewards/confidence_reward": -2.833333373069763,
|
|
"rewards/format_reward": 0.3333333432674408,
|
|
"step": 231
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3584.0833740234375,
|
|
"epoch": 0.19880034275921166,
|
|
"grad_norm": 2.108079433441162,
|
|
"kl": 0.408203125,
|
|
"learning_rate": 2.336585241584522e-07,
|
|
"loss": 0.0707,
|
|
"reward": -2.6944445371627808,
|
|
"reward_std": 0.06804138422012329,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3055555671453476,
|
|
"step": 232
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3742.2777099609375,
|
|
"epoch": 0.19965724078834618,
|
|
"grad_norm": 2.253603219985962,
|
|
"kl": 0.4912109375,
|
|
"learning_rate": 2.299561425954383e-07,
|
|
"loss": 0.1806,
|
|
"reward": -2.6388888359069824,
|
|
"reward_std": 0.47208072245121,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.305555559694767,
|
|
"step": 233
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2601.3056640625,
|
|
"epoch": 0.20051413881748073,
|
|
"grad_norm": 4.904290676116943,
|
|
"kl": 0.3974609375,
|
|
"learning_rate": 2.2629708984760706e-07,
|
|
"loss": 0.2103,
|
|
"reward": -1.3475482165813446,
|
|
"reward_std": 1.2650677561759949,
|
|
"rewards/confidence_reward": -2.0975481271743774,
|
|
"rewards/format_reward": 0.75,
|
|
"step": 234
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2671.6944580078125,
|
|
"epoch": 0.20137103684661525,
|
|
"grad_norm": 1.6857246160507202,
|
|
"kl": 0.3486328125,
|
|
"learning_rate": 2.2268186129212807e-07,
|
|
"loss": 0.0329,
|
|
"reward": -1.7870370149612427,
|
|
"reward_std": 0.40415799617767334,
|
|
"rewards/confidence_reward": -2.425925850868225,
|
|
"rewards/format_reward": 0.638888880610466,
|
|
"step": 235
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3342.5277099609375,
|
|
"epoch": 0.20222793487574978,
|
|
"grad_norm": 1.9274342060089111,
|
|
"kl": 0.4453125,
|
|
"learning_rate": 2.1911094637307714e-07,
|
|
"loss": 0.108,
|
|
"reward": -2.3796294927597046,
|
|
"reward_std": 0.48020143806934357,
|
|
"rewards/confidence_reward": -2.8518518209457397,
|
|
"rewards/format_reward": 0.4722222238779068,
|
|
"step": 236
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2953.388916015625,
|
|
"epoch": 0.20308483290488433,
|
|
"grad_norm": 2.970611810684204,
|
|
"kl": 0.4501953125,
|
|
"learning_rate": 2.1558482853517253e-07,
|
|
"loss": 0.1648,
|
|
"reward": -1.9360325932502747,
|
|
"reward_std": 0.8466525003314018,
|
|
"rewards/confidence_reward": -2.4082547426223755,
|
|
"rewards/format_reward": 0.4722222164273262,
|
|
"step": 237
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3137.4166259765625,
|
|
"epoch": 0.20394173093401885,
|
|
"grad_norm": 0.7161883115768433,
|
|
"kl": 0.5009765625,
|
|
"learning_rate": 2.1210398515832536e-07,
|
|
"loss": 0.0883,
|
|
"reward": -2.138888895511627,
|
|
"reward_std": 0.7037928402423859,
|
|
"rewards/confidence_reward": -2.6111111640930176,
|
|
"rewards/format_reward": 0.4722222164273262,
|
|
"step": 238
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3184.7222900390625,
|
|
"epoch": 0.20479862896315337,
|
|
"grad_norm": 2.5145044326782227,
|
|
"kl": 0.638671875,
|
|
"learning_rate": 2.08668887493009e-07,
|
|
"loss": 0.1968,
|
|
"reward": -2.231481432914734,
|
|
"reward_std": 1.0531733930110931,
|
|
"rewards/confidence_reward": -2.703703761100769,
|
|
"rewards/format_reward": 0.472222238779068,
|
|
"step": 239
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3364.638916015625,
|
|
"epoch": 0.20565552699228792,
|
|
"grad_norm": 1.4906601905822754,
|
|
"kl": 0.533203125,
|
|
"learning_rate": 2.0528000059645995e-07,
|
|
"loss": 0.1209,
|
|
"reward": -2.3888888359069824,
|
|
"reward_std": 0.3082153648138046,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.6111111342906952,
|
|
"step": 240
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3299.583251953125,
|
|
"epoch": 0.20651242502142245,
|
|
"grad_norm": 2.7216079235076904,
|
|
"kl": 0.529296875,
|
|
"learning_rate": 2.0193778326971628e-07,
|
|
"loss": 0.089,
|
|
"reward": -2.4722222089767456,
|
|
"reward_std": 0.24017397314310074,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.5277777910232544,
|
|
"step": 241
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3405.8055419921875,
|
|
"epoch": 0.207369323050557,
|
|
"grad_norm": 2.5555975437164307,
|
|
"kl": 0.71875,
|
|
"learning_rate": 1.986426879955034e-07,
|
|
"loss": 0.1951,
|
|
"reward": -2.0925925374031067,
|
|
"reward_std": 0.9167544841766357,
|
|
"rewards/confidence_reward": -2.5370370149612427,
|
|
"rewards/format_reward": 0.4444444477558136,
|
|
"step": 242
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3524.3055419921875,
|
|
"epoch": 0.20822622107969152,
|
|
"grad_norm": 1.0115513801574707,
|
|
"kl": 0.794921875,
|
|
"learning_rate": 1.9539516087697517e-07,
|
|
"loss": 0.1157,
|
|
"reward": -2.666666626930237,
|
|
"reward_std": 0.30821534991264343,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3333333358168602,
|
|
"step": 243
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3453.833251953125,
|
|
"epoch": 0.20908311910882604,
|
|
"grad_norm": 2.69391131401062,
|
|
"kl": 0.6103515625,
|
|
"learning_rate": 1.9219564157731844e-07,
|
|
"loss": 0.1843,
|
|
"reward": -2.5277777910232544,
|
|
"reward_std": 0.5088144540786743,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.472222238779068,
|
|
"step": 244
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3176.9444580078125,
|
|
"epoch": 0.2099400171379606,
|
|
"grad_norm": 3.2699921131134033,
|
|
"kl": 0.609375,
|
|
"learning_rate": 1.8904456326023027e-07,
|
|
"loss": 0.1116,
|
|
"reward": -1.9722222685813904,
|
|
"reward_std": 0.5316601991653442,
|
|
"rewards/confidence_reward": -2.5,
|
|
"rewards/format_reward": 0.5277777910232544,
|
|
"step": 245
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2784.6944580078125,
|
|
"epoch": 0.21079691516709512,
|
|
"grad_norm": 1.8682541847229004,
|
|
"kl": 0.56640625,
|
|
"learning_rate": 1.8594235253127372e-07,
|
|
"loss": 0.1448,
|
|
"reward": -1.6203704178333282,
|
|
"reward_std": 0.6126016080379486,
|
|
"rewards/confidence_reward": -2.2037037014961243,
|
|
"rewards/format_reward": 0.5833333283662796,
|
|
"step": 246
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3728.638916015625,
|
|
"epoch": 0.21165381319622964,
|
|
"grad_norm": 3.5428807735443115,
|
|
"kl": 0.78515625,
|
|
"learning_rate": 1.8288942938012267e-07,
|
|
"loss": 0.0706,
|
|
"reward": -2.490740656852722,
|
|
"reward_std": 0.46697860956192017,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.4166666716337204,
|
|
"step": 247
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3008.138916015625,
|
|
"epoch": 0.2125107112253642,
|
|
"grad_norm": 1.3764863014221191,
|
|
"kl": 0.57421875,
|
|
"learning_rate": 1.7988620712370195e-07,
|
|
"loss": 0.0807,
|
|
"reward": -1.9478338956832886,
|
|
"reward_std": 0.7096292227506638,
|
|
"rewards/confidence_reward": -2.4478338956832886,
|
|
"rewards/format_reward": 0.5,
|
|
"step": 248
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3118.1944580078125,
|
|
"epoch": 0.2133676092544987,
|
|
"grad_norm": 3.4032788276672363,
|
|
"kl": 0.5830078125,
|
|
"learning_rate": 1.7693309235023127e-07,
|
|
"loss": 0.1638,
|
|
"reward": -2.0277777314186096,
|
|
"reward_std": 0.8689809292554855,
|
|
"rewards/confidence_reward": -2.666666626930237,
|
|
"rewards/format_reward": 0.6388888955116272,
|
|
"step": 249
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3191.4166259765625,
|
|
"epoch": 0.21422450728363324,
|
|
"grad_norm": 1.8875763416290283,
|
|
"kl": 0.689453125,
|
|
"learning_rate": 1.7403048486417868e-07,
|
|
"loss": 0.1055,
|
|
"reward": -1.9545510709285736,
|
|
"reward_std": 0.48649683594703674,
|
|
"rewards/confidence_reward": -2.26010662317276,
|
|
"rewards/format_reward": 0.3055555522441864,
|
|
"step": 250
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2924.638916015625,
|
|
"epoch": 0.2150814053127678,
|
|
"grad_norm": 2.471829414367676,
|
|
"kl": 0.5673828125,
|
|
"learning_rate": 1.711787776321341e-07,
|
|
"loss": 0.1465,
|
|
"reward": -2.101851761341095,
|
|
"reward_std": 0.6704803705215454,
|
|
"rewards/confidence_reward": -2.7407407760620117,
|
|
"rewards/format_reward": 0.638888880610466,
|
|
"step": 251
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3481.8056640625,
|
|
"epoch": 0.2159383033419023,
|
|
"grad_norm": 1.7675886154174805,
|
|
"kl": 0.8046875,
|
|
"learning_rate": 1.6837835672960831e-07,
|
|
"loss": 0.1342,
|
|
"reward": -2.314814805984497,
|
|
"reward_std": 0.5083017498254776,
|
|
"rewards/confidence_reward": -2.7037036418914795,
|
|
"rewards/format_reward": 0.3888888955116272,
|
|
"step": 252
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3527.4166259765625,
|
|
"epoch": 0.21679520137103683,
|
|
"grad_norm": 2.999549627304077,
|
|
"kl": 0.759765625,
|
|
"learning_rate": 1.6562960128876353e-07,
|
|
"loss": 0.0954,
|
|
"reward": -2.5185184478759766,
|
|
"reward_std": 0.3628873750567436,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.3888888955116272,
|
|
"step": 253
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3826.916748046875,
|
|
"epoch": 0.21765209940017138,
|
|
"grad_norm": 4.181079864501953,
|
|
"kl": 0.927734375,
|
|
"learning_rate": 1.6293288344708566e-07,
|
|
"loss": 0.0935,
|
|
"reward": -2.6944445371627808,
|
|
"reward_std": 0.3134361654520035,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.305555559694767,
|
|
"step": 254
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3448.47216796875,
|
|
"epoch": 0.2185089974293059,
|
|
"grad_norm": 1.8785117864608765,
|
|
"kl": 0.7265625,
|
|
"learning_rate": 1.6028856829700258e-07,
|
|
"loss": 0.0921,
|
|
"reward": -2.4175140261650085,
|
|
"reward_std": 0.5672558695077896,
|
|
"rewards/confidence_reward": -2.83418071269989,
|
|
"rewards/format_reward": 0.4166666604578495,
|
|
"step": 255
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3920.5833740234375,
|
|
"epoch": 0.21936589545844046,
|
|
"grad_norm": 7.422764778137207,
|
|
"kl": 0.84375,
|
|
"learning_rate": 1.5769701383645698e-07,
|
|
"loss": 0.034,
|
|
"reward": -2.7777777910232544,
|
|
"reward_std": 0.1360827535390854,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.2222222276031971,
|
|
"step": 256
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3276.9444580078125,
|
|
"epoch": 0.22022279348757498,
|
|
"grad_norm": 6.746924877166748,
|
|
"kl": 0.640625,
|
|
"learning_rate": 1.551585709204381e-07,
|
|
"loss": 0.1587,
|
|
"reward": -1.9537036418914795,
|
|
"reward_std": 0.6170237511396408,
|
|
"rewards/confidence_reward": -2.425925850868225,
|
|
"rewards/format_reward": 0.4722222238779068,
|
|
"step": 257
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2808.27783203125,
|
|
"epoch": 0.2210796915167095,
|
|
"grad_norm": 9.45202350616455,
|
|
"kl": 0.42578125,
|
|
"learning_rate": 1.5267358321348285e-07,
|
|
"loss": 0.224,
|
|
"reward": -1.5564029514789581,
|
|
"reward_std": 0.6021379753947258,
|
|
"rewards/confidence_reward": -2.0564029812812805,
|
|
"rewards/format_reward": 0.5,
|
|
"step": 258
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 1506.3611145019531,
|
|
"epoch": 0.22193658954584405,
|
|
"grad_norm": 1.4079711437225342,
|
|
"kl": 0.208984375,
|
|
"learning_rate": 1.5024238714314825e-07,
|
|
"loss": 0.0526,
|
|
"reward": -0.9999999701976776,
|
|
"reward_std": 1.1379348039627075,
|
|
"rewards/confidence_reward": -2.0000001192092896,
|
|
"rewards/format_reward": 1.0,
|
|
"step": 259
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3447.47216796875,
|
|
"epoch": 0.22279348757497858,
|
|
"grad_norm": 2.1422104835510254,
|
|
"kl": 0.6162109375,
|
|
"learning_rate": 1.4786531185446452e-07,
|
|
"loss": 0.1618,
|
|
"reward": -2.4018328189849854,
|
|
"reward_std": 0.5933989137411118,
|
|
"rewards/confidence_reward": -2.7907216548919678,
|
|
"rewards/format_reward": 0.3888889029622078,
|
|
"step": 260
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3852.666748046875,
|
|
"epoch": 0.2236503856041131,
|
|
"grad_norm": 2.077338457107544,
|
|
"kl": 0.70703125,
|
|
"learning_rate": 1.4554267916537495e-07,
|
|
"loss": 0.1074,
|
|
"reward": -2.805555582046509,
|
|
"reward_std": 0.3134361580014229,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.1944444514811039,
|
|
"step": 261
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3562.7222900390625,
|
|
"epoch": 0.22450728363324765,
|
|
"grad_norm": 0.7478838562965393,
|
|
"kl": 0.5234375,
|
|
"learning_rate": 1.432748035231658e-07,
|
|
"loss": 0.0809,
|
|
"reward": -2.546296238899231,
|
|
"reward_std": 0.4309287667274475,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.3611111082136631,
|
|
"step": 262
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3409.388916015625,
|
|
"epoch": 0.22536418166238217,
|
|
"grad_norm": 1.5875946283340454,
|
|
"kl": 0.560546875,
|
|
"learning_rate": 1.4106199196189608e-07,
|
|
"loss": 0.0874,
|
|
"reward": -2.5,
|
|
"reward_std": 0.44037309288978577,
|
|
"rewards/confidence_reward": -2.722222328186035,
|
|
"rewards/format_reward": 0.2222222276031971,
|
|
"step": 263
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3616.8055419921875,
|
|
"epoch": 0.2262210796915167,
|
|
"grad_norm": 4.355445384979248,
|
|
"kl": 0.47265625,
|
|
"learning_rate": 1.3890454406082956e-07,
|
|
"loss": 0.1239,
|
|
"reward": -2.6111111640930176,
|
|
"reward_std": 0.3814775198698044,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3888888955116272,
|
|
"step": 264
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3070.0833740234375,
|
|
"epoch": 0.22707797772065125,
|
|
"grad_norm": 1.2011481523513794,
|
|
"kl": 0.4541015625,
|
|
"learning_rate": 1.3680275190387675e-07,
|
|
"loss": 0.1067,
|
|
"reward": -2.2592591047286987,
|
|
"reward_std": 0.46683208644390106,
|
|
"rewards/confidence_reward": -2.7592592239379883,
|
|
"rewards/format_reward": 0.5000000149011612,
|
|
"step": 265
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3636.3333740234375,
|
|
"epoch": 0.22793487574978577,
|
|
"grad_norm": 2.6138198375701904,
|
|
"kl": 0.5703125,
|
|
"learning_rate": 1.3475690004005097e-07,
|
|
"loss": 0.1152,
|
|
"reward": -2.6111111640930176,
|
|
"reward_std": 0.29541122913360596,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3888888955116272,
|
|
"step": 266
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2078.4444580078125,
|
|
"epoch": 0.22879177377892032,
|
|
"grad_norm": 1.8686110973358154,
|
|
"kl": 0.294921875,
|
|
"learning_rate": 1.3276726544494571e-07,
|
|
"loss": 0.0517,
|
|
"reward": -1.2870370149612427,
|
|
"reward_std": 1.2355431914329529,
|
|
"rewards/confidence_reward": -2.203703761100769,
|
|
"rewards/format_reward": 0.9166666567325592,
|
|
"step": 267
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3596.3055419921875,
|
|
"epoch": 0.22964867180805484,
|
|
"grad_norm": 5.496092319488525,
|
|
"kl": 0.66796875,
|
|
"learning_rate": 1.308341174832359e-07,
|
|
"loss": 0.023,
|
|
"reward": -2.75,
|
|
"reward_std": 0.2901904433965683,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.2500000074505806,
|
|
"step": 268
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2841.416748046875,
|
|
"epoch": 0.23050556983718937,
|
|
"grad_norm": 8.94375991821289,
|
|
"kl": 0.4580078125,
|
|
"learning_rate": 1.2895771787221088e-07,
|
|
"loss": 0.2275,
|
|
"reward": -1.8611111044883728,
|
|
"reward_std": 1.1322061419487,
|
|
"rewards/confidence_reward": -2.3888890743255615,
|
|
"rewards/format_reward": 0.5277777910232544,
|
|
"step": 269
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2849.9722900390625,
|
|
"epoch": 0.23136246786632392,
|
|
"grad_norm": 3.3087215423583984,
|
|
"kl": 0.439453125,
|
|
"learning_rate": 1.2713832064634125e-07,
|
|
"loss": 0.1111,
|
|
"reward": -1.6481481790542603,
|
|
"reward_std": 0.5850364714860916,
|
|
"rewards/confidence_reward": -2.203703761100769,
|
|
"rewards/format_reward": 0.5555555671453476,
|
|
"step": 270
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3841.3055419921875,
|
|
"epoch": 0.23221936589545844,
|
|
"grad_norm": 4.227456569671631,
|
|
"kl": 0.708984375,
|
|
"learning_rate": 1.2537617212288742e-07,
|
|
"loss": 0.0629,
|
|
"reward": -2.6666667461395264,
|
|
"reward_std": 0.2453947588801384,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3333333432674408,
|
|
"step": 271
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2746.083251953125,
|
|
"epoch": 0.23307626392459296,
|
|
"grad_norm": 10.011512756347656,
|
|
"kl": 0.48046875,
|
|
"learning_rate": 1.2367151086855187e-07,
|
|
"loss": 0.2351,
|
|
"reward": -2.0833332538604736,
|
|
"reward_std": 0.8133133947849274,
|
|
"rewards/confidence_reward": -2.6666667461395264,
|
|
"rewards/format_reward": 0.5833333432674408,
|
|
"step": 272
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3753.7501220703125,
|
|
"epoch": 0.23393316195372751,
|
|
"grad_norm": 2.311757802963257,
|
|
"kl": 0.7109375,
|
|
"learning_rate": 1.220245676671809e-07,
|
|
"loss": 0.0975,
|
|
"reward": -2.6111111640930176,
|
|
"reward_std": 0.41752736270427704,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3888889029622078,
|
|
"step": 273
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3697.3333740234375,
|
|
"epoch": 0.23479005998286204,
|
|
"grad_norm": 1.5095380544662476,
|
|
"kl": 0.76953125,
|
|
"learning_rate": 1.2043556548852063e-07,
|
|
"loss": 0.1038,
|
|
"reward": -2.666666626930237,
|
|
"reward_std": 0.245394766330719,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.3333333432674408,
|
|
"step": 274
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3706.2222900390625,
|
|
"epoch": 0.23564695801199656,
|
|
"grad_norm": 1.542810320854187,
|
|
"kl": 0.72265625,
|
|
"learning_rate": 1.1890471945802999e-07,
|
|
"loss": 0.1013,
|
|
"reward": -2.694444417953491,
|
|
"reward_std": 0.20412413030862808,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.30555556900799274,
|
|
"step": 275
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2976.6944580078125,
|
|
"epoch": 0.2365038560411311,
|
|
"grad_norm": 5.164428234100342,
|
|
"kl": 0.4794921875,
|
|
"learning_rate": 1.1743223682775649e-07,
|
|
"loss": 0.1522,
|
|
"reward": -1.759259283542633,
|
|
"reward_std": 0.9753341376781464,
|
|
"rewards/confidence_reward": -2.425925850868225,
|
|
"rewards/format_reward": 0.6666666567325592,
|
|
"step": 276
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3707.5555419921875,
|
|
"epoch": 0.23736075407026563,
|
|
"grad_norm": 4.900428295135498,
|
|
"kl": 0.76171875,
|
|
"learning_rate": 1.160183169482775e-07,
|
|
"loss": 0.0093,
|
|
"reward": -2.6666667461395264,
|
|
"reward_std": 0.2221490666270256,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.2777777835726738,
|
|
"step": 277
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3226.5,
|
|
"epoch": 0.23821765209940018,
|
|
"grad_norm": 2.8093297481536865,
|
|
"kl": 0.66796875,
|
|
"learning_rate": 1.1466315124171128e-07,
|
|
"loss": 0.0633,
|
|
"reward": -2.314814805984497,
|
|
"reward_std": 0.5322824418544769,
|
|
"rewards/confidence_reward": -2.8148149251937866,
|
|
"rewards/format_reward": 0.5,
|
|
"step": 278
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2714.388916015625,
|
|
"epoch": 0.2390745501285347,
|
|
"grad_norm": 12.48595142364502,
|
|
"kl": 0.61328125,
|
|
"learning_rate": 1.1336692317580158e-07,
|
|
"loss": 0.2795,
|
|
"reward": -1.5740740895271301,
|
|
"reward_std": 0.9820237159729004,
|
|
"rewards/confidence_reward": -2.2407408952713013,
|
|
"rewards/format_reward": 0.6666666716337204,
|
|
"step": 279
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2800.166748046875,
|
|
"epoch": 0.23993144815766923,
|
|
"grad_norm": 1.2126076221466064,
|
|
"kl": 0.53125,
|
|
"learning_rate": 1.1212980823907929e-07,
|
|
"loss": 0.0903,
|
|
"reward": -1.9259259104728699,
|
|
"reward_std": 0.8028348684310913,
|
|
"rewards/confidence_reward": -2.425926089286804,
|
|
"rewards/format_reward": 0.5000000149011612,
|
|
"step": 280
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3716.638916015625,
|
|
"epoch": 0.24078834618680378,
|
|
"grad_norm": 2.4237496852874756,
|
|
"kl": 0.677734375,
|
|
"learning_rate": 1.1095197391710362e-07,
|
|
"loss": 0.0919,
|
|
"reward": -2.7222222089767456,
|
|
"reward_std": 0.3419027030467987,
|
|
"rewards/confidence_reward": -2.944444417953491,
|
|
"rewards/format_reward": 0.22222222574055195,
|
|
"step": 281
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3308.6944580078125,
|
|
"epoch": 0.2416452442159383,
|
|
"grad_norm": 1.1492081880569458,
|
|
"kl": 0.5546875,
|
|
"learning_rate": 1.0983357966978745e-07,
|
|
"loss": 0.0932,
|
|
"reward": -2.2592592239379883,
|
|
"reward_std": 0.5841350704431534,
|
|
"rewards/confidence_reward": -2.759259343147278,
|
|
"rewards/format_reward": 0.5000000149011612,
|
|
"step": 282
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3500.416748046875,
|
|
"epoch": 0.24250214224507283,
|
|
"grad_norm": 2.292973756790161,
|
|
"kl": 0.67578125,
|
|
"learning_rate": 1.0877477690980931e-07,
|
|
"loss": 0.1152,
|
|
"reward": -2.4629629850387573,
|
|
"reward_std": 0.44895368814468384,
|
|
"rewards/confidence_reward": -2.9074074029922485,
|
|
"rewards/format_reward": 0.4444444552063942,
|
|
"step": 283
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3665.1666259765625,
|
|
"epoch": 0.24335904027420738,
|
|
"grad_norm": 3.3193001747131348,
|
|
"kl": 0.6337890625,
|
|
"learning_rate": 1.0777570898211405e-07,
|
|
"loss": 0.0586,
|
|
"reward": -2.5370370149612427,
|
|
"reward_std": 0.41200246661901474,
|
|
"rewards/confidence_reward": -2.7592592239379883,
|
|
"rewards/format_reward": 0.22222222574055195,
|
|
"step": 284
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3264.75,
|
|
"epoch": 0.2442159383033419,
|
|
"grad_norm": 2.722202777862549,
|
|
"kl": 0.541015625,
|
|
"learning_rate": 1.068365111445064e-07,
|
|
"loss": 0.1034,
|
|
"reward": -2.185185134410858,
|
|
"reward_std": 0.6440667659044266,
|
|
"rewards/confidence_reward": -2.7962963581085205,
|
|
"rewards/format_reward": 0.6111111044883728,
|
|
"step": 285
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3782.638916015625,
|
|
"epoch": 0.24507283633247642,
|
|
"grad_norm": 1.6381499767303467,
|
|
"kl": 0.720703125,
|
|
"learning_rate": 1.0595731054933934e-07,
|
|
"loss": 0.0854,
|
|
"reward": -2.75,
|
|
"reward_std": 0.09128709882497787,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.25,
|
|
"step": 286
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3756.0,
|
|
"epoch": 0.24592973436161097,
|
|
"grad_norm": 1.275122880935669,
|
|
"kl": 0.6171875,
|
|
"learning_rate": 1.0513822622629978e-07,
|
|
"loss": 0.0672,
|
|
"reward": -2.694444417953491,
|
|
"reward_std": 0.1773533970117569,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.305555559694767,
|
|
"step": 287
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3949.111083984375,
|
|
"epoch": 0.2467866323907455,
|
|
"grad_norm": 2.43125057220459,
|
|
"kl": 0.619140625,
|
|
"learning_rate": 1.0437936906629334e-07,
|
|
"loss": 0.0689,
|
|
"reward": -2.8888888359069824,
|
|
"reward_std": 0.15932847559452057,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.11111111380159855,
|
|
"step": 288
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2842.9722900390625,
|
|
"epoch": 0.24764353041988005,
|
|
"grad_norm": 1.0015897750854492,
|
|
"kl": 0.3935546875,
|
|
"learning_rate": 1.0368084180643224e-07,
|
|
"loss": 0.0667,
|
|
"reward": -1.8897362351417542,
|
|
"reward_std": 0.7023327946662903,
|
|
"rewards/confidence_reward": -2.500847339630127,
|
|
"rewards/format_reward": 0.6111111342906952,
|
|
"step": 289
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3495.5833740234375,
|
|
"epoch": 0.24850042844901457,
|
|
"grad_norm": 3.5078701972961426,
|
|
"kl": 0.6806640625,
|
|
"learning_rate": 1.0304273901612565e-07,
|
|
"loss": 0.0511,
|
|
"reward": -2.3055556416511536,
|
|
"reward_std": 0.5096173286437988,
|
|
"rewards/confidence_reward": -2.6111111640930176,
|
|
"rewards/format_reward": 0.305555559694767,
|
|
"step": 290
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3017.638916015625,
|
|
"epoch": 0.2493573264781491,
|
|
"grad_norm": 5.197386264801025,
|
|
"kl": 0.466796875,
|
|
"learning_rate": 1.0246514708427701e-07,
|
|
"loss": 0.1364,
|
|
"reward": -1.9907406568527222,
|
|
"reward_std": 0.7819060683250427,
|
|
"rewards/confidence_reward": -2.518518567085266,
|
|
"rewards/format_reward": 0.5277777910232544,
|
|
"step": 291
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3224.52783203125,
|
|
"epoch": 0.25021422450728364,
|
|
"grad_norm": 3.3562777042388916,
|
|
"kl": 0.552734375,
|
|
"learning_rate": 1.0194814420758804e-07,
|
|
"loss": 0.1602,
|
|
"reward": -2.0740740299224854,
|
|
"reward_std": 0.3628873825073242,
|
|
"rewards/confidence_reward": -2.5740740299224854,
|
|
"rewards/format_reward": 0.5,
|
|
"step": 292
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3234.861083984375,
|
|
"epoch": 0.25107112253641817,
|
|
"grad_norm": 3.984017848968506,
|
|
"kl": 0.572265625,
|
|
"learning_rate": 1.0149180037997228e-07,
|
|
"loss": 0.1834,
|
|
"reward": -2.114795684814453,
|
|
"reward_std": 0.7361937463283539,
|
|
"rewards/confidence_reward": -2.5870180130004883,
|
|
"rewards/format_reward": 0.472222238779068,
|
|
"step": 293
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3694.6666259765625,
|
|
"epoch": 0.2519280205655527,
|
|
"grad_norm": 1.1605571508407593,
|
|
"kl": 0.556640625,
|
|
"learning_rate": 1.0109617738307911e-07,
|
|
"loss": 0.1052,
|
|
"reward": -2.6574074029922485,
|
|
"reward_std": 0.6070278584957123,
|
|
"rewards/confidence_reward": -2.8518518209457397,
|
|
"rewards/format_reward": 0.1944444514811039,
|
|
"step": 294
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3409.02783203125,
|
|
"epoch": 0.2527849185946872,
|
|
"grad_norm": 2.5758039951324463,
|
|
"kl": 0.54296875,
|
|
"learning_rate": 1.0076132877792932e-07,
|
|
"loss": 0.1731,
|
|
"reward": -2.5277777910232544,
|
|
"reward_std": 0.3936592862010002,
|
|
"rewards/confidence_reward": -2.833333373069763,
|
|
"rewards/format_reward": 0.3055555559694767,
|
|
"step": 295
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3830.166748046875,
|
|
"epoch": 0.2536418166238218,
|
|
"grad_norm": 2.108941078186035,
|
|
"kl": 0.630859375,
|
|
"learning_rate": 1.0048729989766394e-07,
|
|
"loss": 0.0874,
|
|
"reward": -2.805555582046509,
|
|
"reward_std": 0.20412414520978928,
|
|
"rewards/confidence_reward": -3.0,
|
|
"rewards/format_reward": 0.1944444514811039,
|
|
"step": 296
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3393.3333740234375,
|
|
"epoch": 0.2544987146529563,
|
|
"grad_norm": 3.105771064758301,
|
|
"kl": 0.484375,
|
|
"learning_rate": 1.002741278414069e-07,
|
|
"loss": 0.1061,
|
|
"reward": -2.240740716457367,
|
|
"reward_std": 0.654969185590744,
|
|
"rewards/confidence_reward": -2.796296238899231,
|
|
"rewards/format_reward": 0.5555555671453476,
|
|
"step": 297
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2507.611114501953,
|
|
"epoch": 0.25535561268209084,
|
|
"grad_norm": 9.724431991577148,
|
|
"kl": 0.693359375,
|
|
"learning_rate": 1.0012184146924223e-07,
|
|
"loss": 0.0067,
|
|
"reward": -2.000847339630127,
|
|
"reward_std": 0.5755332708358765,
|
|
"rewards/confidence_reward": -2.500847339630127,
|
|
"rewards/format_reward": 0.5,
|
|
"step": 298
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 3396.9444580078125,
|
|
"epoch": 0.25621251071122536,
|
|
"grad_norm": 5.069945335388184,
|
|
"kl": 0.4638671875,
|
|
"learning_rate": 1.0003046139830701e-07,
|
|
"loss": 0.1936,
|
|
"reward": -2.2314815521240234,
|
|
"reward_std": 0.8970945179462433,
|
|
"rewards/confidence_reward": -2.759259343147278,
|
|
"rewards/format_reward": 0.5277777910232544,
|
|
"step": 299
|
|
},
|
|
{
|
|
"clip_ratio": 0.0,
|
|
"completion_length": 2547.6945190429688,
|
|
"epoch": 0.2570694087403599,
|
|
"grad_norm": 6.607841491699219,
|
|
"kl": 0.3818359375,
|
|
"learning_rate": 1e-07,
|
|
"loss": 0.1813,
|
|
"reward": -1.5749214887619019,
|
|
"reward_std": 0.6023647785186768,
|
|
"rewards/confidence_reward": -2.1860325932502747,
|
|
"rewards/format_reward": 0.6111111044883728,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.2570694087403599,
|
|
"step": 300,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.09362538470280318,
|
|
"train_runtime": 37056.8122,
|
|
"train_samples_per_second": 0.291,
|
|
"train_steps_per_second": 0.008
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 300,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 50,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 6,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|