56cb9bbd99
Model: jackf857/llama-3-8b-base-ipo-ultrafeedback-4xh200-batch-128-20260428-004616 Source: Original Platform
796 lines
27 KiB
JSON
796 lines
27 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.9989528795811519,
|
|
"eval_steps": 200,
|
|
"global_step": 477,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.0020942408376963353,
|
|
"grad_norm": 6690.55859375,
|
|
"learning_rate": 0.0,
|
|
"logits/chosen": -0.5995081663131714,
|
|
"logits/rejected": -0.6144353747367859,
|
|
"logps/chosen": -1.1853606700897217,
|
|
"logps/rejected": -1.4816904067993164,
|
|
"loss": 20000.3594,
|
|
"rewards/accuracies": 0.59375,
|
|
"rewards/chosen": 1.3230741387815215e-05,
|
|
"rewards/margins": 7.880535122239962e-06,
|
|
"rewards/rejected": 5.350205356080551e-06,
|
|
"step": 1
|
|
},
|
|
{
|
|
"epoch": 0.020942408376963352,
|
|
"grad_norm": 5719.5029296875,
|
|
"learning_rate": 9.375e-08,
|
|
"logits/chosen": -0.6324554681777954,
|
|
"logits/rejected": -0.6374175548553467,
|
|
"logps/chosen": -1.1415884494781494,
|
|
"logps/rejected": -1.3229657411575317,
|
|
"loss": 20000.1285,
|
|
"rewards/accuracies": 0.5243055820465088,
|
|
"rewards/chosen": -1.3922724519943586e-06,
|
|
"rewards/margins": -3.091482767558773e-07,
|
|
"rewards/rejected": -1.0831237204911304e-06,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.041884816753926704,
|
|
"grad_norm": 8928.5966796875,
|
|
"learning_rate": 1.9791666666666664e-07,
|
|
"logits/chosen": -0.597779393196106,
|
|
"logits/rejected": -0.6287046670913696,
|
|
"logps/chosen": -1.1651686429977417,
|
|
"logps/rejected": -1.311543583869934,
|
|
"loss": 19999.3844,
|
|
"rewards/accuracies": 0.5249999761581421,
|
|
"rewards/chosen": 1.0694350748963188e-05,
|
|
"rewards/margins": 6.846777978353202e-06,
|
|
"rewards/rejected": 3.847571861115284e-06,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.06282722513089005,
|
|
"grad_norm": 6757.13037109375,
|
|
"learning_rate": 3.020833333333333e-07,
|
|
"logits/chosen": -0.6134611368179321,
|
|
"logits/rejected": -0.605442225933075,
|
|
"logps/chosen": -1.1174745559692383,
|
|
"logps/rejected": -1.2003570795059204,
|
|
"loss": 19997.9344,
|
|
"rewards/accuracies": 0.5406249761581421,
|
|
"rewards/chosen": 6.686393317067996e-05,
|
|
"rewards/margins": 3.225356340408325e-05,
|
|
"rewards/rejected": 3.461036249063909e-05,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.08376963350785341,
|
|
"grad_norm": 8304.0927734375,
|
|
"learning_rate": 4.0625e-07,
|
|
"logits/chosen": -0.6177842617034912,
|
|
"logits/rejected": -0.6428981423377991,
|
|
"logps/chosen": -1.1158215999603271,
|
|
"logps/rejected": -1.2773981094360352,
|
|
"loss": 19990.6594,
|
|
"rewards/accuracies": 0.612500011920929,
|
|
"rewards/chosen": 0.0002113355149049312,
|
|
"rewards/margins": 8.95043122000061e-05,
|
|
"rewards/rejected": 0.0001218312099808827,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.10471204188481675,
|
|
"grad_norm": 6644.90380859375,
|
|
"learning_rate": 4.999932966293553e-07,
|
|
"logits/chosen": -0.6271435022354126,
|
|
"logits/rejected": -0.6673277020454407,
|
|
"logps/chosen": -1.047498345375061,
|
|
"logps/rejected": -1.2514413595199585,
|
|
"loss": 19978.0703,
|
|
"rewards/accuracies": 0.621874988079071,
|
|
"rewards/chosen": 0.0006389970076270401,
|
|
"rewards/margins": 0.0003597235190682113,
|
|
"rewards/rejected": 0.00027927348855882883,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.1256544502617801,
|
|
"grad_norm": 9410.4248046875,
|
|
"learning_rate": 4.991893270335525e-07,
|
|
"logits/chosen": -0.6456407308578491,
|
|
"logits/rejected": -0.6601846814155579,
|
|
"logps/chosen": -1.1118009090423584,
|
|
"logps/rejected": -1.2955642938613892,
|
|
"loss": 19958.8578,
|
|
"rewards/accuracies": 0.6625000238418579,
|
|
"rewards/chosen": 0.0007649646140635014,
|
|
"rewards/margins": 0.00041284109465777874,
|
|
"rewards/rejected": 0.0003521234611980617,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.14659685863874344,
|
|
"grad_norm": 9895.3310546875,
|
|
"learning_rate": 4.970496218214204e-07,
|
|
"logits/chosen": -0.710538923740387,
|
|
"logits/rejected": -0.7164761424064636,
|
|
"logps/chosen": -1.1343839168548584,
|
|
"logps/rejected": -1.3560715913772583,
|
|
"loss": 19939.225,
|
|
"rewards/accuracies": 0.6499999761581421,
|
|
"rewards/chosen": 0.00039382357499562204,
|
|
"rewards/margins": 0.0006254783947952092,
|
|
"rewards/rejected": -0.0002316548052476719,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.16753926701570682,
|
|
"grad_norm": 16232.4453125,
|
|
"learning_rate": 4.935856505068998e-07,
|
|
"logits/chosen": -0.6893922090530396,
|
|
"logits/rejected": -0.6864418387413025,
|
|
"logps/chosen": -1.1864311695098877,
|
|
"logps/rejected": -1.511588215827942,
|
|
"loss": 19887.7375,
|
|
"rewards/accuracies": 0.675000011920929,
|
|
"rewards/chosen": -0.0005077069508843124,
|
|
"rewards/margins": 0.0014377882471308112,
|
|
"rewards/rejected": -0.0019454952562227845,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.18848167539267016,
|
|
"grad_norm": 15383.0810546875,
|
|
"learning_rate": 4.8881598109976e-07,
|
|
"logits/chosen": -0.6936720609664917,
|
|
"logits/rejected": -0.6995416879653931,
|
|
"logps/chosen": -1.2832069396972656,
|
|
"logps/rejected": -1.6835159063339233,
|
|
"loss": 19835.5531,
|
|
"rewards/accuracies": 0.6625000238418579,
|
|
"rewards/chosen": -0.0014368193224072456,
|
|
"rewards/margins": 0.0021624970249831676,
|
|
"rewards/rejected": -0.003599316580221057,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.2094240837696335,
|
|
"grad_norm": 51433.94921875,
|
|
"learning_rate": 4.827661805750437e-07,
|
|
"logits/chosen": -0.6618175506591797,
|
|
"logits/rejected": -0.6787184476852417,
|
|
"logps/chosen": -1.3208234310150146,
|
|
"logps/rejected": -1.6875203847885132,
|
|
"loss": 19789.7328,
|
|
"rewards/accuracies": 0.668749988079071,
|
|
"rewards/chosen": -0.001908238627947867,
|
|
"rewards/margins": 0.0021552981343120337,
|
|
"rewards/rejected": -0.004063536878675222,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.23036649214659685,
|
|
"grad_norm": 39666.109375,
|
|
"learning_rate": 4.75468677825789e-07,
|
|
"logits/chosen": -0.6607747077941895,
|
|
"logits/rejected": -0.6833058595657349,
|
|
"logps/chosen": -1.3800714015960693,
|
|
"logps/rejected": -2.1090340614318848,
|
|
"loss": 19729.8531,
|
|
"rewards/accuracies": 0.690625011920929,
|
|
"rewards/chosen": -0.002765479264780879,
|
|
"rewards/margins": 0.004542567301541567,
|
|
"rewards/rejected": -0.007308047264814377,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.2513089005235602,
|
|
"grad_norm": 26585.208984375,
|
|
"learning_rate": 4.669625898336438e-07,
|
|
"logits/chosen": -0.6459494829177856,
|
|
"logits/rejected": -0.6470843553543091,
|
|
"logps/chosen": -1.5455596446990967,
|
|
"logps/rejected": -1.9538530111312866,
|
|
"loss": 19736.5594,
|
|
"rewards/accuracies": 0.6312500238418579,
|
|
"rewards/chosen": -0.004025847185403109,
|
|
"rewards/margins": 0.0025805418845266104,
|
|
"rewards/rejected": -0.006606388837099075,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.27225130890052357,
|
|
"grad_norm": 39655.12890625,
|
|
"learning_rate": 4.5729351198915705e-07,
|
|
"logits/chosen": -0.6332282423973083,
|
|
"logits/rejected": -0.6239765286445618,
|
|
"logps/chosen": -1.8061062097549438,
|
|
"logps/rejected": -2.3397936820983887,
|
|
"loss": 19656.4844,
|
|
"rewards/accuracies": 0.6187499761581421,
|
|
"rewards/chosen": -0.006146098021417856,
|
|
"rewards/margins": 0.003909106366336346,
|
|
"rewards/rejected": -0.010055203922092915,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.2931937172774869,
|
|
"grad_norm": 63126.5234375,
|
|
"learning_rate": 4.4651327368569684e-07,
|
|
"logits/chosen": -0.6292804479598999,
|
|
"logits/rejected": -0.6358648538589478,
|
|
"logps/chosen": -2.2388834953308105,
|
|
"logps/rejected": -2.7073793411254883,
|
|
"loss": 19726.4313,
|
|
"rewards/accuracies": 0.578125,
|
|
"rewards/chosen": -0.01039391104131937,
|
|
"rewards/margins": 0.003122520400211215,
|
|
"rewards/rejected": -0.013516431674361229,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.31413612565445026,
|
|
"grad_norm": 29782.576171875,
|
|
"learning_rate": 4.346796604970912e-07,
|
|
"logits/chosen": -0.6361591815948486,
|
|
"logits/rejected": -0.635583758354187,
|
|
"logps/chosen": -2.1060826778411865,
|
|
"logps/rejected": -3.041602373123169,
|
|
"loss": 19551.1266,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": -0.009226142428815365,
|
|
"rewards/margins": 0.007642732001841068,
|
|
"rewards/rejected": -0.016868876293301582,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.33507853403141363,
|
|
"grad_norm": 37246.7578125,
|
|
"learning_rate": 4.218561044282098e-07,
|
|
"logits/chosen": -0.6211342811584473,
|
|
"logits/rejected": -0.6241915822029114,
|
|
"logps/chosen": -2.1096720695495605,
|
|
"logps/rejected": -2.964322566986084,
|
|
"loss": 19569.3875,
|
|
"rewards/accuracies": 0.6468750238418579,
|
|
"rewards/chosen": -0.009453673847019672,
|
|
"rewards/margins": 0.0064040967263281345,
|
|
"rewards/rejected": -0.015857771039009094,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.35602094240837695,
|
|
"grad_norm": 46280.3671875,
|
|
"learning_rate": 4.081113438988443e-07,
|
|
"logits/chosen": -0.6069876551628113,
|
|
"logits/rejected": -0.6020215749740601,
|
|
"logps/chosen": -2.012474536895752,
|
|
"logps/rejected": -3.0016605854034424,
|
|
"loss": 19453.225,
|
|
"rewards/accuracies": 0.703125,
|
|
"rewards/chosen": -0.008933757431805134,
|
|
"rewards/margins": 0.008303998038172722,
|
|
"rewards/rejected": -0.01723775453865528,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.3769633507853403,
|
|
"grad_norm": 47927.20703125,
|
|
"learning_rate": 3.935190552834828e-07,
|
|
"logits/chosen": -0.5999588966369629,
|
|
"logits/rejected": -0.6140414476394653,
|
|
"logps/chosen": -2.4049391746520996,
|
|
"logps/rejected": -3.287382125854492,
|
|
"loss": 19432.0125,
|
|
"rewards/accuracies": 0.6156250238418579,
|
|
"rewards/chosen": -0.012067523784935474,
|
|
"rewards/margins": 0.007707494311034679,
|
|
"rewards/rejected": -0.019775018095970154,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.39790575916230364,
|
|
"grad_norm": 188406.578125,
|
|
"learning_rate": 3.781574579820464e-07,
|
|
"logits/chosen": -0.6130908727645874,
|
|
"logits/rejected": -0.6305940747261047,
|
|
"logps/chosen": -2.832430362701416,
|
|
"logps/rejected": -3.6175262928009033,
|
|
"loss": 19449.6109,
|
|
"rewards/accuracies": 0.609375,
|
|
"rewards/chosen": -0.017013628035783768,
|
|
"rewards/margins": 0.0060007283464074135,
|
|
"rewards/rejected": -0.023014355450868607,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.418848167539267,
|
|
"grad_norm": 47073.16015625,
|
|
"learning_rate": 3.621088951385353e-07,
|
|
"logits/chosen": -0.6221901178359985,
|
|
"logits/rejected": -0.6376347541809082,
|
|
"logps/chosen": -2.5921828746795654,
|
|
"logps/rejected": -3.7902023792266846,
|
|
"loss": 19275.1328,
|
|
"rewards/accuracies": 0.609375,
|
|
"rewards/chosen": -0.01452858280390501,
|
|
"rewards/margins": 0.01014014147222042,
|
|
"rewards/rejected": -0.024668725207448006,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.418848167539267,
|
|
"eval_logits/chosen": -0.6352167725563049,
|
|
"eval_logits/rejected": -0.6299898624420166,
|
|
"eval_logps/chosen": -2.4551029205322266,
|
|
"eval_logps/rejected": -3.4695351123809814,
|
|
"eval_loss": 2417.49609375,
|
|
"eval_rewards/accuracies": 0.656000018119812,
|
|
"eval_rewards/chosen": -0.013385402970016003,
|
|
"eval_rewards/margins": 0.00833566952496767,
|
|
"eval_rewards/rejected": -0.021721070632338524,
|
|
"eval_runtime": 78.7567,
|
|
"eval_samples_per_second": 25.395,
|
|
"eval_steps_per_second": 1.587,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.4397905759162304,
|
|
"grad_norm": 27402.61328125,
|
|
"learning_rate": 3.454593922550693e-07,
|
|
"logits/chosen": -0.6011684536933899,
|
|
"logits/rejected": -0.6206346750259399,
|
|
"logps/chosen": -2.257577419281006,
|
|
"logps/rejected": -3.6189427375793457,
|
|
"loss": 19377.5328,
|
|
"rewards/accuracies": 0.643750011920929,
|
|
"rewards/chosen": -0.01108148880302906,
|
|
"rewards/margins": 0.011725915595889091,
|
|
"rewards/rejected": -0.022807404398918152,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.4607329842931937,
|
|
"grad_norm": 55464.56640625,
|
|
"learning_rate": 3.2829819606729477e-07,
|
|
"logits/chosen": -0.6579941511154175,
|
|
"logits/rejected": -0.6557571291923523,
|
|
"logps/chosen": -2.2686405181884766,
|
|
"logps/rejected": -3.539849042892456,
|
|
"loss": 19303.3125,
|
|
"rewards/accuracies": 0.668749988079071,
|
|
"rewards/chosen": -0.0108075812458992,
|
|
"rewards/margins": 0.011197235435247421,
|
|
"rewards/rejected": -0.02200481668114662,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.4816753926701571,
|
|
"grad_norm": 66891.46875,
|
|
"learning_rate": 3.1071729615293424e-07,
|
|
"logits/chosen": -0.6366732716560364,
|
|
"logits/rejected": -0.6232300996780396,
|
|
"logps/chosen": -2.8289544582366943,
|
|
"logps/rejected": -3.995779037475586,
|
|
"loss": 19231.3969,
|
|
"rewards/accuracies": 0.659375011920929,
|
|
"rewards/chosen": -0.016343776136636734,
|
|
"rewards/margins": 0.01012236438691616,
|
|
"rewards/rejected": -0.026466142386198044,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.5026178010471204,
|
|
"grad_norm": 69423.734375,
|
|
"learning_rate": 2.9281093183781403e-07,
|
|
"logits/chosen": -0.6015263795852661,
|
|
"logits/rejected": -0.6166636943817139,
|
|
"logps/chosen": -3.3009753227233887,
|
|
"logps/rejected": -4.603926658630371,
|
|
"loss": 19201.2047,
|
|
"rewards/accuracies": 0.659375011920929,
|
|
"rewards/chosen": -0.021539034321904182,
|
|
"rewards/margins": 0.011443909257650375,
|
|
"rewards/rejected": -0.03298294544219971,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.5235602094240838,
|
|
"grad_norm": 64731.2421875,
|
|
"learning_rate": 2.7467508704251135e-07,
|
|
"logits/chosen": -0.6268733739852905,
|
|
"logits/rejected": -0.6166061162948608,
|
|
"logps/chosen": -2.9978890419006348,
|
|
"logps/rejected": -4.9570441246032715,
|
|
"loss": 18975.8219,
|
|
"rewards/accuracies": 0.6968749761581421,
|
|
"rewards/chosen": -0.018680796027183533,
|
|
"rewards/margins": 0.01662489026784897,
|
|
"rewards/rejected": -0.0353056825697422,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.5445026178010471,
|
|
"grad_norm": 64183.33203125,
|
|
"learning_rate": 2.5640697577740815e-07,
|
|
"logits/chosen": -0.6079710721969604,
|
|
"logits/rejected": -0.5957759618759155,
|
|
"logps/chosen": -3.0889039039611816,
|
|
"logps/rejected": -4.906301021575928,
|
|
"loss": 18967.3656,
|
|
"rewards/accuracies": 0.6625000238418579,
|
|
"rewards/chosen": -0.019036870449781418,
|
|
"rewards/margins": 0.016856301575899124,
|
|
"rewards/rejected": -0.03589317575097084,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.5654450261780105,
|
|
"grad_norm": 103451.53125,
|
|
"learning_rate": 2.381045210440644e-07,
|
|
"logits/chosen": -0.5650458335876465,
|
|
"logits/rejected": -0.5668447613716125,
|
|
"logps/chosen": -3.9116883277893066,
|
|
"logps/rejected": -5.719117164611816,
|
|
"loss": 18905.8031,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": -0.027667338028550148,
|
|
"rewards/margins": 0.016292227432131767,
|
|
"rewards/rejected": -0.043959565460681915,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.5863874345549738,
|
|
"grad_norm": 74060.140625,
|
|
"learning_rate": 2.1986582993616925e-07,
|
|
"logits/chosen": -0.562662661075592,
|
|
"logits/rejected": -0.5547593235969543,
|
|
"logps/chosen": -3.6009249687194824,
|
|
"logps/rejected": -6.734810829162598,
|
|
"loss": 18614.3531,
|
|
"rewards/accuracies": 0.7124999761581421,
|
|
"rewards/chosen": -0.02406414784491062,
|
|
"rewards/margins": 0.029486840590834618,
|
|
"rewards/rejected": -0.05355098843574524,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.6073298429319371,
|
|
"grad_norm": 197744.75,
|
|
"learning_rate": 2.0178866775369774e-07,
|
|
"logits/chosen": -0.5442657470703125,
|
|
"logits/rejected": -0.5202258825302124,
|
|
"logps/chosen": -4.191218852996826,
|
|
"logps/rejected": -5.84238862991333,
|
|
"loss": 18823.9969,
|
|
"rewards/accuracies": 0.596875011920929,
|
|
"rewards/chosen": -0.0303532425314188,
|
|
"rewards/margins": 0.014872364699840546,
|
|
"rewards/rejected": -0.045225612819194794,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.6282722513089005,
|
|
"grad_norm": 82288.1015625,
|
|
"learning_rate": 1.839699339491937e-07,
|
|
"logits/chosen": -0.5231214165687561,
|
|
"logits/rejected": -0.5048767328262329,
|
|
"logps/chosen": -4.525745868682861,
|
|
"logps/rejected": -5.8891825675964355,
|
|
"loss": 18956.8344,
|
|
"rewards/accuracies": 0.6343749761581421,
|
|
"rewards/chosen": -0.03319484740495682,
|
|
"rewards/margins": 0.013094061985611916,
|
|
"rewards/rejected": -0.046288907527923584,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.6492146596858639,
|
|
"grad_norm": 83390.5,
|
|
"learning_rate": 1.6650514271527465e-07,
|
|
"logits/chosen": -0.47909316420555115,
|
|
"logits/rejected": -0.4821901321411133,
|
|
"logps/chosen": -3.890228271484375,
|
|
"logps/rejected": -5.4383721351623535,
|
|
"loss": 18785.9547,
|
|
"rewards/accuracies": 0.671875,
|
|
"rewards/chosen": -0.027516454458236694,
|
|
"rewards/margins": 0.014197492972016335,
|
|
"rewards/rejected": -0.04171394929289818,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.6701570680628273,
|
|
"grad_norm": 119347.234375,
|
|
"learning_rate": 1.4948791099758052e-07,
|
|
"logits/chosen": -0.47245222330093384,
|
|
"logits/rejected": -0.48146170377731323,
|
|
"logps/chosen": -4.161627769470215,
|
|
"logps/rejected": -6.324838161468506,
|
|
"loss": 18573.5031,
|
|
"rewards/accuracies": 0.6781250238418579,
|
|
"rewards/chosen": -0.030029457062482834,
|
|
"rewards/margins": 0.020363491028547287,
|
|
"rewards/rejected": -0.05039294809103012,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.6910994764397905,
|
|
"grad_norm": 108725.671875,
|
|
"learning_rate": 1.3300945667758012e-07,
|
|
"logits/chosen": -0.4325336515903473,
|
|
"logits/rejected": -0.43659958243370056,
|
|
"logps/chosen": -4.2359724044799805,
|
|
"logps/rejected": -7.189882755279541,
|
|
"loss": 18710.9344,
|
|
"rewards/accuracies": 0.6781250238418579,
|
|
"rewards/chosen": -0.03000812791287899,
|
|
"rewards/margins": 0.028335105627775192,
|
|
"rewards/rejected": -0.05834323167800903,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.7120418848167539,
|
|
"grad_norm": 250133.859375,
|
|
"learning_rate": 1.1715810961514072e-07,
|
|
"logits/chosen": -0.40766844153404236,
|
|
"logits/rejected": -0.40503817796707153,
|
|
"logps/chosen": -5.0073561668396,
|
|
"logps/rejected": -7.256580352783203,
|
|
"loss": 18789.4781,
|
|
"rewards/accuracies": 0.6875,
|
|
"rewards/chosen": -0.037794455885887146,
|
|
"rewards/margins": 0.021081697195768356,
|
|
"rewards/rejected": -0.0588761568069458,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.7329842931937173,
|
|
"grad_norm": 65695.078125,
|
|
"learning_rate": 1.0201883817182949e-07,
|
|
"logits/chosen": -0.402383416891098,
|
|
"logits/rejected": -0.4145272374153137,
|
|
"logps/chosen": -4.494850158691406,
|
|
"logps/rejected": -6.888899803161621,
|
|
"loss": 18565.8484,
|
|
"rewards/accuracies": 0.6968749761581421,
|
|
"rewards/chosen": -0.03312065079808235,
|
|
"rewards/margins": 0.0219460166990757,
|
|
"rewards/rejected": -0.05506666377186775,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.7539267015706806,
|
|
"grad_norm": 182331.0625,
|
|
"learning_rate": 8.76727937529367e-08,
|
|
"logits/chosen": -0.3540952503681183,
|
|
"logits/rejected": -0.36194995045661926,
|
|
"logps/chosen": -4.524051189422607,
|
|
"logps/rejected": -7.192444801330566,
|
|
"loss": 18681.9938,
|
|
"rewards/accuracies": 0.628125011920929,
|
|
"rewards/chosen": -0.03363392874598503,
|
|
"rewards/margins": 0.024621980264782906,
|
|
"rewards/rejected": -0.058255910873413086,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.774869109947644,
|
|
"grad_norm": 99043.015625,
|
|
"learning_rate": 7.419687580962222e-08,
|
|
"logits/chosen": -0.38287925720214844,
|
|
"logits/rejected": -0.3868221640586853,
|
|
"logps/chosen": -4.889115810394287,
|
|
"logps/rejected": -7.343722343444824,
|
|
"loss": 18350.1594,
|
|
"rewards/accuracies": 0.6625000238418579,
|
|
"rewards/chosen": -0.036828476935625076,
|
|
"rewards/margins": 0.02258094772696495,
|
|
"rewards/rejected": -0.059409428387880325,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.7958115183246073,
|
|
"grad_norm": 118281.1953125,
|
|
"learning_rate": 6.166331963291519e-08,
|
|
"logits/chosen": -0.29700514674186707,
|
|
"logits/rejected": -0.3104197382926941,
|
|
"logps/chosen": -4.535180568695068,
|
|
"logps/rejected": -7.199211120605469,
|
|
"loss": 18479.7844,
|
|
"rewards/accuracies": 0.699999988079071,
|
|
"rewards/chosen": -0.033621106296777725,
|
|
"rewards/margins": 0.02514580450952053,
|
|
"rewards/rejected": -0.058766912668943405,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.8167539267015707,
|
|
"grad_norm": 116531.1015625,
|
|
"learning_rate": 5.013930914912476e-08,
|
|
"logits/chosen": -0.28081822395324707,
|
|
"logits/rejected": -0.29538464546203613,
|
|
"logps/chosen": -5.330414772033691,
|
|
"logps/rejected": -7.864521026611328,
|
|
"loss": 18660.3094,
|
|
"rewards/accuracies": 0.668749988079071,
|
|
"rewards/chosen": -0.041854970157146454,
|
|
"rewards/margins": 0.023673858493566513,
|
|
"rewards/rejected": -0.06552883237600327,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.837696335078534,
|
|
"grad_norm": 115313.1640625,
|
|
"learning_rate": 3.968661679220467e-08,
|
|
"logits/chosen": -0.2553872764110565,
|
|
"logits/rejected": -0.27801090478897095,
|
|
"logps/chosen": -5.410425662994385,
|
|
"logps/rejected": -7.3440046310424805,
|
|
"loss": 18486.2438,
|
|
"rewards/accuracies": 0.637499988079071,
|
|
"rewards/chosen": -0.04237430915236473,
|
|
"rewards/margins": 0.018501609563827515,
|
|
"rewards/rejected": -0.060875922441482544,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.837696335078534,
|
|
"eval_logits/chosen": -0.2619972229003906,
|
|
"eval_logits/rejected": -0.27771249413490295,
|
|
"eval_logps/chosen": -5.502514839172363,
|
|
"eval_logps/rejected": -8.184652328491211,
|
|
"eval_loss": 2313.8056640625,
|
|
"eval_rewards/accuracies": 0.6800000071525574,
|
|
"eval_rewards/chosen": -0.04385951906442642,
|
|
"eval_rewards/margins": 0.025012729689478874,
|
|
"eval_rewards/rejected": -0.06887225061655045,
|
|
"eval_runtime": 78.5743,
|
|
"eval_samples_per_second": 25.454,
|
|
"eval_steps_per_second": 1.591,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.8586387434554974,
|
|
"grad_norm": 116385.09375,
|
|
"learning_rate": 3.036127238347164e-08,
|
|
"logits/chosen": -0.24211814999580383,
|
|
"logits/rejected": -0.2585967183113098,
|
|
"logps/chosen": -5.910313606262207,
|
|
"logps/rejected": -8.293985366821289,
|
|
"loss": 18503.9844,
|
|
"rewards/accuracies": 0.6343749761581421,
|
|
"rewards/chosen": -0.047523993998765945,
|
|
"rewards/margins": 0.02237233892083168,
|
|
"rewards/rejected": -0.06989633291959763,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.8795811518324608,
|
|
"grad_norm": 134170.890625,
|
|
"learning_rate": 2.2213262793589482e-08,
|
|
"logits/chosen": -0.22771398723125458,
|
|
"logits/rejected": -0.2378026694059372,
|
|
"logps/chosen": -5.347130298614502,
|
|
"logps/rejected": -8.04057788848877,
|
|
"loss": 18328.1453,
|
|
"rewards/accuracies": 0.671875,
|
|
"rewards/chosen": -0.042228274047374725,
|
|
"rewards/margins": 0.025333663448691368,
|
|
"rewards/rejected": -0.06756193935871124,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.900523560209424,
|
|
"grad_norm": 140065.46875,
|
|
"learning_rate": 1.5286263996730026e-08,
|
|
"logits/chosen": -0.22656838595867157,
|
|
"logits/rejected": -0.26688051223754883,
|
|
"logps/chosen": -5.213972568511963,
|
|
"logps/rejected": -8.997479438781738,
|
|
"loss": 18304.7516,
|
|
"rewards/accuracies": 0.6781250238418579,
|
|
"rewards/chosen": -0.040835700929164886,
|
|
"rewards/margins": 0.03499193489551544,
|
|
"rewards/rejected": -0.07582763582468033,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.9214659685863874,
|
|
"grad_norm": 147417.296875,
|
|
"learning_rate": 9.617406953185136e-09,
|
|
"logits/chosen": -0.21701665222644806,
|
|
"logits/rejected": -0.22004380822181702,
|
|
"logps/chosen": -5.577446937561035,
|
|
"logps/rejected": -8.317059516906738,
|
|
"loss": 18362.8578,
|
|
"rewards/accuracies": 0.643750011920929,
|
|
"rewards/chosen": -0.04439837858080864,
|
|
"rewards/margins": 0.025850927457213402,
|
|
"rewards/rejected": -0.07024930417537689,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.9424083769633508,
|
|
"grad_norm": 187662.15625,
|
|
"learning_rate": 5.2370785753763356e-09,
|
|
"logits/chosen": -0.19742074608802795,
|
|
"logits/rejected": -0.21276791393756866,
|
|
"logps/chosen": -5.669443607330322,
|
|
"logps/rejected": -8.552831649780273,
|
|
"loss": 18458.9266,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": -0.0450492687523365,
|
|
"rewards/margins": 0.027155417948961258,
|
|
"rewards/rejected": -0.07220469415187836,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.9633507853403142,
|
|
"grad_norm": 124422.875,
|
|
"learning_rate": 2.168758844148272e-09,
|
|
"logits/chosen": -0.18504954874515533,
|
|
"logits/rejected": -0.22272753715515137,
|
|
"logps/chosen": -5.632592678070068,
|
|
"logps/rejected": -8.428214073181152,
|
|
"loss": 18551.7969,
|
|
"rewards/accuracies": 0.6781250238418579,
|
|
"rewards/chosen": -0.04460041597485542,
|
|
"rewards/margins": 0.026442039757966995,
|
|
"rewards/rejected": -0.07104245573282242,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.9842931937172775,
|
|
"grad_norm": 157096.296875,
|
|
"learning_rate": 4.288949484559934e-10,
|
|
"logits/chosen": -0.14469654858112335,
|
|
"logits/rejected": -0.15391038358211517,
|
|
"logps/chosen": -5.636763572692871,
|
|
"logps/rejected": -8.912847518920898,
|
|
"loss": 18241.4094,
|
|
"rewards/accuracies": 0.6812499761581421,
|
|
"rewards/chosen": -0.04426239803433418,
|
|
"rewards/margins": 0.031570300459861755,
|
|
"rewards/rejected": -0.07583270221948624,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.9989528795811519,
|
|
"step": 477,
|
|
"total_flos": 0.0,
|
|
"train_loss": 19137.84001572327,
|
|
"train_runtime": 6782.201,
|
|
"train_samples_per_second": 9.014,
|
|
"train_steps_per_second": 0.07
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 477,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 200,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 4,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|