c16205e044
Model: jackf857/llama-3-8b-base-kto-ultrafeedback-4xh200-batch-128-20260427-194056 Source: Original Platform
1548 lines
49 KiB
JSON
1548 lines
49 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 1.0,
|
|
"eval_steps": 200,
|
|
"global_step": 955,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.0010471204188481676,
|
|
"grad_norm": 4.605347633361816,
|
|
"kl": 0.03192205727100372,
|
|
"learning_rate": 0.0,
|
|
"logits/chosen": -333891584.0,
|
|
"logits/rejected": -265467136.0,
|
|
"logps/chosen": -199.38503196022728,
|
|
"logps/rejected": -248.56980846774192,
|
|
"loss": 2.0,
|
|
"rewards/chosen": -0.0006534994551629732,
|
|
"rewards/margins": 0.0002265003958638695,
|
|
"rewards/rejected": -0.0008799998510268427,
|
|
"step": 1
|
|
},
|
|
{
|
|
"epoch": 0.010471204188481676,
|
|
"grad_norm": 4.578050136566162,
|
|
"kl": 0.056504733860492706,
|
|
"learning_rate": 4.6875e-08,
|
|
"logits/chosen": -294903168.0,
|
|
"logits/rejected": -293959072.0,
|
|
"logps/chosen": -280.9227300995025,
|
|
"logps/rejected": -255.27308173952642,
|
|
"loss": 2.0001,
|
|
"rewards/chosen": 1.1030201542239086e-05,
|
|
"rewards/margins": -7.682456811541358e-05,
|
|
"rewards/rejected": 8.785476965765266e-05,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.020942408376963352,
|
|
"grad_norm": 4.967247009277344,
|
|
"kl": 0.05700124055147171,
|
|
"learning_rate": 9.895833333333332e-08,
|
|
"logits/chosen": -323860896.0,
|
|
"logits/rejected": -310657184.0,
|
|
"logps/chosen": -279.0876918038922,
|
|
"logps/rejected": -261.8295292075163,
|
|
"loss": 1.9999,
|
|
"rewards/chosen": 0.0003772152219703811,
|
|
"rewards/margins": 0.00016149118441195642,
|
|
"rewards/rejected": 0.0002157240375584247,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.031413612565445025,
|
|
"grad_norm": 4.6223273277282715,
|
|
"kl": 0.08535922318696976,
|
|
"learning_rate": 1.5104166666666664e-07,
|
|
"logits/chosen": -308568672.0,
|
|
"logits/rejected": -295664416.0,
|
|
"logps/chosen": -294.7120636261261,
|
|
"logps/rejected": -242.7735901872964,
|
|
"loss": 2.0,
|
|
"rewards/chosen": 0.0009878192756985042,
|
|
"rewards/margins": 9.039931823623067e-05,
|
|
"rewards/rejected": 0.0008974199574622735,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.041884816753926704,
|
|
"grad_norm": 5.402420520782471,
|
|
"kl": 0.20308740437030792,
|
|
"learning_rate": 2.03125e-07,
|
|
"logits/chosen": -302993152.0,
|
|
"logits/rejected": -312362304.0,
|
|
"logps/chosen": -306.06473214285717,
|
|
"logps/rejected": -278.4209029937792,
|
|
"loss": 1.9996,
|
|
"rewards/chosen": 0.003333506067656273,
|
|
"rewards/margins": 0.0008349405391781992,
|
|
"rewards/rejected": 0.0024985655284780737,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.05235602094240838,
|
|
"grad_norm": 5.194827556610107,
|
|
"kl": 0.34073737263679504,
|
|
"learning_rate": 2.552083333333333e-07,
|
|
"logits/chosen": -299651232.0,
|
|
"logits/rejected": -298563392.0,
|
|
"logps/chosen": -311.1816826923077,
|
|
"logps/rejected": -268.8357142857143,
|
|
"loss": 1.9988,
|
|
"rewards/chosen": 0.007691890276395358,
|
|
"rewards/margins": 0.0022930555786114188,
|
|
"rewards/rejected": 0.005398834697783939,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.06282722513089005,
|
|
"grad_norm": 5.1792426109313965,
|
|
"kl": 0.5095587372779846,
|
|
"learning_rate": 3.0729166666666665e-07,
|
|
"logits/chosen": -302699456.0,
|
|
"logits/rejected": -313209856.0,
|
|
"logps/chosen": -299.3851291403785,
|
|
"logps/rejected": -272.6368034055728,
|
|
"loss": 1.9976,
|
|
"rewards/chosen": 0.012803548518039451,
|
|
"rewards/margins": 0.004993875586639943,
|
|
"rewards/rejected": 0.007809672931399508,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.07329842931937172,
|
|
"grad_norm": 5.62812614440918,
|
|
"kl": 0.324247807264328,
|
|
"learning_rate": 3.59375e-07,
|
|
"logits/chosen": -282744192.0,
|
|
"logits/rejected": -310504768.0,
|
|
"logps/chosen": -281.65147709003213,
|
|
"logps/rejected": -274.1980433130699,
|
|
"loss": 1.9967,
|
|
"rewards/chosen": 0.017537592308314285,
|
|
"rewards/margins": 0.007583660111415985,
|
|
"rewards/rejected": 0.0099539321968983,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.08376963350785341,
|
|
"grad_norm": 5.519657135009766,
|
|
"kl": 0.19997477531433105,
|
|
"learning_rate": 4.114583333333333e-07,
|
|
"logits/chosen": -320332992.0,
|
|
"logits/rejected": -294599264.0,
|
|
"logps/chosen": -309.67646918070443,
|
|
"logps/rejected": -254.8375697767145,
|
|
"loss": 1.9917,
|
|
"rewards/chosen": 0.025304861492256293,
|
|
"rewards/margins": 0.01606653321257549,
|
|
"rewards/rejected": 0.009238328279680803,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.09424083769633508,
|
|
"grad_norm": 4.8804121017456055,
|
|
"kl": 0.039679840207099915,
|
|
"learning_rate": 4.6354166666666664e-07,
|
|
"logits/chosen": -292024928.0,
|
|
"logits/rejected": -305713184.0,
|
|
"logps/chosen": -255.28255413385827,
|
|
"logps/rejected": -255.59004360465116,
|
|
"loss": 1.9908,
|
|
"rewards/chosen": 0.0300817714901421,
|
|
"rewards/margins": 0.018853704668711096,
|
|
"rewards/rejected": 0.011228066821431005,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.10471204188481675,
|
|
"grad_norm": 5.486176490783691,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.999849525959245e-07,
|
|
"logits/chosen": -303109792.0,
|
|
"logits/rejected": -348087872.0,
|
|
"logps/chosen": -298.6987417491749,
|
|
"logps/rejected": -256.3508902077151,
|
|
"loss": 1.9828,
|
|
"rewards/chosen": 0.040776573785460825,
|
|
"rewards/margins": 0.03677532848489619,
|
|
"rewards/rejected": 0.004001245300564639,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.11518324607329843,
|
|
"grad_norm": 5.567543029785156,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.997174935782199e-07,
|
|
"logits/chosen": -289720032.0,
|
|
"logits/rejected": -312817568.0,
|
|
"logps/chosen": -288.18920101088645,
|
|
"logps/rejected": -248.39857240973313,
|
|
"loss": 1.9794,
|
|
"rewards/chosen": 0.026099390412563483,
|
|
"rewards/margins": 0.04113652131036093,
|
|
"rewards/rejected": -0.015037130897797445,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.1256544502617801,
|
|
"grad_norm": 5.6301703453063965,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.9911605954668e-07,
|
|
"logits/chosen": -322413632.0,
|
|
"logits/rejected": -313895360.0,
|
|
"logps/chosen": -272.9092261904762,
|
|
"logps/rejected": -290.2653765898251,
|
|
"loss": 1.971,
|
|
"rewards/chosen": 0.01783415798767371,
|
|
"rewards/margins": 0.058556688091881776,
|
|
"rewards/rejected": -0.040722530104208066,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.13612565445026178,
|
|
"grad_norm": 5.450737953186035,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.981814548660135e-07,
|
|
"logits/chosen": -298956864.0,
|
|
"logits/rejected": -361433408.0,
|
|
"logps/chosen": -287.07413453565505,
|
|
"logps/rejected": -262.082371676514,
|
|
"loss": 1.9574,
|
|
"rewards/chosen": 0.0099076030661613,
|
|
"rewards/margins": 0.08200543773355305,
|
|
"rewards/rejected": -0.07209783466739175,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.14659685863874344,
|
|
"grad_norm": 5.516458511352539,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.969149294871417e-07,
|
|
"logits/chosen": -338851456.0,
|
|
"logits/rejected": -332391360.0,
|
|
"logps/chosen": -274.5335463258786,
|
|
"logps/rejected": -291.4233084862385,
|
|
"loss": 1.9518,
|
|
"rewards/chosen": -0.05588988526560628,
|
|
"rewards/margins": 0.09285041070450553,
|
|
"rewards/rejected": -0.14874029597011182,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.15706806282722513,
|
|
"grad_norm": 7.548930644989014,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.953181772754997e-07,
|
|
"logits/chosen": -356664576.0,
|
|
"logits/rejected": -329555744.0,
|
|
"logps/chosen": -280.82564408396945,
|
|
"logps/rejected": -277.920425,
|
|
"loss": 1.9325,
|
|
"rewards/chosen": -0.08151665025084984,
|
|
"rewards/margins": 0.14491142592102516,
|
|
"rewards/rejected": -0.226428076171875,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.16753926701570682,
|
|
"grad_norm": 8.331445693969727,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.93393333745642e-07,
|
|
"logits/chosen": -344808288.0,
|
|
"logits/rejected": -352486720.0,
|
|
"logps/chosen": -282.0112621753247,
|
|
"logps/rejected": -285.5078125,
|
|
"loss": 1.9096,
|
|
"rewards/chosen": -0.13890903027026685,
|
|
"rewards/margins": 0.1691989662911742,
|
|
"rewards/rejected": -0.30810799656144106,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.17801047120418848,
|
|
"grad_norm": 27.522336959838867,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.9114297320518e-07,
|
|
"logits/chosen": -395774560.0,
|
|
"logits/rejected": -387744128.0,
|
|
"logps/chosen": -317.2337382445141,
|
|
"logps/rejected": -320.66834598909657,
|
|
"loss": 1.9024,
|
|
"rewards/chosen": -0.29974554175493484,
|
|
"rewards/margins": 0.20640403544826313,
|
|
"rewards/rejected": -0.506149577203198,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.18848167539267016,
|
|
"grad_norm": 14.592561721801758,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.885701053118751e-07,
|
|
"logits/chosen": -390902016.0,
|
|
"logits/rejected": -382734656.0,
|
|
"logps/chosen": -309.92156105100463,
|
|
"logps/rejected": -319.86998913902056,
|
|
"loss": 1.8962,
|
|
"rewards/chosen": -0.27517016261954214,
|
|
"rewards/margins": 0.23024947963151926,
|
|
"rewards/rejected": -0.5054196422510614,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.19895287958115182,
|
|
"grad_norm": 16.364885330200195,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.856781710484872e-07,
|
|
"logits/chosen": -377519712.0,
|
|
"logits/rejected": -384991200.0,
|
|
"logps/chosen": -317.04276315789474,
|
|
"logps/rejected": -343.468415007657,
|
|
"loss": 1.8553,
|
|
"rewards/chosen": -0.35698912892805523,
|
|
"rewards/margins": 0.303288443633679,
|
|
"rewards/rejected": -0.6602775725617342,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.2094240837696335,
|
|
"grad_norm": 13.272473335266113,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.824710381207655e-07,
|
|
"logits/chosen": -397011136.0,
|
|
"logits/rejected": -412777728.0,
|
|
"logps/chosen": -346.1264821141479,
|
|
"logps/rejected": -359.5524316109422,
|
|
"loss": 1.8447,
|
|
"rewards/chosen": -0.5472822771961666,
|
|
"rewards/margins": 0.3435558820577278,
|
|
"rewards/rejected": -0.8908381592538944,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2094240837696335,
|
|
"eval_kl": 0.0,
|
|
"eval_logits/chosen": -401673280.0,
|
|
"eval_logits/rejected": -397073248.0,
|
|
"eval_logps/chosen": -350.8658125,
|
|
"eval_logps/rejected": -366.78053125,
|
|
"eval_loss": 0.464598149061203,
|
|
"eval_rewards/chosen": -0.6301004028320313,
|
|
"eval_rewards/margins": 0.3681937866210937,
|
|
"eval_rewards/rejected": -0.998294189453125,
|
|
"eval_runtime": 173.013,
|
|
"eval_samples_per_second": 23.12,
|
|
"eval_steps_per_second": 0.722,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2198952879581152,
|
|
"grad_norm": 12.835283279418945,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.789529957847353e-07,
|
|
"logits/chosen": -377839680.0,
|
|
"logits/rejected": -360811104.0,
|
|
"logps/chosen": -342.6239265267176,
|
|
"logps/rejected": -337.57145,
|
|
"loss": 1.8619,
|
|
"rewards/chosen": -0.4235861275942271,
|
|
"rewards/margins": 0.35626909701514786,
|
|
"rewards/rejected": -0.779855224609375,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.23036649214659685,
|
|
"grad_norm": 11.761977195739746,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.751287491101977e-07,
|
|
"logits/chosen": -363950592.0,
|
|
"logits/rejected": -346935360.0,
|
|
"logps/chosen": -327.6511063664596,
|
|
"logps/rejected": -331.7046481918239,
|
|
"loss": 1.8678,
|
|
"rewards/chosen": -0.45187747552528146,
|
|
"rewards/margins": 0.33270157958908075,
|
|
"rewards/rejected": -0.7845790551143622,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.24083769633507854,
|
|
"grad_norm": 14.949240684509277,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.710034126881159e-07,
|
|
"logits/chosen": -387794592.0,
|
|
"logits/rejected": -332998048.0,
|
|
"logps/chosen": -345.15252001455605,
|
|
"logps/rejected": -363.9196089797639,
|
|
"loss": 1.8654,
|
|
"rewards/chosen": -0.42814500744859896,
|
|
"rewards/margins": 0.43583469882511516,
|
|
"rewards/rejected": -0.8639797062737141,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.2513089005235602,
|
|
"grad_norm": 28.15343475341797,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.665825037903035e-07,
|
|
"logits/chosen": -384477856.0,
|
|
"logits/rejected": -373712864.0,
|
|
"logps/chosen": -335.26793624807397,
|
|
"logps/rejected": -360.7674574088748,
|
|
"loss": 1.8256,
|
|
"rewards/chosen": -0.5404140196155143,
|
|
"rewards/margins": 0.46967980234986806,
|
|
"rewards/rejected": -1.0100938219653823,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.2617801047120419,
|
|
"grad_norm": 13.149576187133789,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.618719349905619e-07,
|
|
"logits/chosen": -401351104.0,
|
|
"logits/rejected": -374455584.0,
|
|
"logps/chosen": -363.6633110687023,
|
|
"logps/rejected": -375.2053,
|
|
"loss": 1.8287,
|
|
"rewards/chosen": -0.6616745810472329,
|
|
"rewards/margins": 0.5001745400465172,
|
|
"rewards/rejected": -1.16184912109375,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.27225130890052357,
|
|
"grad_norm": 16.712739944458008,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.568780062571374e-07,
|
|
"logits/chosen": -386476864.0,
|
|
"logits/rejected": -400174592.0,
|
|
"logps/chosen": -339.71411758814105,
|
|
"logps/rejected": -382.5932736280488,
|
|
"loss": 1.7953,
|
|
"rewards/chosen": -0.6005113063714443,
|
|
"rewards/margins": 0.5215838481814806,
|
|
"rewards/rejected": -1.122095154552925,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.28272251308900526,
|
|
"grad_norm": 22.0365047454834,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.516073965270717e-07,
|
|
"logits/chosen": -384655008.0,
|
|
"logits/rejected": -365928352.0,
|
|
"logps/chosen": -338.5719385758998,
|
|
"logps/rejected": -392.81059867394697,
|
|
"loss": 1.8065,
|
|
"rewards/chosen": -0.6256347083149941,
|
|
"rewards/margins": 0.5414030889195515,
|
|
"rewards/rejected": -1.1670377972345456,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.2931937172774869,
|
|
"grad_norm": 43.19585037231445,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.460671547737158e-07,
|
|
"logits/chosen": -361381056.0,
|
|
"logits/rejected": -369068928.0,
|
|
"logps/chosen": -408.50211012861735,
|
|
"logps/rejected": -410.9393047112462,
|
|
"loss": 1.7984,
|
|
"rewards/chosen": -1.0311282240692825,
|
|
"rewards/margins": 0.5366287766074842,
|
|
"rewards/rejected": -1.5677570006767667,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.3036649214659686,
|
|
"grad_norm": 16.71083641052246,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.40264690579353e-07,
|
|
"logits/chosen": -398537024.0,
|
|
"logits/rejected": -368430528.0,
|
|
"logps/chosen": -387.8846227134146,
|
|
"logps/rejected": -399.3923527644231,
|
|
"loss": 1.8419,
|
|
"rewards/chosen": -0.9159838048423209,
|
|
"rewards/margins": 0.5333353988523405,
|
|
"rewards/rejected": -1.4493192036946614,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.31413612565445026,
|
|
"grad_norm": 13.06988525390625,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.3420776422553916e-07,
|
|
"logits/chosen": -379714016.0,
|
|
"logits/rejected": -362430816.0,
|
|
"logps/chosen": -351.87913321865443,
|
|
"logps/rejected": -380.3795926517572,
|
|
"loss": 1.8073,
|
|
"rewards/chosen": -0.6257981233275994,
|
|
"rewards/margins": 0.5890270041247269,
|
|
"rewards/rejected": -1.2148251274523263,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.32460732984293195,
|
|
"grad_norm": 23.772066116333008,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.279044763144141e-07,
|
|
"logits/chosen": -356793760.0,
|
|
"logits/rejected": -387742400.0,
|
|
"logps/chosen": -313.406973841853,
|
|
"logps/rejected": -383.028311353211,
|
|
"loss": 1.7819,
|
|
"rewards/chosen": -0.4327624141217801,
|
|
"rewards/margins": 0.5333329298850529,
|
|
"rewards/rejected": -0.966095344006833,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.33507853403141363,
|
|
"grad_norm": 16.960844039916992,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.213632569348639e-07,
|
|
"logits/chosen": -431567776.0,
|
|
"logits/rejected": -367267008.0,
|
|
"logps/chosen": -342.6104959736457,
|
|
"logps/rejected": -379.4291771356784,
|
|
"loss": 1.8291,
|
|
"rewards/chosen": -0.5189258134207998,
|
|
"rewards/margins": 0.6056777136546817,
|
|
"rewards/rejected": -1.1246035270754815,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.34554973821989526,
|
|
"grad_norm": 38.159210205078125,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.145928543880249e-07,
|
|
"logits/chosen": -397418144.0,
|
|
"logits/rejected": -396005696.0,
|
|
"logps/chosen": -347.52253653238546,
|
|
"logps/rejected": -389.09370170015455,
|
|
"loss": 1.7527,
|
|
"rewards/chosen": -0.5500312793123026,
|
|
"rewards/margins": 0.6948817658722029,
|
|
"rewards/rejected": -1.2449130451845054,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.35602094240837695,
|
|
"grad_norm": 17.87345314025879,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.076023234872057e-07,
|
|
"logits/chosen": -360866112.0,
|
|
"logits/rejected": -396226624.0,
|
|
"logps/chosen": -372.8658622778675,
|
|
"logps/rejected": -422.6806448562784,
|
|
"loss": 1.7247,
|
|
"rewards/chosen": -0.8265112659657714,
|
|
"rewards/margins": 0.7862990622537822,
|
|
"rewards/rejected": -1.6128103282195536,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.36649214659685864,
|
|
"grad_norm": 32.15557861328125,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.004010134478771e-07,
|
|
"logits/chosen": -402362112.0,
|
|
"logits/rejected": -383912448.0,
|
|
"logps/chosen": -347.5367717978395,
|
|
"logps/rejected": -395.23214992088606,
|
|
"loss": 1.7853,
|
|
"rewards/chosen": -0.6819350160198447,
|
|
"rewards/margins": 0.6560503486768028,
|
|
"rewards/rejected": -1.3379853646966475,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.3769633507853403,
|
|
"grad_norm": 17.032840728759766,
|
|
"kl": 0.0,
|
|
"learning_rate": 3.9299855538392534e-07,
|
|
"logits/chosen": -373061568.0,
|
|
"logits/rejected": -376975744.0,
|
|
"logps/chosen": -340.9822198275862,
|
|
"logps/rejected": -385.4313181464174,
|
|
"loss": 1.7507,
|
|
"rewards/chosen": -0.4902129457288401,
|
|
"rewards/margins": 0.722005627060052,
|
|
"rewards/rejected": -1.2122185727888921,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.387434554973822,
|
|
"grad_norm": 22.606733322143555,
|
|
"kl": 0.0,
|
|
"learning_rate": 3.8540484942689075e-07,
|
|
"logits/chosen": -371107936.0,
|
|
"logits/rejected": -383718688.0,
|
|
"logps/chosen": -353.1767515923567,
|
|
"logps/rejected": -418.5740030674847,
|
|
"loss": 1.7464,
|
|
"rewards/chosen": -0.7054660578442228,
|
|
"rewards/margins": 0.7065043030345636,
|
|
"rewards/rejected": -1.4119703608787864,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.39790575916230364,
|
|
"grad_norm": 33.263973236083984,
|
|
"kl": 0.0,
|
|
"learning_rate": 3.77630051485419e-07,
|
|
"logits/chosen": -406904672.0,
|
|
"logits/rejected": -344286496.0,
|
|
"logps/chosen": -403.9385601032448,
|
|
"logps/rejected": -432.968853820598,
|
|
"loss": 1.8567,
|
|
"rewards/chosen": -1.0629785588357301,
|
|
"rewards/margins": 0.6323324203547287,
|
|
"rewards/rejected": -1.6953109791904588,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.4083769633507853,
|
|
"grad_norm": 18.568082809448242,
|
|
"kl": 0.0,
|
|
"learning_rate": 3.696845596626342e-07,
|
|
"logits/chosen": -359421728.0,
|
|
"logits/rejected": -367630624.0,
|
|
"logps/chosen": -348.77261904761906,
|
|
"logps/rejected": -418.41769230769233,
|
|
"loss": 1.7356,
|
|
"rewards/chosen": -0.7873753138950893,
|
|
"rewards/margins": 0.7420602517599587,
|
|
"rewards/rejected": -1.529435565655048,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.418848167539267,
|
|
"grad_norm": 23.56498146057129,
|
|
"kl": 0.0,
|
|
"learning_rate": 3.61579000349597e-07,
|
|
"logits/chosen": -379061824.0,
|
|
"logits/rejected": -363287360.0,
|
|
"logps/chosen": -362.0563360091743,
|
|
"logps/rejected": -416.0301767172524,
|
|
"loss": 1.7296,
|
|
"rewards/chosen": -0.6648301990754014,
|
|
"rewards/margins": 0.84632178197367,
|
|
"rewards/rejected": -1.5111519810490714,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.418848167539267,
|
|
"eval_kl": 0.0,
|
|
"eval_logits/chosen": -377831392.0,
|
|
"eval_logits/rejected": -377408832.0,
|
|
"eval_logps/chosen": -356.89978125,
|
|
"eval_logps/rejected": -406.783625,
|
|
"eval_loss": 0.44080978631973267,
|
|
"eval_rewards/chosen": -0.690440185546875,
|
|
"eval_rewards/margins": 0.7078846435546874,
|
|
"eval_rewards/rejected": -1.3983248291015624,
|
|
"eval_runtime": 172.5358,
|
|
"eval_samples_per_second": 23.184,
|
|
"eval_steps_per_second": 0.724,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.4293193717277487,
|
|
"grad_norm": 25.634418487548828,
|
|
"kl": 0.0,
|
|
"learning_rate": 3.5332421401344837e-07,
|
|
"logits/chosen": -325346176.0,
|
|
"logits/rejected": -386428736.0,
|
|
"logps/chosen": -357.732086489899,
|
|
"logps/rejected": -401.4339467930029,
|
|
"loss": 1.6995,
|
|
"rewards/chosen": -0.6876018793895992,
|
|
"rewards/margins": 0.7492819858333873,
|
|
"rewards/rejected": -1.4368838652229865,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.4397905759162304,
|
|
"grad_norm": 34.874794006347656,
|
|
"kl": 0.0,
|
|
"learning_rate": 3.4493124069924635e-07,
|
|
"logits/chosen": -378771648.0,
|
|
"logits/rejected": -384461664.0,
|
|
"logps/chosen": -364.9864,
|
|
"logps/rejected": -393.6061545801527,
|
|
"loss": 1.7407,
|
|
"rewards/chosen": -0.69082021484375,
|
|
"rewards/margins": 0.7534859971374045,
|
|
"rewards/rejected": -1.4443062119811545,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.450261780104712,
|
|
"grad_norm": 35.7071533203125,
|
|
"kl": 0.0,
|
|
"learning_rate": 3.3641130526488335e-07,
|
|
"logits/chosen": -346615360.0,
|
|
"logits/rejected": -370164512.0,
|
|
"logps/chosen": -328.9187352825746,
|
|
"logps/rejected": -424.24766718507,
|
|
"loss": 1.7463,
|
|
"rewards/chosen": -0.6537484209428964,
|
|
"rewards/margins": 0.7605128467354668,
|
|
"rewards/rejected": -1.4142612676783632,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.4607329842931937,
|
|
"grad_norm": 34.8936653137207,
|
|
"kl": 0.0,
|
|
"learning_rate": 3.2777580236883473e-07,
|
|
"logits/chosen": -361869248.0,
|
|
"logits/rejected": -375545024.0,
|
|
"logps/chosen": -328.751697284345,
|
|
"logps/rejected": -397.5517010703364,
|
|
"loss": 1.7409,
|
|
"rewards/chosen": -0.610927008973143,
|
|
"rewards/margins": 0.7168888955566736,
|
|
"rewards/rejected": -1.3278159045298166,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.4712041884816754,
|
|
"grad_norm": 26.618633270263672,
|
|
"kl": 0.0,
|
|
"learning_rate": 3.1903628123081196e-07,
|
|
"logits/chosen": -384088768.0,
|
|
"logits/rejected": -362557504.0,
|
|
"logps/chosen": -352.25322690217394,
|
|
"logps/rejected": -407.9415290880503,
|
|
"loss": 1.7293,
|
|
"rewards/chosen": -0.6755538845654601,
|
|
"rewards/margins": 0.846232595264532,
|
|
"rewards/rejected": -1.521786479829992,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.4816753926701571,
|
|
"grad_norm": 19.443235397338867,
|
|
"kl": 0.0,
|
|
"learning_rate": 3.1020443018570556e-07,
|
|
"logits/chosen": -358506400.0,
|
|
"logits/rejected": -400381632.0,
|
|
"logps/chosen": -348.9179941152597,
|
|
"logps/rejected": -395.03962725903614,
|
|
"loss": 1.7259,
|
|
"rewards/chosen": -0.6845747216955408,
|
|
"rewards/margins": 0.7222973956802122,
|
|
"rewards/rejected": -1.406872117375753,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.49214659685863876,
|
|
"grad_norm": 36.631107330322266,
|
|
"kl": 0.0,
|
|
"learning_rate": 3.0129206105147343e-07,
|
|
"logits/chosen": -353789856.0,
|
|
"logits/rejected": -394140160.0,
|
|
"logps/chosen": -369.1834216965742,
|
|
"logps/rejected": -395.943871814093,
|
|
"loss": 1.7266,
|
|
"rewards/chosen": -0.7301152837607056,
|
|
"rewards/margins": 0.7226990500391913,
|
|
"rewards/rejected": -1.4528143337998969,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.5026178010471204,
|
|
"grad_norm": 21.622982025146484,
|
|
"kl": 0.0,
|
|
"learning_rate": 2.923110933318805e-07,
|
|
"logits/chosen": -380953024.0,
|
|
"logits/rejected": -351669664.0,
|
|
"logps/chosen": -346.1928404850746,
|
|
"logps/rejected": -385.3263575819672,
|
|
"loss": 1.7825,
|
|
"rewards/chosen": -0.6672405185984142,
|
|
"rewards/margins": 0.7526233229743317,
|
|
"rewards/rejected": -1.4198638415727458,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.5130890052356021,
|
|
"grad_norm": 27.391277313232422,
|
|
"kl": 0.0,
|
|
"learning_rate": 2.832735382752194e-07,
|
|
"logits/chosen": -384934912.0,
|
|
"logits/rejected": -371643968.0,
|
|
"logps/chosen": -372.62961810872895,
|
|
"logps/rejected": -431.1251993620415,
|
|
"loss": 1.7894,
|
|
"rewards/chosen": -0.93115365231384,
|
|
"rewards/margins": 0.7237101584915826,
|
|
"rewards/rejected": -1.6548638108054226,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.5235602094240838,
|
|
"grad_norm": 30.544750213623047,
|
|
"kl": 0.0,
|
|
"learning_rate": 2.741914828103307e-07,
|
|
"logits/chosen": -364308672.0,
|
|
"logits/rejected": -375439488.0,
|
|
"logps/chosen": -370.71887264521195,
|
|
"logps/rejected": -424.57008164852255,
|
|
"loss": 1.7381,
|
|
"rewards/chosen": -0.924701876318436,
|
|
"rewards/margins": 0.7957396423197833,
|
|
"rewards/rejected": -1.7204415186382194,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.5340314136125655,
|
|
"grad_norm": 24.350994110107422,
|
|
"kl": 0.0,
|
|
"learning_rate": 2.650770733814065e-07,
|
|
"logits/chosen": -367684672.0,
|
|
"logits/rejected": -364714048.0,
|
|
"logps/chosen": -355.05851275917064,
|
|
"logps/rejected": -403.8284360643185,
|
|
"loss": 1.7188,
|
|
"rewards/chosen": -0.6844002512273226,
|
|
"rewards/margins": 0.8209749551518888,
|
|
"rewards/rejected": -1.5053752063792114,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.5445026178010471,
|
|
"grad_norm": 28.53436279296875,
|
|
"kl": 0.0,
|
|
"learning_rate": 2.55942499703198e-07,
|
|
"logits/chosen": -379056736.0,
|
|
"logits/rejected": -379016544.0,
|
|
"logps/chosen": -345.9064,
|
|
"logps/rejected": -384.8177719465649,
|
|
"loss": 1.7248,
|
|
"rewards/chosen": -0.563288232421875,
|
|
"rewards/margins": 0.7492281678643845,
|
|
"rewards/rejected": -1.3125164002862595,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.5549738219895288,
|
|
"grad_norm": 12.310104370117188,
|
|
"kl": 0.0,
|
|
"learning_rate": 2.467999784583527e-07,
|
|
"logits/chosen": -348551552.0,
|
|
"logits/rejected": -371971776.0,
|
|
"logps/chosen": -327.1228284744409,
|
|
"logps/rejected": -392.4502102446483,
|
|
"loss": 1.7112,
|
|
"rewards/chosen": -0.5498965327351238,
|
|
"rewards/margins": 0.8324248862370666,
|
|
"rewards/rejected": -1.3823214189721904,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.5654450261780105,
|
|
"grad_norm": 26.28302574157715,
|
|
"kl": 0.0,
|
|
"learning_rate": 2.3766173695868388e-07,
|
|
"logits/chosen": -378826880.0,
|
|
"logits/rejected": -363562816.0,
|
|
"logps/chosen": -364.27648832312406,
|
|
"logps/rejected": -418.0172448165869,
|
|
"loss": 1.7646,
|
|
"rewards/chosen": -0.7452207042466261,
|
|
"rewards/margins": 0.772901577888043,
|
|
"rewards/rejected": -1.5181222821346692,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.5759162303664922,
|
|
"grad_norm": 17.37626075744629,
|
|
"kl": 0.0,
|
|
"learning_rate": 2.285399967922253e-07,
|
|
"logits/chosen": -378287168.0,
|
|
"logits/rejected": -397669600.0,
|
|
"logps/chosen": -360.6157647763578,
|
|
"logps/rejected": -439.8442756116208,
|
|
"loss": 1.6957,
|
|
"rewards/chosen": -0.9246350370656949,
|
|
"rewards/margins": 0.944558783712688,
|
|
"rewards/rejected": -1.869193820778383,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.5863874345549738,
|
|
"grad_norm": 24.048419952392578,
|
|
"kl": 0.0,
|
|
"learning_rate": 2.194469574779397e-07,
|
|
"logits/chosen": -419567904.0,
|
|
"logits/rejected": -379702528.0,
|
|
"logps/chosen": -370.91926688163886,
|
|
"logps/rejected": -425.7548309178744,
|
|
"loss": 1.7624,
|
|
"rewards/chosen": -0.8229443285280729,
|
|
"rewards/margins": 0.8691681994635736,
|
|
"rewards/rejected": -1.6921125279916465,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.5968586387434555,
|
|
"grad_norm": 14.945625305175781,
|
|
"kl": 0.0,
|
|
"learning_rate": 2.1039478014994441e-07,
|
|
"logits/chosen": -369516832.0,
|
|
"logits/rejected": -357956992.0,
|
|
"logps/chosen": -322.2456745723173,
|
|
"logps/rejected": -398.4457908163265,
|
|
"loss": 1.7312,
|
|
"rewards/chosen": -0.5156455507174621,
|
|
"rewards/margins": 0.8550450031483149,
|
|
"rewards/rejected": -1.370690553865777,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.6073298429319371,
|
|
"grad_norm": 13.910249710083008,
|
|
"kl": 0.0,
|
|
"learning_rate": 2.0139557129307149e-07,
|
|
"logits/chosen": -369174880.0,
|
|
"logits/rejected": -375438400.0,
|
|
"logps/chosen": -355.16543093152865,
|
|
"logps/rejected": -419.0519555214724,
|
|
"loss": 1.7166,
|
|
"rewards/chosen": -0.5668097817973726,
|
|
"rewards/margins": 0.8209082784679168,
|
|
"rewards/rejected": -1.3877180602652894,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.6178010471204188,
|
|
"grad_norm": 31.937427520751953,
|
|
"kl": 0.0,
|
|
"learning_rate": 1.9246136655151808e-07,
|
|
"logits/chosen": -388834208.0,
|
|
"logits/rejected": -366008416.0,
|
|
"logps/chosen": -362.90542635658915,
|
|
"logps/rejected": -438.9683070866142,
|
|
"loss": 1.7186,
|
|
"rewards/chosen": -0.7052005738250969,
|
|
"rewards/margins": 0.9188863017900605,
|
|
"rewards/rejected": -1.6240868756151574,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.6282722513089005,
|
|
"grad_norm": 51.06322479248047,
|
|
"kl": 0.0,
|
|
"learning_rate": 1.8360411463223873e-07,
|
|
"logits/chosen": -373022624.0,
|
|
"logits/rejected": -388438080.0,
|
|
"logps/chosen": -361.36163553259144,
|
|
"logps/rejected": -437.6943644393241,
|
|
"loss": 1.685,
|
|
"rewards/chosen": -0.7795385412267737,
|
|
"rewards/margins": 0.9562707380671012,
|
|
"rewards/rejected": -1.7358092792938749,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.6282722513089005,
|
|
"eval_kl": 0.0,
|
|
"eval_logits/chosen": -388254240.0,
|
|
"eval_logits/rejected": -387494368.0,
|
|
"eval_logps/chosen": -383.711,
|
|
"eval_logps/rejected": -454.13559375,
|
|
"eval_loss": 0.43252766132354736,
|
|
"eval_rewards/chosen": -0.9585521240234375,
|
|
"eval_rewards/margins": 0.9132923583984375,
|
|
"eval_rewards/rejected": -1.871844482421875,
|
|
"eval_runtime": 172.7536,
|
|
"eval_samples_per_second": 23.154,
|
|
"eval_steps_per_second": 0.724,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.6387434554973822,
|
|
"grad_norm": 17.530250549316406,
|
|
"kl": 0.0,
|
|
"learning_rate": 1.7483566132460865e-07,
|
|
"logits/chosen": -372182848.0,
|
|
"logits/rejected": -386128032.0,
|
|
"logps/chosen": -404.1522943037975,
|
|
"logps/rejected": -436.9659047067901,
|
|
"loss": 1.7515,
|
|
"rewards/chosen": -1.0911100363429589,
|
|
"rewards/margins": 0.7743034881434416,
|
|
"rewards/rejected": -1.8654135244864005,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.6492146596858639,
|
|
"grad_norm": 71.81634521484375,
|
|
"kl": 0.0,
|
|
"learning_rate": 1.66167733657731e-07,
|
|
"logits/chosen": -379878784.0,
|
|
"logits/rejected": -385352000.0,
|
|
"logps/chosen": -417.7991178229665,
|
|
"logps/rejected": -460.5967649310873,
|
|
"loss": 1.7452,
|
|
"rewards/chosen": -1.1937847715435606,
|
|
"rewards/margins": 0.7742432750921593,
|
|
"rewards/rejected": -1.9680280466357198,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.6596858638743456,
|
|
"grad_norm": 35.158390045166016,
|
|
"kl": 0.0,
|
|
"learning_rate": 1.5761192421657456e-07,
|
|
"logits/chosen": -363958816.0,
|
|
"logits/rejected": -387630624.0,
|
|
"logps/chosen": -395.60520666932905,
|
|
"logps/rejected": -463.6549120795107,
|
|
"loss": 1.693,
|
|
"rewards/chosen": -1.0146022223816893,
|
|
"rewards/margins": 0.9162212408016057,
|
|
"rewards/rejected": -1.930823463183295,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.6701570680628273,
|
|
"grad_norm": 73.86211395263672,
|
|
"kl": 0.0,
|
|
"learning_rate": 1.491796756379185e-07,
|
|
"logits/chosen": -397256448.0,
|
|
"logits/rejected": -357917472.0,
|
|
"logps/chosen": -384.6437266791045,
|
|
"logps/rejected": -425.7797643442623,
|
|
"loss": 1.7584,
|
|
"rewards/chosen": -0.7558601493266092,
|
|
"rewards/margins": 0.921012729451567,
|
|
"rewards/rejected": -1.6768728787781761,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.680628272251309,
|
|
"grad_norm": 19.859880447387695,
|
|
"kl": 0.0,
|
|
"learning_rate": 1.4088226530684071e-07,
|
|
"logits/chosen": -384827904.0,
|
|
"logits/rejected": -371061408.0,
|
|
"logps/chosen": -354.42405913978496,
|
|
"logps/rejected": -410.13796701112875,
|
|
"loss": 1.7188,
|
|
"rewards/chosen": -0.5760806504116264,
|
|
"rewards/margins": 0.9016887737114857,
|
|
"rewards/rejected": -1.477769424123112,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.6910994764397905,
|
|
"grad_norm": 36.21221923828125,
|
|
"kl": 0.0,
|
|
"learning_rate": 1.327307902742142e-07,
|
|
"logits/chosen": -416035360.0,
|
|
"logits/rejected": -385663392.0,
|
|
"logps/chosen": -344.91139423076925,
|
|
"logps/rejected": -437.5813492063492,
|
|
"loss": 1.686,
|
|
"rewards/chosen": -0.6315081317608173,
|
|
"rewards/margins": 1.0616693003448374,
|
|
"rewards/rejected": -1.6931774321056547,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.7015706806282722,
|
|
"grad_norm": 44.68547821044922,
|
|
"kl": 0.0,
|
|
"learning_rate": 1.2473615241538523e-07,
|
|
"logits/chosen": -376409280.0,
|
|
"logits/rejected": -334625152.0,
|
|
"logps/chosen": -340.43985190014905,
|
|
"logps/rejected": -424.7399938423645,
|
|
"loss": 1.776,
|
|
"rewards/chosen": -0.6765481917584528,
|
|
"rewards/margins": 0.8097976725234128,
|
|
"rewards/rejected": -1.4863458642818657,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.7120418848167539,
|
|
"grad_norm": 29.446016311645508,
|
|
"kl": 0.0,
|
|
"learning_rate": 1.169090438498816e-07,
|
|
"logits/chosen": -381945856.0,
|
|
"logits/rejected": -390224416.0,
|
|
"logps/chosen": -359.84859154929575,
|
|
"logps/rejected": -424.1903276131045,
|
|
"loss": 1.6951,
|
|
"rewards/chosen": -0.6581172555078736,
|
|
"rewards/margins": 0.9331115450113348,
|
|
"rewards/rejected": -1.5912288005192083,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.7225130890052356,
|
|
"grad_norm": 30.748411178588867,
|
|
"kl": 0.0,
|
|
"learning_rate": 1.0925993264165045e-07,
|
|
"logits/chosen": -370050240.0,
|
|
"logits/rejected": -376552000.0,
|
|
"logps/chosen": -363.6959115415335,
|
|
"logps/rejected": -440.83008409785936,
|
|
"loss": 1.6934,
|
|
"rewards/chosen": -0.7725032526083266,
|
|
"rewards/margins": 0.9274107755477323,
|
|
"rewards/rejected": -1.699914028156059,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 0.7329842931937173,
|
|
"grad_norm": 29.660114288330078,
|
|
"kl": 0.0,
|
|
"learning_rate": 1.0179904879894998e-07,
|
|
"logits/chosen": -384338432.0,
|
|
"logits/rejected": -357984064.0,
|
|
"logps/chosen": -360.97984423981194,
|
|
"logps/rejected": -440.06215926791276,
|
|
"loss": 1.7067,
|
|
"rewards/chosen": -0.7834205747024393,
|
|
"rewards/margins": 0.9409068216784992,
|
|
"rewards/rejected": -1.7243273963809385,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.743455497382199,
|
|
"grad_norm": 42.167049407958984,
|
|
"kl": 0.0,
|
|
"learning_rate": 9.453637059262117e-08,
|
|
"logits/chosen": -355324928.0,
|
|
"logits/rejected": -358938272.0,
|
|
"logps/chosen": -350.62712309160304,
|
|
"logps/rejected": -409.406275,
|
|
"loss": 1.7458,
|
|
"rewards/chosen": -0.7520553472387882,
|
|
"rewards/margins": 0.8503729730737118,
|
|
"rewards/rejected": -1.6024283203125,
|
|
"step": 710
|
|
},
|
|
{
|
|
"epoch": 0.7539267015706806,
|
|
"grad_norm": 76.56432342529297,
|
|
"kl": 0.0,
|
|
"learning_rate": 8.748161121103406e-08,
|
|
"logits/chosen": -380407232.0,
|
|
"logits/rejected": -380544576.0,
|
|
"logps/chosen": -358.77977362204723,
|
|
"logps/rejected": -443.77843992248063,
|
|
"loss": 1.683,
|
|
"rewards/chosen": -0.6690234086644931,
|
|
"rewards/margins": 0.979472193019131,
|
|
"rewards/rejected": -1.648495601683624,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.7643979057591623,
|
|
"grad_norm": 20.125774383544922,
|
|
"kl": 0.0,
|
|
"learning_rate": 8.064420576955965e-08,
|
|
"logits/chosen": -379323904.0,
|
|
"logits/rejected": -390768192.0,
|
|
"logps/chosen": -373.331298828125,
|
|
"logps/rejected": -438.97998046875,
|
|
"loss": 1.7388,
|
|
"rewards/chosen": -0.8371871948242188,
|
|
"rewards/margins": 0.8446130752563475,
|
|
"rewards/rejected": -1.6818002700805663,
|
|
"step": 730
|
|
},
|
|
{
|
|
"epoch": 0.774869109947644,
|
|
"grad_norm": 66.10313415527344,
|
|
"kl": 0.0,
|
|
"learning_rate": 7.403329869193922e-08,
|
|
"logits/chosen": -371630016.0,
|
|
"logits/rejected": -358382208.0,
|
|
"logps/chosen": -354.93711597542244,
|
|
"logps/rejected": -428.1827305246423,
|
|
"loss": 1.6854,
|
|
"rewards/chosen": -0.7562084285894297,
|
|
"rewards/margins": 1.0796500847377761,
|
|
"rewards/rejected": -1.8358585133272058,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.7853403141361257,
|
|
"grad_norm": 57.931419372558594,
|
|
"kl": 0.0,
|
|
"learning_rate": 6.765773148042858e-08,
|
|
"logits/chosen": -389279648.0,
|
|
"logits/rejected": -367319104.0,
|
|
"logps/chosen": -360.12411194316434,
|
|
"logps/rejected": -412.91921701112875,
|
|
"loss": 1.7553,
|
|
"rewards/chosen": -0.7227982434625816,
|
|
"rewards/margins": 0.8275232536434399,
|
|
"rewards/rejected": -1.5503214971060215,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 0.7958115183246073,
|
|
"grad_norm": 35.246177673339844,
|
|
"kl": 0.0,
|
|
"learning_rate": 6.152603089107139e-08,
|
|
"logits/chosen": -364429152.0,
|
|
"logits/rejected": -349219520.0,
|
|
"logps/chosen": -340.38173040334857,
|
|
"logps/rejected": -400.4701043338684,
|
|
"loss": 1.7462,
|
|
"rewards/chosen": -0.6163304066186264,
|
|
"rewards/margins": 0.8346214283156634,
|
|
"rewards/rejected": -1.4509518349342898,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 0.806282722513089,
|
|
"grad_norm": 16.510046005249023,
|
|
"kl": 0.0,
|
|
"learning_rate": 5.5646397529920175e-08,
|
|
"logits/chosen": -363931648.0,
|
|
"logits/rejected": -370810304.0,
|
|
"logps/chosen": -365.2676868044515,
|
|
"logps/rejected": -408.1744911674347,
|
|
"loss": 1.6917,
|
|
"rewards/chosen": -0.5827012297079939,
|
|
"rewards/margins": 0.925817065462897,
|
|
"rewards/rejected": -1.5085182951708909,
|
|
"step": 770
|
|
},
|
|
{
|
|
"epoch": 0.8167539267015707,
|
|
"grad_norm": 15.439537048339844,
|
|
"kl": 0.0,
|
|
"learning_rate": 5.002669488545111e-08,
|
|
"logits/chosen": -354813536.0,
|
|
"logits/rejected": -418654816.0,
|
|
"logps/chosen": -342.4249794745484,
|
|
"logps/rejected": -417.48695976154994,
|
|
"loss": 1.6949,
|
|
"rewards/chosen": -0.5906535432060755,
|
|
"rewards/margins": 0.8275922754763947,
|
|
"rewards/rejected": -1.4182458186824702,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 0.8272251308900523,
|
|
"grad_norm": 53.54264450073242,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.467443881184646e-08,
|
|
"logits/chosen": -380888512.0,
|
|
"logits/rejected": -383134880.0,
|
|
"logps/chosen": -345.58564268867923,
|
|
"logps/rejected": -391.497573757764,
|
|
"loss": 1.7238,
|
|
"rewards/chosen": -0.5928520826423693,
|
|
"rewards/margins": 0.8389644449197424,
|
|
"rewards/rejected": -1.4318165275621118,
|
|
"step": 790
|
|
},
|
|
{
|
|
"epoch": 0.837696335078534,
|
|
"grad_norm": 39.76443862915039,
|
|
"kl": 0.0,
|
|
"learning_rate": 3.959678747720488e-08,
|
|
"logits/chosen": -406014528.0,
|
|
"logits/rejected": -366457024.0,
|
|
"logps/chosen": -337.72622685185183,
|
|
"logps/rejected": -407.47355371900824,
|
|
"loss": 1.7464,
|
|
"rewards/chosen": -0.5733218948929398,
|
|
"rewards/margins": 0.919490181391926,
|
|
"rewards/rejected": -1.4928120762848658,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.837696335078534,
|
|
"eval_kl": 0.0,
|
|
"eval_logits/chosen": -377414720.0,
|
|
"eval_logits/rejected": -376930848.0,
|
|
"eval_logps/chosen": -345.01959375,
|
|
"eval_logps/rejected": -411.8449375,
|
|
"eval_loss": 0.43189236521720886,
|
|
"eval_rewards/chosen": -0.5716385498046875,
|
|
"eval_rewards/margins": 0.87729931640625,
|
|
"eval_rewards/rejected": -1.4489378662109376,
|
|
"eval_runtime": 172.2229,
|
|
"eval_samples_per_second": 23.226,
|
|
"eval_steps_per_second": 0.726,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.8481675392670157,
|
|
"grad_norm": 59.97751998901367,
|
|
"kl": 0.0,
|
|
"learning_rate": 3.480053179012654e-08,
|
|
"logits/chosen": -355148352.0,
|
|
"logits/rejected": -381496640.0,
|
|
"logps/chosen": -333.25054650238474,
|
|
"logps/rejected": -400.6155193932412,
|
|
"loss": 1.7686,
|
|
"rewards/chosen": -0.6660681695741008,
|
|
"rewards/margins": 0.6893864232698413,
|
|
"rewards/rejected": -1.355454592843942,
|
|
"step": 810
|
|
},
|
|
{
|
|
"epoch": 0.8586387434554974,
|
|
"grad_norm": 36.40221405029297,
|
|
"kl": 0.0,
|
|
"learning_rate": 3.029208631747446e-08,
|
|
"logits/chosen": -377712128.0,
|
|
"logits/rejected": -356546464.0,
|
|
"logps/chosen": -329.12806748466255,
|
|
"logps/rejected": -420.65381170382165,
|
|
"loss": 1.6857,
|
|
"rewards/chosen": -0.5312460214813794,
|
|
"rewards/margins": 1.0716216395270302,
|
|
"rewards/rejected": -1.6028676610084096,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 0.8691099476439791,
|
|
"grad_norm": 49.2255859375,
|
|
"kl": 0.0,
|
|
"learning_rate": 2.607748070546037e-08,
|
|
"logits/chosen": -374321088.0,
|
|
"logits/rejected": -385330848.0,
|
|
"logps/chosen": -340.50354889589903,
|
|
"logps/rejected": -425.66645704334366,
|
|
"loss": 1.7034,
|
|
"rewards/chosen": -0.6285861523745564,
|
|
"rewards/margins": 0.9371930228080578,
|
|
"rewards/rejected": -1.5657791751826142,
|
|
"step": 830
|
|
},
|
|
{
|
|
"epoch": 0.8795811518324608,
|
|
"grad_norm": 68.4431381225586,
|
|
"kl": 0.0,
|
|
"learning_rate": 2.2162351615526544e-08,
|
|
"logits/chosen": -379910592.0,
|
|
"logits/rejected": -405686880.0,
|
|
"logps/chosen": -363.5833333333333,
|
|
"logps/rejected": -413.6287980769231,
|
|
"loss": 1.7074,
|
|
"rewards/chosen": -0.6024742005363343,
|
|
"rewards/margins": 0.8925232641571753,
|
|
"rewards/rejected": -1.4949974646935096,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 0.8900523560209425,
|
|
"grad_norm": 47.56254577636719,
|
|
"kl": 0.0,
|
|
"learning_rate": 1.8551935185811717e-08,
|
|
"logits/chosen": -358018880.0,
|
|
"logits/rejected": -372178848.0,
|
|
"logps/chosen": -346.9014880952381,
|
|
"logps/rejected": -434.3848557692308,
|
|
"loss": 1.6874,
|
|
"rewards/chosen": -0.6287514338417659,
|
|
"rewards/margins": 0.9804288170596764,
|
|
"rewards/rejected": -1.6091802509014423,
|
|
"step": 850
|
|
},
|
|
{
|
|
"epoch": 0.900523560209424,
|
|
"grad_norm": 39.030643463134766,
|
|
"kl": 0.0,
|
|
"learning_rate": 1.5251060028279612e-08,
|
|
"logits/chosen": -388987392.0,
|
|
"logits/rejected": -356903168.0,
|
|
"logps/chosen": -339.5614374034003,
|
|
"logps/rejected": -434.5506516587678,
|
|
"loss": 1.7297,
|
|
"rewards/chosen": -0.6679287516904946,
|
|
"rewards/margins": 0.8691605675235654,
|
|
"rewards/rejected": -1.53708931921406,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 0.9109947643979057,
|
|
"grad_norm": 14.048391342163086,
|
|
"kl": 0.0,
|
|
"learning_rate": 1.2264140770878839e-08,
|
|
"logits/chosen": -378864544.0,
|
|
"logits/rejected": -394341504.0,
|
|
"logps/chosen": -366.41154912836765,
|
|
"logps/rejected": -426.361999229584,
|
|
"loss": 1.7101,
|
|
"rewards/chosen": -0.6661284578205477,
|
|
"rewards/margins": 0.9078862146924531,
|
|
"rewards/rejected": -1.5740146725130009,
|
|
"step": 870
|
|
},
|
|
{
|
|
"epoch": 0.9214659685863874,
|
|
"grad_norm": 54.477298736572266,
|
|
"kl": 0.0,
|
|
"learning_rate": 9.59517215336922e-09,
|
|
"logits/chosen": -356389440.0,
|
|
"logits/rejected": -364706528.0,
|
|
"logps/chosen": -348.5319102112676,
|
|
"logps/rejected": -433.04197542901716,
|
|
"loss": 1.6794,
|
|
"rewards/chosen": -0.686365930500538,
|
|
"rewards/margins": 1.0420925633079645,
|
|
"rewards/rejected": -1.7284584938085024,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 0.9319371727748691,
|
|
"grad_norm": 38.72111129760742,
|
|
"kl": 0.0,
|
|
"learning_rate": 7.247723684711382e-09,
|
|
"logits/chosen": -372304384.0,
|
|
"logits/rejected": -349962176.0,
|
|
"logps/chosen": -332.2313262195122,
|
|
"logps/rejected": -413.34995993589746,
|
|
"loss": 1.7364,
|
|
"rewards/chosen": -0.5879382156744236,
|
|
"rewards/margins": 0.8648360278026398,
|
|
"rewards/rejected": -1.4527742434770634,
|
|
"step": 890
|
|
},
|
|
{
|
|
"epoch": 0.9424083769633508,
|
|
"grad_norm": 65.32742309570312,
|
|
"kl": 0.0,
|
|
"learning_rate": 5.224934869164976e-09,
|
|
"logits/chosen": -384202432.0,
|
|
"logits/rejected": -398582304.0,
|
|
"logps/chosen": -358.9657617504052,
|
|
"logps/rejected": -431.56499811463044,
|
|
"loss": 1.7016,
|
|
"rewards/chosen": -0.6722601545881027,
|
|
"rewards/margins": 0.8951942798599554,
|
|
"rewards/rejected": -1.567454434448058,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.9528795811518325,
|
|
"grad_norm": 28.859342575073242,
|
|
"kl": 0.0,
|
|
"learning_rate": 3.529511007479946e-09,
|
|
"logits/chosen": -394682496.0,
|
|
"logits/rejected": -370364992.0,
|
|
"logps/chosen": -354.0907018049155,
|
|
"logps/rejected": -399.780231518283,
|
|
"loss": 1.7362,
|
|
"rewards/chosen": -0.6060863676525298,
|
|
"rewards/margins": 0.8388409995411804,
|
|
"rewards/rejected": -1.4449273671937102,
|
|
"step": 910
|
|
},
|
|
{
|
|
"epoch": 0.9633507853403142,
|
|
"grad_norm": 65.4843521118164,
|
|
"kl": 0.0,
|
|
"learning_rate": 2.1637195787966857e-09,
|
|
"logits/chosen": -364587168.0,
|
|
"logits/rejected": -388965792.0,
|
|
"logps/chosen": -356.6702698635634,
|
|
"logps/rejected": -412.88955479452056,
|
|
"loss": 1.6953,
|
|
"rewards/chosen": -0.5981720117657755,
|
|
"rewards/margins": 0.9335980412093957,
|
|
"rewards/rejected": -1.5317700529751712,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 0.9738219895287958,
|
|
"grad_norm": 41.27497482299805,
|
|
"kl": 0.0,
|
|
"learning_rate": 1.1293872080934963e-09,
|
|
"logits/chosen": -348726528.0,
|
|
"logits/rejected": -397161920.0,
|
|
"logps/chosen": -344.23691152597405,
|
|
"logps/rejected": -425.8096291415663,
|
|
"loss": 1.6647,
|
|
"rewards/chosen": -0.600016011820211,
|
|
"rewards/margins": 0.9757914906506097,
|
|
"rewards/rejected": -1.5758075024708207,
|
|
"step": 930
|
|
},
|
|
{
|
|
"epoch": 0.9842931937172775,
|
|
"grad_norm": 26.048463821411133,
|
|
"kl": 0.0,
|
|
"learning_rate": 4.2789722323760546e-10,
|
|
"logits/chosen": -391641184.0,
|
|
"logits/rejected": -366561632.0,
|
|
"logps/chosen": -350.878531998503,
|
|
"logps/rejected": -422.56597222222223,
|
|
"loss": 1.7305,
|
|
"rewards/chosen": -0.594213657036513,
|
|
"rewards/margins": 0.995801031293246,
|
|
"rewards/rejected": -1.590014688329759,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 0.9947643979057592,
|
|
"grad_norm": 47.700260162353516,
|
|
"kl": 0.0,
|
|
"learning_rate": 6.018780490690822e-11,
|
|
"logits/chosen": -406117312.0,
|
|
"logits/rejected": -365830848.0,
|
|
"logps/chosen": -347.9480587121212,
|
|
"logps/rejected": -418.8353578629032,
|
|
"loss": 1.7033,
|
|
"rewards/chosen": -0.6121012832179215,
|
|
"rewards/margins": 1.0529334477548606,
|
|
"rewards/rejected": -1.6650347309727822,
|
|
"step": 950
|
|
},
|
|
{
|
|
"epoch": 1.0,
|
|
"step": 955,
|
|
"total_flos": 0.0,
|
|
"train_loss": 1.7875109602643557,
|
|
"train_runtime": 10843.0021,
|
|
"train_samples_per_second": 11.276,
|
|
"train_steps_per_second": 0.088
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 955,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 200,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|