jackf857/llama-3-8b-base-cpo-ultrafeedback-8xh200
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: jackf857/llama-3-8b-base-cpo-ultrafeedback-8xh200
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-04-27 23:02:38 +08:00
commit 78abb4a951
19 changed files with 3454 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

845
trainer_state.json Normal file
View File

@@ -0,0 +1,845 @@
{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.9989528795811519,
"eval_steps": 200,
"global_step": 477,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.0020942408376963353,
"grad_norm": 241.46054077148438,
"learning_rate": 0.0,
"logits/chosen": -0.615048885345459,
"logits/rejected": -0.6184952855110168,
"logps/chosen": -340.7542724609375,
"logps/rejected": -224.71102905273438,
"loss": 46.3402,
"nll_loss": 1.1951930522918701,
"rewards/accuracies": 0.5,
"rewards/chosen": -34.0754280090332,
"rewards/margins": -11.60432243347168,
"rewards/rejected": -22.47110366821289,
"step": 1
},
{
"epoch": 0.020942408376963352,
"grad_norm": 207.28042602539062,
"learning_rate": 9.375e-08,
"logits/chosen": -0.6425137519836426,
"logits/rejected": -0.636458158493042,
"logps/chosen": -259.6365051269531,
"logps/rejected": -258.07281494140625,
"loss": 42.6859,
"nll_loss": 1.0321696996688843,
"rewards/accuracies": 0.4930555522441864,
"rewards/chosen": -25.96364974975586,
"rewards/margins": -0.1563688963651657,
"rewards/rejected": -25.807281494140625,
"step": 10
},
{
"epoch": 0.041884816753926704,
"grad_norm": 241.39328002929688,
"learning_rate": 1.9791666666666664e-07,
"logits/chosen": -0.6058956384658813,
"logits/rejected": -0.6296488046646118,
"logps/chosen": -292.1470642089844,
"logps/rejected": -243.1712188720703,
"loss": 42.1261,
"nll_loss": 1.0961360931396484,
"rewards/accuracies": 0.45625001192092896,
"rewards/chosen": -29.214702606201172,
"rewards/margins": -4.897582530975342,
"rewards/rejected": -24.317123413085938,
"step": 20
},
{
"epoch": 0.06282722513089005,
"grad_norm": 214.8778839111328,
"learning_rate": 3.020833333333333e-07,
"logits/chosen": -0.5835741758346558,
"logits/rejected": -0.5625507831573486,
"logps/chosen": -266.05560302734375,
"logps/rejected": -251.7474822998047,
"loss": 41.0696,
"nll_loss": 1.0376346111297607,
"rewards/accuracies": 0.46875,
"rewards/chosen": -26.60555648803711,
"rewards/margins": -1.4308110475540161,
"rewards/rejected": -25.174747467041016,
"step": 30
},
{
"epoch": 0.08376963350785341,
"grad_norm": 228.7670440673828,
"learning_rate": 4.0625e-07,
"logits/chosen": -0.6142115592956543,
"logits/rejected": -0.6461960077285767,
"logps/chosen": -280.4624938964844,
"logps/rejected": -260.13128662109375,
"loss": 42.8858,
"nll_loss": 1.001647710800171,
"rewards/accuracies": 0.512499988079071,
"rewards/chosen": -28.046253204345703,
"rewards/margins": -2.0331199169158936,
"rewards/rejected": -26.013132095336914,
"step": 40
},
{
"epoch": 0.10471204188481675,
"grad_norm": 203.48428344726562,
"learning_rate": 4.999932966293553e-07,
"logits/chosen": -0.6760377287864685,
"logits/rejected": -0.6930453777313232,
"logps/chosen": -257.87176513671875,
"logps/rejected": -250.3403778076172,
"loss": 39.7368,
"nll_loss": 0.9915801286697388,
"rewards/accuracies": 0.4375,
"rewards/chosen": -25.78717613220215,
"rewards/margins": -0.7531374096870422,
"rewards/rejected": -25.034040451049805,
"step": 50
},
{
"epoch": 0.1256544502617801,
"grad_norm": 204.22805786132812,
"learning_rate": 4.991893270335525e-07,
"logits/chosen": -0.6431644558906555,
"logits/rejected": -0.6466089487075806,
"logps/chosen": -284.398681640625,
"logps/rejected": -275.51971435546875,
"loss": 40.0522,
"nll_loss": 1.030461311340332,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -28.439865112304688,
"rewards/margins": -0.8878933191299438,
"rewards/rejected": -27.55197525024414,
"step": 60
},
{
"epoch": 0.14659685863874344,
"grad_norm": 216.3765411376953,
"learning_rate": 4.970496218214204e-07,
"logits/chosen": -0.7524436712265015,
"logits/rejected": -0.7512598633766174,
"logps/chosen": -288.4020080566406,
"logps/rejected": -285.08770751953125,
"loss": 37.5469,
"nll_loss": 1.122981309890747,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -28.840198516845703,
"rewards/margins": -0.3314289450645447,
"rewards/rejected": -28.508769989013672,
"step": 70
},
{
"epoch": 0.16753926701570682,
"grad_norm": 221.7291259765625,
"learning_rate": 4.935856505068998e-07,
"logits/chosen": -0.7214940786361694,
"logits/rejected": -0.7364234328269958,
"logps/chosen": -263.50787353515625,
"logps/rejected": -245.77099609375,
"loss": 36.5621,
"nll_loss": 1.0321334600448608,
"rewards/accuracies": 0.42500001192092896,
"rewards/chosen": -26.350788116455078,
"rewards/margins": -1.7736867666244507,
"rewards/rejected": -24.577102661132812,
"step": 80
},
{
"epoch": 0.18848167539267016,
"grad_norm": 209.3531951904297,
"learning_rate": 4.8881598109976e-07,
"logits/chosen": -0.6903802752494812,
"logits/rejected": -0.7218402028083801,
"logps/chosen": -283.48724365234375,
"logps/rejected": -249.9278564453125,
"loss": 37.2678,
"nll_loss": 1.0136864185333252,
"rewards/accuracies": 0.4437499940395355,
"rewards/chosen": -28.348724365234375,
"rewards/margins": -3.35593843460083,
"rewards/rejected": -24.992786407470703,
"step": 90
},
{
"epoch": 0.2094240837696335,
"grad_norm": 222.45443725585938,
"learning_rate": 4.827661805750437e-07,
"logits/chosen": -0.6955899000167847,
"logits/rejected": -0.7055763006210327,
"logps/chosen": -265.42913818359375,
"logps/rejected": -276.52716064453125,
"loss": 36.8305,
"nll_loss": 0.9817202687263489,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -26.542917251586914,
"rewards/margins": 1.1097980737686157,
"rewards/rejected": -27.6527156829834,
"step": 100
},
{
"epoch": 0.23036649214659685,
"grad_norm": 233.69064331054688,
"learning_rate": 4.75468677825789e-07,
"logits/chosen": -0.6523857116699219,
"logits/rejected": -0.7047474980354309,
"logps/chosen": -286.0872802734375,
"logps/rejected": -300.1228942871094,
"loss": 34.9865,
"nll_loss": 0.9495639801025391,
"rewards/accuracies": 0.512499988079071,
"rewards/chosen": -28.608728408813477,
"rewards/margins": 1.403562307357788,
"rewards/rejected": -30.012292861938477,
"step": 110
},
{
"epoch": 0.2513089005235602,
"grad_norm": 489.7384338378906,
"learning_rate": 4.669625898336438e-07,
"logits/chosen": -0.7176281213760376,
"logits/rejected": -0.7226043939590454,
"logps/chosen": -289.14080810546875,
"logps/rejected": -291.45965576171875,
"loss": 34.8572,
"nll_loss": 1.205727458000183,
"rewards/accuracies": 0.45625001192092896,
"rewards/chosen": -28.914077758789062,
"rewards/margins": 0.2318885773420334,
"rewards/rejected": -29.14596939086914,
"step": 120
},
{
"epoch": 0.27225130890052357,
"grad_norm": 344.9328308105469,
"learning_rate": 4.5729351198915705e-07,
"logits/chosen": -0.6907280683517456,
"logits/rejected": -0.6784687042236328,
"logps/chosen": -315.9068298339844,
"logps/rejected": -321.81317138671875,
"loss": 33.9604,
"nll_loss": 1.2358386516571045,
"rewards/accuracies": 0.518750011920929,
"rewards/chosen": -31.590686798095703,
"rewards/margins": 0.5906292796134949,
"rewards/rejected": -32.18132019042969,
"step": 130
},
{
"epoch": 0.2931937172774869,
"grad_norm": 940.4654541015625,
"learning_rate": 4.4651327368569684e-07,
"logits/chosen": -0.7276722192764282,
"logits/rejected": -0.6987274289131165,
"logps/chosen": -334.77227783203125,
"logps/rejected": -322.3913269042969,
"loss": 31.7891,
"nll_loss": 1.4334642887115479,
"rewards/accuracies": 0.518750011920929,
"rewards/chosen": -33.47722625732422,
"rewards/margins": -1.2380931377410889,
"rewards/rejected": -32.239131927490234,
"step": 140
},
{
"epoch": 0.31413612565445026,
"grad_norm": 532.178955078125,
"learning_rate": 4.346796604970912e-07,
"logits/chosen": -0.7420992851257324,
"logits/rejected": -0.7299410104751587,
"logps/chosen": -352.385498046875,
"logps/rejected": -325.99505615234375,
"loss": 31.6743,
"nll_loss": 1.2520462274551392,
"rewards/accuracies": 0.4124999940395355,
"rewards/chosen": -35.238548278808594,
"rewards/margins": -2.63904070854187,
"rewards/rejected": -32.599510192871094,
"step": 150
},
{
"epoch": 0.33507853403141363,
"grad_norm": 791.0310668945312,
"learning_rate": 4.218561044282098e-07,
"logits/chosen": -0.7037047147750854,
"logits/rejected": -0.7137006521224976,
"logps/chosen": -320.0517578125,
"logps/rejected": -320.142578125,
"loss": 30.3343,
"nll_loss": 1.385374903678894,
"rewards/accuracies": 0.518750011920929,
"rewards/chosen": -32.00517654418945,
"rewards/margins": 0.009086417965590954,
"rewards/rejected": -32.014259338378906,
"step": 160
},
{
"epoch": 0.35602094240837695,
"grad_norm": 458.65313720703125,
"learning_rate": 4.081113438988443e-07,
"logits/chosen": -0.7242492437362671,
"logits/rejected": -0.7235726118087769,
"logps/chosen": -355.2301330566406,
"logps/rejected": -352.45684814453125,
"loss": 31.6656,
"nll_loss": 1.4098021984100342,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -35.52301025390625,
"rewards/margins": -0.2773243486881256,
"rewards/rejected": -35.24568557739258,
"step": 170
},
{
"epoch": 0.3769633507853403,
"grad_norm": 332.9512023925781,
"learning_rate": 3.935190552834828e-07,
"logits/chosen": -0.6966814994812012,
"logits/rejected": -0.7147270441055298,
"logps/chosen": -338.35791015625,
"logps/rejected": -338.9778137207031,
"loss": 28.4701,
"nll_loss": 1.5228649377822876,
"rewards/accuracies": 0.518750011920929,
"rewards/chosen": -33.835792541503906,
"rewards/margins": 0.06198873370885849,
"rewards/rejected": -33.89778137207031,
"step": 180
},
{
"epoch": 0.39790575916230364,
"grad_norm": 504.8504333496094,
"learning_rate": 3.781574579820464e-07,
"logits/chosen": -0.7642617225646973,
"logits/rejected": -0.7676379084587097,
"logps/chosen": -336.16644287109375,
"logps/rejected": -335.4501953125,
"loss": 28.6168,
"nll_loss": 1.2395371198654175,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -33.61664581298828,
"rewards/margins": -0.0716233029961586,
"rewards/rejected": -33.545021057128906,
"step": 190
},
{
"epoch": 0.418848167539267,
"grad_norm": 662.9274291992188,
"learning_rate": 3.621088951385353e-07,
"logits/chosen": -0.7552827000617981,
"logits/rejected": -0.778997540473938,
"logps/chosen": -305.7959289550781,
"logps/rejected": -326.4095764160156,
"loss": 27.4789,
"nll_loss": 1.2367076873779297,
"rewards/accuracies": 0.581250011920929,
"rewards/chosen": -30.5795955657959,
"rewards/margins": 2.0613608360290527,
"rewards/rejected": -32.640953063964844,
"step": 200
},
{
"epoch": 0.418848167539267,
"eval_logits/chosen": -0.7581648826599121,
"eval_logits/rejected": -0.7404313087463379,
"eval_logps/chosen": -342.4026184082031,
"eval_logps/rejected": -349.4523620605469,
"eval_loss": 6.657596111297607,
"eval_nll_loss": 1.230733036994934,
"eval_rewards/accuracies": 0.5766128897666931,
"eval_rewards/chosen": -34.240257263183594,
"eval_rewards/margins": 0.704971432685852,
"eval_rewards/rejected": -34.94523239135742,
"eval_runtime": 23.2353,
"eval_samples_per_second": 86.076,
"eval_steps_per_second": 2.711,
"step": 200
},
{
"epoch": 0.4397905759162304,
"grad_norm": 579.301025390625,
"learning_rate": 3.454593922550693e-07,
"logits/chosen": -0.6766096949577332,
"logits/rejected": -0.6979749798774719,
"logps/chosen": -336.41845703125,
"logps/rejected": -355.58294677734375,
"loss": 26.676,
"nll_loss": 1.2723300457000732,
"rewards/accuracies": 0.5687500238418579,
"rewards/chosen": -33.641845703125,
"rewards/margins": 1.916452407836914,
"rewards/rejected": -35.55829620361328,
"step": 210
},
{
"epoch": 0.4607329842931937,
"grad_norm": 1089.4422607421875,
"learning_rate": 3.2829819606729477e-07,
"logits/chosen": -0.7449113130569458,
"logits/rejected": -0.7312273979187012,
"logps/chosen": -325.61102294921875,
"logps/rejected": -380.57421875,
"loss": 27.9027,
"nll_loss": 1.3178092241287231,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -32.56110763549805,
"rewards/margins": 5.496316432952881,
"rewards/rejected": -38.05742263793945,
"step": 220
},
{
"epoch": 0.4816753926701571,
"grad_norm": 529.4258422851562,
"learning_rate": 3.1071729615293424e-07,
"logits/chosen": -0.7479566335678101,
"logits/rejected": -0.7232205271720886,
"logps/chosen": -376.5633239746094,
"logps/rejected": -377.84942626953125,
"loss": 25.7607,
"nll_loss": 1.54669189453125,
"rewards/accuracies": 0.5062500238418579,
"rewards/chosen": -37.65633773803711,
"rewards/margins": 0.12861032783985138,
"rewards/rejected": -37.784950256347656,
"step": 230
},
{
"epoch": 0.5026178010471204,
"grad_norm": 497.5738525390625,
"learning_rate": 2.9281093183781403e-07,
"logits/chosen": -0.6627580523490906,
"logits/rejected": -0.704414427280426,
"logps/chosen": -320.0458679199219,
"logps/rejected": -344.0610656738281,
"loss": 27.43,
"nll_loss": 1.3753007650375366,
"rewards/accuracies": 0.59375,
"rewards/chosen": -32.00458908081055,
"rewards/margins": 2.401517391204834,
"rewards/rejected": -34.406105041503906,
"step": 240
},
{
"epoch": 0.5235602094240838,
"grad_norm": 388.00811767578125,
"learning_rate": 2.7467508704251135e-07,
"logits/chosen": -0.7279710173606873,
"logits/rejected": -0.7237112522125244,
"logps/chosen": -366.0332336425781,
"logps/rejected": -404.2882080078125,
"loss": 26.6833,
"nll_loss": 1.5022847652435303,
"rewards/accuracies": 0.5562499761581421,
"rewards/chosen": -36.60332489013672,
"rewards/margins": 3.8254940509796143,
"rewards/rejected": -40.42881774902344,
"step": 250
},
{
"epoch": 0.5445026178010471,
"grad_norm": 488.0087585449219,
"learning_rate": 2.5640697577740815e-07,
"logits/chosen": -0.7367153167724609,
"logits/rejected": -0.7415083646774292,
"logps/chosen": -342.8262023925781,
"logps/rejected": -365.2310791015625,
"loss": 26.6956,
"nll_loss": 1.3738378286361694,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": -34.282615661621094,
"rewards/margins": 2.2404911518096924,
"rewards/rejected": -36.523109436035156,
"step": 260
},
{
"epoch": 0.5654450261780105,
"grad_norm": 516.7625732421875,
"learning_rate": 2.381045210440644e-07,
"logits/chosen": -0.721376895904541,
"logits/rejected": -0.7088552713394165,
"logps/chosen": -339.37744140625,
"logps/rejected": -377.49615478515625,
"loss": 23.9279,
"nll_loss": 1.59787917137146,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": -33.937744140625,
"rewards/margins": 3.811868667602539,
"rewards/rejected": -37.749610900878906,
"step": 270
},
{
"epoch": 0.5863874345549738,
"grad_norm": 540.905029296875,
"learning_rate": 2.1986582993616925e-07,
"logits/chosen": -0.7323180437088013,
"logits/rejected": -0.7498946189880371,
"logps/chosen": -361.0552673339844,
"logps/rejected": -382.8980712890625,
"loss": 23.6927,
"nll_loss": 1.3734923601150513,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": -36.10552978515625,
"rewards/margins": 2.184277057647705,
"rewards/rejected": -38.2898063659668,
"step": 280
},
{
"epoch": 0.6073298429319371,
"grad_norm": 614.5367431640625,
"learning_rate": 2.0178866775369774e-07,
"logits/chosen": -0.7599818110466003,
"logits/rejected": -0.7415232062339783,
"logps/chosen": -390.739013671875,
"logps/rejected": -402.10894775390625,
"loss": 26.7315,
"nll_loss": 1.4149469137191772,
"rewards/accuracies": 0.606249988079071,
"rewards/chosen": -39.07390594482422,
"rewards/margins": 1.1369867324829102,
"rewards/rejected": -40.21089553833008,
"step": 290
},
{
"epoch": 0.6282722513089005,
"grad_norm": 611.55517578125,
"learning_rate": 1.839699339491937e-07,
"logits/chosen": -0.7755026817321777,
"logits/rejected": -0.7531968355178833,
"logps/chosen": -340.93896484375,
"logps/rejected": -375.37054443359375,
"loss": 23.0069,
"nll_loss": 1.4688938856124878,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -34.093894958496094,
"rewards/margins": 3.4431610107421875,
"rewards/rejected": -37.53705596923828,
"step": 300
},
{
"epoch": 0.6492146596858639,
"grad_norm": 559.4390258789062,
"learning_rate": 1.6650514271527465e-07,
"logits/chosen": -0.720358669757843,
"logits/rejected": -0.7089865803718567,
"logps/chosen": -381.8735656738281,
"logps/rejected": -424.19805908203125,
"loss": 23.6405,
"nll_loss": 1.3657066822052002,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": -38.187355041503906,
"rewards/margins": 4.23245096206665,
"rewards/rejected": -42.419803619384766,
"step": 310
},
{
"epoch": 0.6701570680628273,
"grad_norm": 566.0950927734375,
"learning_rate": 1.4948791099758052e-07,
"logits/chosen": -0.751908004283905,
"logits/rejected": -0.7763121724128723,
"logps/chosen": -361.6825256347656,
"logps/rejected": -368.67047119140625,
"loss": 24.4483,
"nll_loss": 1.596862554550171,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -36.168251037597656,
"rewards/margins": 0.6988004446029663,
"rewards/rejected": -36.8670539855957,
"step": 320
},
{
"epoch": 0.6910994764397905,
"grad_norm": 776.851806640625,
"learning_rate": 1.3300945667758012e-07,
"logits/chosen": -0.7262955904006958,
"logits/rejected": -0.7191631197929382,
"logps/chosen": -372.2191162109375,
"logps/rejected": -400.3269348144531,
"loss": 23.3532,
"nll_loss": 1.5126838684082031,
"rewards/accuracies": 0.606249988079071,
"rewards/chosen": -37.22191619873047,
"rewards/margins": 2.810784101486206,
"rewards/rejected": -40.03269577026367,
"step": 330
},
{
"epoch": 0.7120418848167539,
"grad_norm": 518.089111328125,
"learning_rate": 1.1715810961514072e-07,
"logits/chosen": -0.7572416067123413,
"logits/rejected": -0.7408580780029297,
"logps/chosen": -341.0003356933594,
"logps/rejected": -368.7081604003906,
"loss": 23.7244,
"nll_loss": 1.5676231384277344,
"rewards/accuracies": 0.6312500238418579,
"rewards/chosen": -34.10003662109375,
"rewards/margins": 2.7707810401916504,
"rewards/rejected": -36.870811462402344,
"step": 340
},
{
"epoch": 0.7329842931937173,
"grad_norm": 778.4837646484375,
"learning_rate": 1.0201883817182949e-07,
"logits/chosen": -0.7688989639282227,
"logits/rejected": -0.7650505304336548,
"logps/chosen": -354.57867431640625,
"logps/rejected": -391.5706481933594,
"loss": 22.8915,
"nll_loss": 1.5152699947357178,
"rewards/accuracies": 0.581250011920929,
"rewards/chosen": -35.45786666870117,
"rewards/margins": 3.6991982460021973,
"rewards/rejected": -39.157066345214844,
"step": 350
},
{
"epoch": 0.7539267015706806,
"grad_norm": 949.5282592773438,
"learning_rate": 8.76727937529367e-08,
"logits/chosen": -0.7362595796585083,
"logits/rejected": -0.7329570651054382,
"logps/chosen": -350.1195068359375,
"logps/rejected": -367.3720703125,
"loss": 23.4713,
"nll_loss": 1.4863172769546509,
"rewards/accuracies": 0.5562499761581421,
"rewards/chosen": -35.01194763183594,
"rewards/margins": 1.7252594232559204,
"rewards/rejected": -36.73720932006836,
"step": 360
},
{
"epoch": 0.774869109947644,
"grad_norm": 623.1497802734375,
"learning_rate": 7.419687580962222e-08,
"logits/chosen": -0.7591558694839478,
"logits/rejected": -0.7718418836593628,
"logps/chosen": -376.29229736328125,
"logps/rejected": -411.29266357421875,
"loss": 23.6771,
"nll_loss": 1.399916410446167,
"rewards/accuracies": 0.581250011920929,
"rewards/chosen": -37.629234313964844,
"rewards/margins": 3.500037431716919,
"rewards/rejected": -41.12926483154297,
"step": 370
},
{
"epoch": 0.7958115183246073,
"grad_norm": 551.5185546875,
"learning_rate": 6.166331963291519e-08,
"logits/chosen": -0.7095741033554077,
"logits/rejected": -0.7066128849983215,
"logps/chosen": -371.6845703125,
"logps/rejected": -378.3497619628906,
"loss": 24.297,
"nll_loss": 1.507645845413208,
"rewards/accuracies": 0.59375,
"rewards/chosen": -37.16845703125,
"rewards/margins": 0.6665191054344177,
"rewards/rejected": -37.8349723815918,
"step": 380
},
{
"epoch": 0.8167539267015707,
"grad_norm": 537.383056640625,
"learning_rate": 5.013930914912476e-08,
"logits/chosen": -0.7830756902694702,
"logits/rejected": -0.7760699987411499,
"logps/chosen": -338.2967224121094,
"logps/rejected": -373.68524169921875,
"loss": 20.8009,
"nll_loss": 1.4299148321151733,
"rewards/accuracies": 0.606249988079071,
"rewards/chosen": -33.829673767089844,
"rewards/margins": 3.538851261138916,
"rewards/rejected": -37.36852264404297,
"step": 390
},
{
"epoch": 0.837696335078534,
"grad_norm": 562.61572265625,
"learning_rate": 3.968661679220467e-08,
"logits/chosen": -0.7369313836097717,
"logits/rejected": -0.7653532028198242,
"logps/chosen": -361.5699462890625,
"logps/rejected": -360.4308166503906,
"loss": 23.6103,
"nll_loss": 1.4506025314331055,
"rewards/accuracies": 0.5687500238418579,
"rewards/chosen": -36.1569938659668,
"rewards/margins": -0.11391093581914902,
"rewards/rejected": -36.04308319091797,
"step": 400
},
{
"epoch": 0.837696335078534,
"eval_logits/chosen": -0.7660086750984192,
"eval_logits/rejected": -0.7502105832099915,
"eval_logps/chosen": -368.87127685546875,
"eval_logps/rejected": -387.3282775878906,
"eval_loss": 5.662460803985596,
"eval_nll_loss": 1.3492006063461304,
"eval_rewards/accuracies": 0.625,
"eval_rewards/chosen": -36.88712692260742,
"eval_rewards/margins": 1.845699429512024,
"eval_rewards/rejected": -38.732826232910156,
"eval_runtime": 23.0343,
"eval_samples_per_second": 86.827,
"eval_steps_per_second": 2.735,
"step": 400
},
{
"epoch": 0.8586387434554974,
"grad_norm": 519.74755859375,
"learning_rate": 3.036127238347164e-08,
"logits/chosen": -0.7296323180198669,
"logits/rejected": -0.7103043794631958,
"logps/chosen": -362.31890869140625,
"logps/rejected": -402.84063720703125,
"loss": 23.8299,
"nll_loss": 1.475502371788025,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": -36.231895446777344,
"rewards/margins": 4.052168369293213,
"rewards/rejected": -40.284061431884766,
"step": 410
},
{
"epoch": 0.8795811518324608,
"grad_norm": 791.4617309570312,
"learning_rate": 2.2213262793589482e-08,
"logits/chosen": -0.7181447744369507,
"logits/rejected": -0.7083781957626343,
"logps/chosen": -403.479248046875,
"logps/rejected": -397.3186950683594,
"loss": 22.7265,
"nll_loss": 1.512879490852356,
"rewards/accuracies": 0.5687500238418579,
"rewards/chosen": -40.347923278808594,
"rewards/margins": -0.6160532236099243,
"rewards/rejected": -39.731868743896484,
"step": 420
},
{
"epoch": 0.900523560209424,
"grad_norm": 559.8237915039062,
"learning_rate": 1.5286263996730026e-08,
"logits/chosen": -0.7705979943275452,
"logits/rejected": -0.7675026655197144,
"logps/chosen": -378.0232238769531,
"logps/rejected": -406.818115234375,
"loss": 23.3836,
"nll_loss": 1.365422248840332,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": -37.80231857299805,
"rewards/margins": 2.879490613937378,
"rewards/rejected": -40.68181610107422,
"step": 430
},
{
"epoch": 0.9214659685863874,
"grad_norm": 655.5603637695312,
"learning_rate": 9.617406953185136e-09,
"logits/chosen": -0.7454264760017395,
"logits/rejected": -0.7567831873893738,
"logps/chosen": -374.8846130371094,
"logps/rejected": -412.54425048828125,
"loss": 23.5113,
"nll_loss": 1.4483903646469116,
"rewards/accuracies": 0.625,
"rewards/chosen": -37.488460540771484,
"rewards/margins": 3.765960216522217,
"rewards/rejected": -41.25442123413086,
"step": 440
},
{
"epoch": 0.9424083769633508,
"grad_norm": 1095.772216796875,
"learning_rate": 5.2370785753763356e-09,
"logits/chosen": -0.7458157539367676,
"logits/rejected": -0.7518310546875,
"logps/chosen": -365.1365051269531,
"logps/rejected": -369.6548156738281,
"loss": 22.6642,
"nll_loss": 1.3850022554397583,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": -36.513648986816406,
"rewards/margins": 0.4518311023712158,
"rewards/rejected": -36.96548080444336,
"step": 450
},
{
"epoch": 0.9633507853403142,
"grad_norm": 877.9326782226562,
"learning_rate": 2.168758844148272e-09,
"logits/chosen": -0.7453271150588989,
"logits/rejected": -0.7776076197624207,
"logps/chosen": -409.984375,
"logps/rejected": -432.01141357421875,
"loss": 23.1426,
"nll_loss": 1.4942188262939453,
"rewards/accuracies": 0.5874999761581421,
"rewards/chosen": -40.998443603515625,
"rewards/margins": 2.202702760696411,
"rewards/rejected": -43.201148986816406,
"step": 460
},
{
"epoch": 0.9842931937172775,
"grad_norm": 576.8656616210938,
"learning_rate": 4.288949484559934e-10,
"logits/chosen": -0.7500009536743164,
"logits/rejected": -0.7585066556930542,
"logps/chosen": -346.47332763671875,
"logps/rejected": -393.7072448730469,
"loss": 24.6934,
"nll_loss": 1.4887354373931885,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": -34.64733123779297,
"rewards/margins": 4.723390102386475,
"rewards/rejected": -39.37071990966797,
"step": 470
},
{
"epoch": 0.9989528795811519,
"step": 477,
"total_flos": 0.0,
"train_loss": 28.95567873588898,
"train_runtime": 2691.8939,
"train_samples_per_second": 22.711,
"train_steps_per_second": 0.177
}
],
"logging_steps": 10,
"max_steps": 477,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 500,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": false,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}