2da38c3175
Model: Gueule-d-ange/aup-fullft-kto_kl-klam0.0333_beta0.1-seed6 Source: Original Platform
2344 lines
86 KiB
JSON
2344 lines
86 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 14.287727355957031,
|
|
"kl": 9.702888488769531,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -37766296.787301585,
|
|
"logits/rejected": -37879117.98153846,
|
|
"logps/chosen": -493.69444444444446,
|
|
"logps/rejected": -370.3830288461538,
|
|
"loss": 0.5508,
|
|
"loss/base_kto": 3.9777638912200928,
|
|
"loss/total_with_kl": 4.406555652618408,
|
|
"metrics/advantage_var": 257.5327453613281,
|
|
"regularization/advantage_var": 257.5327453613281,
|
|
"regularization/kl": 0.42879199981689453,
|
|
"rewards/chosen": 0.014911466931539868,
|
|
"rewards/margins": 0.008889756022209941,
|
|
"rewards/rejected": 0.006021710909329928,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 14.998821258544922,
|
|
"kl": 5.43628454208374,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -36476287.80093312,
|
|
"logits/rejected": -37493198.97017268,
|
|
"logps/chosen": -500.11683514774495,
|
|
"logps/rejected": -377.49774332810046,
|
|
"loss": 0.5203,
|
|
"loss/base_kto": 3.8238632678985596,
|
|
"loss/total_with_kl": 4.1620378494262695,
|
|
"metrics/advantage_var": 203.10780334472656,
|
|
"regularization/advantage_var": 203.10780334472656,
|
|
"regularization/kl": 0.3381744921207428,
|
|
"rewards/chosen": 0.0926294400940416,
|
|
"rewards/margins": 0.1690259690237475,
|
|
"rewards/rejected": -0.0763965289297059,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 16.887611389160156,
|
|
"kl": 18.383010864257812,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -36128742.94801223,
|
|
"logits/rejected": -36806898.095846646,
|
|
"logps/chosen": -471.84064411314984,
|
|
"logps/rejected": -394.3551567492013,
|
|
"loss": 0.52,
|
|
"loss/base_kto": 3.775472402572632,
|
|
"loss/total_with_kl": 4.160192012786865,
|
|
"metrics/advantage_var": 231.06277465820312,
|
|
"regularization/advantage_var": 231.06277465820312,
|
|
"regularization/kl": 0.38471952080726624,
|
|
"rewards/chosen": 0.39787493131211776,
|
|
"rewards/margins": 0.1800138654627892,
|
|
"rewards/rejected": 0.21786106584932857,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 15.66333293914795,
|
|
"kl": 11.085951805114746,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -36256484.90363349,
|
|
"logits/rejected": -38733222.57805255,
|
|
"logps/chosen": -478.31161137440756,
|
|
"logps/rejected": -388.8999227202473,
|
|
"loss": 0.5267,
|
|
"loss/base_kto": 3.828054904937744,
|
|
"loss/total_with_kl": 4.213367938995361,
|
|
"metrics/advantage_var": 231.4193115234375,
|
|
"regularization/advantage_var": 231.4193115234375,
|
|
"regularization/kl": 0.38531312346458435,
|
|
"rewards/chosen": 0.20424282080000988,
|
|
"rewards/margins": 0.1500360551991598,
|
|
"rewards/rejected": 0.054206765600850075,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 12.986501693725586,
|
|
"kl": 4.356201171875,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -36584841.09952607,
|
|
"logits/rejected": -36553039.5301391,
|
|
"logps/chosen": -472.4236769352291,
|
|
"logps/rejected": -380.4377897990726,
|
|
"loss": 0.5204,
|
|
"loss/base_kto": 3.8169708251953125,
|
|
"loss/total_with_kl": 4.163290500640869,
|
|
"metrics/advantage_var": 207.9998779296875,
|
|
"regularization/advantage_var": 207.9998779296875,
|
|
"regularization/kl": 0.3463198244571686,
|
|
"rewards/chosen": 0.04626592000325521,
|
|
"rewards/margins": 0.18192025744257853,
|
|
"rewards/rejected": -0.1356543374393233,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 15.260271072387695,
|
|
"kl": 18.893110275268555,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -38005558.91238671,
|
|
"logits/rejected": -38157421.3592233,
|
|
"logps/chosen": -475.55258685800607,
|
|
"logps/rejected": -365.3332322006473,
|
|
"loss": 0.5187,
|
|
"loss/base_kto": 3.7780284881591797,
|
|
"loss/total_with_kl": 4.149636268615723,
|
|
"metrics/advantage_var": 223.18765258789062,
|
|
"regularization/advantage_var": 223.18765258789062,
|
|
"regularization/kl": 0.3716074526309967,
|
|
"rewards/chosen": 0.3693013724243533,
|
|
"rewards/margins": 0.20797817072943267,
|
|
"rewards/rejected": 0.16132320169492062,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 19.728679656982422,
|
|
"kl": 13.507367134094238,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -36914409.6,
|
|
"logits/rejected": -38909216.0,
|
|
"logps/chosen": -496.243603515625,
|
|
"logps/rejected": -365.408544921875,
|
|
"loss": 0.5181,
|
|
"loss/base_kto": 3.7337799072265625,
|
|
"loss/total_with_kl": 4.145013332366943,
|
|
"metrics/advantage_var": 246.9873046875,
|
|
"regularization/advantage_var": 246.9873046875,
|
|
"regularization/kl": 0.4112338721752167,
|
|
"rewards/chosen": 0.2690389156341553,
|
|
"rewards/margins": 0.23199591934680938,
|
|
"rewards/rejected": 0.037042996287345885,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 14.038232803344727,
|
|
"kl": 6.005091667175293,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -36259670.3803681,
|
|
"logits/rejected": -37505125.0955414,
|
|
"logps/chosen": -479.71985237730064,
|
|
"logps/rejected": -366.8520103503185,
|
|
"loss": 0.5149,
|
|
"loss/base_kto": 3.7456071376800537,
|
|
"loss/total_with_kl": 4.119256973266602,
|
|
"metrics/advantage_var": 224.414306640625,
|
|
"regularization/advantage_var": 224.414306640625,
|
|
"regularization/kl": 0.37364980578422546,
|
|
"rewards/chosen": 0.10584174782220572,
|
|
"rewards/margins": 0.24838081469574397,
|
|
"rewards/rejected": -0.14253906687353826,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 13.73609447479248,
|
|
"kl": 3.312875509262085,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -36806536.699029125,
|
|
"logits/rejected": -36657430.42900302,
|
|
"logps/chosen": -471.7578883495146,
|
|
"logps/rejected": -371.85309667673715,
|
|
"loss": 0.5312,
|
|
"loss/base_kto": 3.809211492538452,
|
|
"loss/total_with_kl": 4.249508857727051,
|
|
"metrics/advantage_var": 264.4427185058594,
|
|
"regularization/advantage_var": 264.4427185058594,
|
|
"regularization/kl": 0.44029712677001953,
|
|
"rewards/chosen": -0.05681952072192936,
|
|
"rewards/margins": 0.16657122561079818,
|
|
"rewards/rejected": -0.22339074633272754,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 15.996874809265137,
|
|
"kl": 7.149564266204834,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -36565105.40716612,
|
|
"logits/rejected": -37994268.44444445,
|
|
"logps/chosen": -473.9944523615635,
|
|
"logps/rejected": -369.78915634384384,
|
|
"loss": 0.5165,
|
|
"loss/base_kto": 3.7622039318084717,
|
|
"loss/total_with_kl": 4.132062911987305,
|
|
"metrics/advantage_var": 222.13755798339844,
|
|
"regularization/advantage_var": 222.13755798339844,
|
|
"regularization/kl": 0.3698590397834778,
|
|
"rewards/chosen": 0.0624604861977046,
|
|
"rewards/margins": 0.2242974368263117,
|
|
"rewards/rejected": -0.16183695062860712,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 13.845978736877441,
|
|
"kl": 12.047240257263184,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -36781844.73637702,
|
|
"logits/rejected": -37275191.374376036,
|
|
"logps/chosen": -493.8017765095729,
|
|
"logps/rejected": -380.1634255407654,
|
|
"loss": 0.5152,
|
|
"loss/base_kto": 3.6904220581054688,
|
|
"loss/total_with_kl": 4.1215033531188965,
|
|
"metrics/advantage_var": 258.90771484375,
|
|
"regularization/advantage_var": 258.90771484375,
|
|
"regularization/kl": 0.4310813844203949,
|
|
"rewards/chosen": 0.24988557031825295,
|
|
"rewards/margins": 0.3052420958599933,
|
|
"rewards/rejected": -0.05535652554174033,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 15.366321563720703,
|
|
"kl": 10.131718635559082,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -35879686.48101266,
|
|
"logits/rejected": -37246780.04938272,
|
|
"logps/chosen": -497.8791534810127,
|
|
"logps/rejected": -377.56018518518516,
|
|
"loss": 0.523,
|
|
"loss/base_kto": 3.7566299438476562,
|
|
"loss/total_with_kl": 4.18420934677124,
|
|
"metrics/advantage_var": 256.8043518066406,
|
|
"regularization/advantage_var": 256.8043518066406,
|
|
"regularization/kl": 0.42757925391197205,
|
|
"rewards/chosen": 0.18561133855505835,
|
|
"rewards/margins": 0.2270865403855102,
|
|
"rewards/rejected": -0.04147520183045187,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 13.450138092041016,
|
|
"kl": 6.0438127517700195,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -36760532.81927711,
|
|
"logits/rejected": -37086972.67532468,
|
|
"logps/chosen": -480.73188064759034,
|
|
"logps/rejected": -364.3080357142857,
|
|
"loss": 0.5156,
|
|
"loss/base_kto": 3.6959915161132812,
|
|
"loss/total_with_kl": 4.12469482421875,
|
|
"metrics/advantage_var": 257.4795837402344,
|
|
"regularization/advantage_var": 257.4795837402344,
|
|
"regularization/kl": 0.42870351672172546,
|
|
"rewards/chosen": 0.19070629901196584,
|
|
"rewards/margins": 0.2909384976528635,
|
|
"rewards/rejected": -0.10023219864089768,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 17.692373275756836,
|
|
"kl": 24.13968849182129,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -36702020.110091746,
|
|
"logits/rejected": -38419276.063897766,
|
|
"logps/chosen": -526.2847859327217,
|
|
"logps/rejected": -366.9308107028754,
|
|
"loss": 0.5204,
|
|
"loss/base_kto": 3.643434524536133,
|
|
"loss/total_with_kl": 4.1631999015808105,
|
|
"metrics/advantage_var": 312.1711730957031,
|
|
"regularization/advantage_var": 312.1711730957031,
|
|
"regularization/kl": 0.5197650194168091,
|
|
"rewards/chosen": 0.4845032298236812,
|
|
"rewards/margins": 0.3362895782092331,
|
|
"rewards/rejected": 0.14821365161444813,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 13.04470443725586,
|
|
"kl": 14.349547386169434,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -36293129.54037267,
|
|
"logits/rejected": -37886757.03144654,
|
|
"logps/chosen": -478.4535131987578,
|
|
"logps/rejected": -356.00395538522014,
|
|
"loss": 0.5112,
|
|
"loss/base_kto": 3.674733877182007,
|
|
"loss/total_with_kl": 4.089953899383545,
|
|
"metrics/advantage_var": 249.3812713623047,
|
|
"regularization/advantage_var": 249.3812713623047,
|
|
"regularization/kl": 0.41521984338760376,
|
|
"rewards/chosen": 0.2765871604777271,
|
|
"rewards/margins": 0.29934188064976947,
|
|
"rewards/rejected": -0.022754720172042366,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 12.400165557861328,
|
|
"kl": 6.819758892059326,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -37125177.16108453,
|
|
"logits/rejected": -38182331.7856049,
|
|
"logps/chosen": -465.477173046252,
|
|
"logps/rejected": -384.0022731623277,
|
|
"loss": 0.5239,
|
|
"loss/base_kto": 3.7515556812286377,
|
|
"loss/total_with_kl": 4.190969944000244,
|
|
"metrics/advantage_var": 263.91241455078125,
|
|
"regularization/advantage_var": 263.91241455078125,
|
|
"regularization/kl": 0.4394141137599945,
|
|
"rewards/chosen": 0.14862793066094745,
|
|
"rewards/margins": 0.24194457988321189,
|
|
"rewards/rejected": -0.09331664922226443,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 13.425152778625488,
|
|
"kl": 7.600010871887207,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -36457960.656534955,
|
|
"logits/rejected": -38494563.60128617,
|
|
"logps/chosen": -470.4335106382979,
|
|
"logps/rejected": -369.4139368971061,
|
|
"loss": 0.5208,
|
|
"loss/base_kto": 3.7777416706085205,
|
|
"loss/total_with_kl": 4.166408061981201,
|
|
"metrics/advantage_var": 233.433349609375,
|
|
"regularization/advantage_var": 233.433349609375,
|
|
"regularization/kl": 0.3886665403842926,
|
|
"rewards/chosen": 0.1351085999091708,
|
|
"rewards/margins": 0.21848600873717017,
|
|
"rewards/rejected": -0.08337740882799939,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 14.666336059570312,
|
|
"kl": 5.202136039733887,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -36578678.05619834,
|
|
"logits/rejected": -38463466.76148148,
|
|
"logps/chosen": -483.46172520661156,
|
|
"logps/rejected": -375.6144675925926,
|
|
"loss": 0.5227,
|
|
"loss/base_kto": 3.748065948486328,
|
|
"loss/total_with_kl": 4.181732177734375,
|
|
"metrics/advantage_var": 260.46014404296875,
|
|
"regularization/advantage_var": 260.46014404296875,
|
|
"regularization/kl": 0.4336661398410797,
|
|
"rewards/chosen": -0.03499351690623386,
|
|
"rewards/margins": 0.22428518318057167,
|
|
"rewards/rejected": -0.25927870008680554,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 22.678667068481445,
|
|
"kl": 7.763389587402344,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -36146883.43217666,
|
|
"logits/rejected": -38410030.761609904,
|
|
"logps/chosen": -493.2384660883281,
|
|
"logps/rejected": -359.3801035216718,
|
|
"loss": 0.5159,
|
|
"loss/base_kto": 3.710904836654663,
|
|
"loss/total_with_kl": 4.12742280960083,
|
|
"metrics/advantage_var": 250.16091918945312,
|
|
"regularization/advantage_var": 250.16091918945312,
|
|
"regularization/kl": 0.4165179431438446,
|
|
"rewards/chosen": 0.15228206502150285,
|
|
"rewards/margins": 0.27428922833797564,
|
|
"rewards/rejected": -0.12200716331647277,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 13.313922882080078,
|
|
"kl": 6.7822747230529785,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -34893658.11838006,
|
|
"logits/rejected": -36028602.18181818,
|
|
"logps/chosen": -509.363707165109,
|
|
"logps/rejected": -383.93568769592474,
|
|
"loss": 0.5176,
|
|
"loss/base_kto": 3.703699827194214,
|
|
"loss/total_with_kl": 4.141143321990967,
|
|
"metrics/advantage_var": 262.72857666015625,
|
|
"regularization/advantage_var": 262.72857666015625,
|
|
"regularization/kl": 0.4374430775642395,
|
|
"rewards/chosen": 0.10898092157001436,
|
|
"rewards/margins": 0.273612588752004,
|
|
"rewards/rejected": -0.16463166718198963,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 19.352764129638672,
|
|
"kl": 5.236758708953857,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -36377299.60453809,
|
|
"logits/rejected": -37695037.00754148,
|
|
"logps/chosen": -467.047102917342,
|
|
"logps/rejected": -380.1214177978884,
|
|
"loss": 0.5212,
|
|
"loss/base_kto": 3.7206857204437256,
|
|
"loss/total_with_kl": 4.169682025909424,
|
|
"metrics/advantage_var": 269.667724609375,
|
|
"regularization/advantage_var": 269.667724609375,
|
|
"regularization/kl": 0.44899675250053406,
|
|
"rewards/chosen": -0.0003839505163158744,
|
|
"rewards/margins": 0.2590480984826707,
|
|
"rewards/rejected": -0.2594320489989866,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 15.895614624023438,
|
|
"kl": 11.7037992477417,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -36959899.15151515,
|
|
"logits/rejected": -36991963.66451613,
|
|
"logps/chosen": -480.2222537878788,
|
|
"logps/rejected": -399.8467741935484,
|
|
"loss": 0.5338,
|
|
"loss/base_kto": 3.7008800506591797,
|
|
"loss/total_with_kl": 4.270693302154541,
|
|
"metrics/advantage_var": 342.22998046875,
|
|
"regularization/advantage_var": 342.22998046875,
|
|
"regularization/kl": 0.5698128938674927,
|
|
"rewards/chosen": 0.23761749267578125,
|
|
"rewards/margins": 0.2842864990234375,
|
|
"rewards/rejected": -0.04666900634765625,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 17.784793853759766,
|
|
"kl": 6.9153032302856445,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -36415484.61487603,
|
|
"logits/rejected": -36975347.48444445,
|
|
"logps/chosen": -482.83212809917353,
|
|
"logps/rejected": -383.46078703703705,
|
|
"loss": 0.5196,
|
|
"loss/base_kto": 3.6761536598205566,
|
|
"loss/total_with_kl": 4.156997203826904,
|
|
"metrics/advantage_var": 288.7945861816406,
|
|
"regularization/advantage_var": 288.7945861816406,
|
|
"regularization/kl": 0.4808429777622223,
|
|
"rewards/chosen": -0.010790485981082128,
|
|
"rewards/margins": 0.3027459361832697,
|
|
"rewards/rejected": -0.31353642216435185,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 16.268333435058594,
|
|
"kl": 3.050342321395874,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -36121260.8,
|
|
"logits/rejected": -37083052.8,
|
|
"logps/chosen": -513.03330078125,
|
|
"logps/rejected": -379.258740234375,
|
|
"loss": 0.532,
|
|
"loss/base_kto": 3.7237770557403564,
|
|
"loss/total_with_kl": 4.256200313568115,
|
|
"metrics/advantage_var": 319.773681640625,
|
|
"regularization/advantage_var": 319.773681640625,
|
|
"regularization/kl": 0.532423198223114,
|
|
"rewards/chosen": -0.14262712001800537,
|
|
"rewards/margins": 0.2905952215194702,
|
|
"rewards/rejected": -0.4332223415374756,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 14.93693733215332,
|
|
"kl": 4.392341136932373,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -36505365.19745223,
|
|
"logits/rejected": -36451726.9202454,
|
|
"logps/chosen": -517.0012440286624,
|
|
"logps/rejected": -398.4025594325153,
|
|
"loss": 0.5147,
|
|
"loss/base_kto": 3.651310682296753,
|
|
"loss/total_with_kl": 4.1177496910095215,
|
|
"metrics/advantage_var": 280.1435241699219,
|
|
"regularization/advantage_var": 280.1435241699219,
|
|
"regularization/kl": 0.46643900871276855,
|
|
"rewards/chosen": 0.025168209318902082,
|
|
"rewards/margins": 0.3317868822573367,
|
|
"rewards/rejected": -0.30661867293843464,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 12.531059265136719,
|
|
"kl": 6.63722562789917,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -36253532.66257669,
|
|
"logits/rejected": -37696313.070063695,
|
|
"logps/chosen": -491.13266871165644,
|
|
"logps/rejected": -369.7501492834395,
|
|
"loss": 0.5167,
|
|
"loss/base_kto": 3.648242235183716,
|
|
"loss/total_with_kl": 4.133980751037598,
|
|
"metrics/advantage_var": 291.7347106933594,
|
|
"regularization/advantage_var": 291.7347106933594,
|
|
"regularization/kl": 0.4857383370399475,
|
|
"rewards/chosen": 0.10919229238311205,
|
|
"rewards/margins": 0.3440026052252199,
|
|
"rewards/rejected": -0.23481031284210788,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 12.516692161560059,
|
|
"kl": 20.211999893188477,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -34461282.171091445,
|
|
"logits/rejected": -37180133.634551495,
|
|
"logps/chosen": -509.4652931415929,
|
|
"logps/rejected": -348.75534676079735,
|
|
"loss": 0.5067,
|
|
"loss/base_kto": 3.613668918609619,
|
|
"loss/total_with_kl": 4.0532965660095215,
|
|
"metrics/advantage_var": 264.040771484375,
|
|
"regularization/advantage_var": 264.040771484375,
|
|
"regularization/kl": 0.43962788581848145,
|
|
"rewards/chosen": 0.4538445486783278,
|
|
"rewards/margins": 0.3372245007383556,
|
|
"rewards/rejected": 0.11662004793997223,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 17.768901824951172,
|
|
"kl": 21.419124603271484,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -34868115.61453397,
|
|
"logits/rejected": -37534750.071097374,
|
|
"logps/chosen": -483.51135466034754,
|
|
"logps/rejected": -345.60717735703247,
|
|
"loss": 0.5122,
|
|
"loss/base_kto": 3.655799150466919,
|
|
"loss/total_with_kl": 4.097526550292969,
|
|
"metrics/advantage_var": 265.30169677734375,
|
|
"regularization/advantage_var": 265.30169677734375,
|
|
"regularization/kl": 0.44172731041908264,
|
|
"rewards/chosen": 0.389554136737263,
|
|
"rewards/margins": 0.28585758778008685,
|
|
"rewards/rejected": 0.10369654895717614,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 17.838653564453125,
|
|
"kl": 15.633729934692383,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -36619463.64779874,
|
|
"logits/rejected": -38321676.72049689,
|
|
"logps/chosen": -479.3098958333333,
|
|
"logps/rejected": -387.63936335403724,
|
|
"loss": 0.5083,
|
|
"loss/base_kto": 3.6335079669952393,
|
|
"loss/total_with_kl": 4.066558361053467,
|
|
"metrics/advantage_var": 260.09027099609375,
|
|
"regularization/advantage_var": 260.09027099609375,
|
|
"regularization/kl": 0.4330502450466156,
|
|
"rewards/chosen": 0.2864372685270489,
|
|
"rewards/margins": 0.33113921201342256,
|
|
"rewards/rejected": -0.044701943486373616,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 12.624896049499512,
|
|
"kl": 11.278129577636719,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -37027584.797507785,
|
|
"logits/rejected": -38227454.39498433,
|
|
"logps/chosen": -480.63843457943926,
|
|
"logps/rejected": -366.7788989028213,
|
|
"loss": 0.5114,
|
|
"loss/base_kto": 3.6688125133514404,
|
|
"loss/total_with_kl": 4.090945720672607,
|
|
"metrics/advantage_var": 253.5330047607422,
|
|
"regularization/advantage_var": 253.5330047607422,
|
|
"regularization/kl": 0.4221324622631073,
|
|
"rewards/chosen": 0.1828555526020371,
|
|
"rewards/margins": 0.3193783897954454,
|
|
"rewards/rejected": -0.13652283719340835,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 17.730945587158203,
|
|
"kl": 14.693155288696289,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -37004197.888,
|
|
"logits/rejected": -37275582.3389313,
|
|
"logps/chosen": -494.2066,
|
|
"logps/rejected": -365.50939885496183,
|
|
"loss": 0.5077,
|
|
"loss/base_kto": 3.5983726978302,
|
|
"loss/total_with_kl": 4.061243534088135,
|
|
"metrics/advantage_var": 278.00042724609375,
|
|
"regularization/advantage_var": 278.00042724609375,
|
|
"regularization/kl": 0.46287068724632263,
|
|
"rewards/chosen": 0.289713916015625,
|
|
"rewards/margins": 0.3801908824658576,
|
|
"rewards/rejected": -0.09047696645023258,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 13.537254333496094,
|
|
"kl": 10.964723587036133,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -37419458.496099845,
|
|
"logits/rejected": -37086187.16744914,
|
|
"logps/chosen": -462.9302847113885,
|
|
"logps/rejected": -386.6461023082942,
|
|
"loss": 0.5228,
|
|
"loss/base_kto": 3.738948106765747,
|
|
"loss/total_with_kl": 4.182588577270508,
|
|
"metrics/advantage_var": 266.4507751464844,
|
|
"regularization/advantage_var": 266.4507751464844,
|
|
"regularization/kl": 0.44364047050476074,
|
|
"rewards/chosen": 0.15449582247205904,
|
|
"rewards/margins": 0.23441620168090588,
|
|
"rewards/rejected": -0.07992037920884683,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 14.72453784942627,
|
|
"kl": 17.46700668334961,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -36471108.37267081,
|
|
"logits/rejected": -36829097.056603774,
|
|
"logps/chosen": -475.0952057453416,
|
|
"logps/rejected": -380.38630110062894,
|
|
"loss": 0.5079,
|
|
"loss/base_kto": 3.6252617835998535,
|
|
"loss/total_with_kl": 4.063351631164551,
|
|
"metrics/advantage_var": 263.1170349121094,
|
|
"regularization/advantage_var": 263.1170349121094,
|
|
"regularization/kl": 0.43808984756469727,
|
|
"rewards/chosen": 0.3141225258015698,
|
|
"rewards/margins": 0.329195285393878,
|
|
"rewards/rejected": -0.015072759592308188,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 17.89183807373047,
|
|
"kl": 19.215808868408203,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -37998084.93890675,
|
|
"logits/rejected": -39071149.51975684,
|
|
"logps/chosen": -484.5552652733119,
|
|
"logps/rejected": -366.13658814589667,
|
|
"loss": 0.5161,
|
|
"loss/base_kto": 3.643170118331909,
|
|
"loss/total_with_kl": 4.129195213317871,
|
|
"metrics/advantage_var": 291.90667724609375,
|
|
"regularization/advantage_var": 291.90667724609375,
|
|
"regularization/kl": 0.4860245883464813,
|
|
"rewards/chosen": 0.31148678696807175,
|
|
"rewards/margins": 0.31004061378424796,
|
|
"rewards/rejected": 0.0014461731838237913,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 13.756281852722168,
|
|
"kl": 16.016250610351562,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -37853036.5941807,
|
|
"logits/rejected": -38387267.77671451,
|
|
"logps/chosen": -479.4279287901991,
|
|
"logps/rejected": -370.8888058213716,
|
|
"loss": 0.5116,
|
|
"loss/base_kto": 3.6113240718841553,
|
|
"loss/total_with_kl": 4.092836856842041,
|
|
"metrics/advantage_var": 289.1969299316406,
|
|
"regularization/advantage_var": 289.1969299316406,
|
|
"regularization/kl": 0.4815129339694977,
|
|
"rewards/chosen": 0.30620667251657013,
|
|
"rewards/margins": 0.34496932027616134,
|
|
"rewards/rejected": -0.038762647759591186,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 10.585813522338867,
|
|
"kl": 23.239669799804688,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -36298412.7607362,
|
|
"logits/rejected": -36672932.68789809,
|
|
"logps/chosen": -467.09950153374234,
|
|
"logps/rejected": -380.66510748407643,
|
|
"loss": 0.5165,
|
|
"loss/base_kto": 3.685466766357422,
|
|
"loss/total_with_kl": 4.1322503089904785,
|
|
"metrics/advantage_var": 268.3381652832031,
|
|
"regularization/advantage_var": 268.3381652832031,
|
|
"regularization/kl": 0.44678306579589844,
|
|
"rewards/chosen": 0.3398432351328844,
|
|
"rewards/margins": 0.25556592418442137,
|
|
"rewards/rejected": 0.084277310948463,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 15.321822166442871,
|
|
"kl": 24.060562133789062,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -36477377.00946372,
|
|
"logits/rejected": -38850021.05263158,
|
|
"logps/chosen": -498.72032728706625,
|
|
"logps/rejected": -345.2563128869969,
|
|
"loss": 0.5066,
|
|
"loss/base_kto": 3.6295089721679688,
|
|
"loss/total_with_kl": 4.053165435791016,
|
|
"metrics/advantage_var": 254.447998046875,
|
|
"regularization/advantage_var": 254.447998046875,
|
|
"regularization/kl": 0.4236558973789215,
|
|
"rewards/chosen": 0.42650529638825907,
|
|
"rewards/margins": 0.30403308250141936,
|
|
"rewards/rejected": 0.12247221388683968,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 17.947206497192383,
|
|
"kl": 16.620363235473633,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -35866036.61349693,
|
|
"logits/rejected": -37016876.02547771,
|
|
"logps/chosen": -466.13113496932516,
|
|
"logps/rejected": -372.83399681528664,
|
|
"loss": 0.5134,
|
|
"loss/base_kto": 3.6483466625213623,
|
|
"loss/total_with_kl": 4.107553482055664,
|
|
"metrics/advantage_var": 275.79986572265625,
|
|
"regularization/advantage_var": 275.79986572265625,
|
|
"regularization/kl": 0.45920678973197937,
|
|
"rewards/chosen": 0.28322280696564656,
|
|
"rewards/margins": 0.30739223529900195,
|
|
"rewards/rejected": -0.024169428333355364,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 459.1017761230469,
|
|
"kl": 19.47315788269043,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -36016873.32689211,
|
|
"logits/rejected": -37386255.585996956,
|
|
"logps/chosen": -440.35899758454104,
|
|
"logps/rejected": -350.07453386605783,
|
|
"loss": 0.7386,
|
|
"loss/base_kto": 3.484525442123413,
|
|
"loss/total_with_kl": 5.908759593963623,
|
|
"metrics/advantage_var": 1455.99658203125,
|
|
"regularization/advantage_var": 1455.99658203125,
|
|
"regularization/kl": 2.424234390258789,
|
|
"rewards/chosen": 0.42956891881478965,
|
|
"rewards/margins": 0.5956617899828595,
|
|
"rewards/rejected": -0.1660928711680698,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 15.112618446350098,
|
|
"kl": 16.712034225463867,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -37949446.57463884,
|
|
"logits/rejected": -37887629.05327245,
|
|
"logps/chosen": -477.8843298555377,
|
|
"logps/rejected": -374.05260654490104,
|
|
"loss": 0.5371,
|
|
"loss/base_kto": 3.203089952468872,
|
|
"loss/total_with_kl": 4.2968573570251465,
|
|
"metrics/advantage_var": 656.9172973632812,
|
|
"regularization/advantage_var": 656.9172973632812,
|
|
"regularization/kl": 1.0937672853469849,
|
|
"rewards/chosen": 0.5266771637991573,
|
|
"rewards/margins": 0.8739736129465603,
|
|
"rewards/rejected": -0.347296449147403,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 16.0929012298584,
|
|
"kl": 14.954050064086914,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -35607803.16981132,
|
|
"logits/rejected": -36990912.39751553,
|
|
"logps/chosen": -480.3087657232704,
|
|
"logps/rejected": -373.5547360248447,
|
|
"loss": 0.4954,
|
|
"loss/base_kto": 3.169015645980835,
|
|
"loss/total_with_kl": 3.9630134105682373,
|
|
"metrics/advantage_var": 476.8757019042969,
|
|
"regularization/advantage_var": 476.8757019042969,
|
|
"regularization/kl": 0.7939980626106262,
|
|
"rewards/chosen": 0.5190320764697572,
|
|
"rewards/margins": 0.8666407967634538,
|
|
"rewards/rejected": -0.3476087202936966,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 33.52922439575195,
|
|
"kl": 11.597731590270996,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -35957991.22580645,
|
|
"logits/rejected": -37399228.0317965,
|
|
"logps/chosen": -482.7825460829493,
|
|
"logps/rejected": -371.5338831478537,
|
|
"loss": 0.5432,
|
|
"loss/base_kto": 3.100201368331909,
|
|
"loss/total_with_kl": 4.345668315887451,
|
|
"metrics/advantage_var": 748.0279541015625,
|
|
"regularization/advantage_var": 748.0279541015625,
|
|
"regularization/kl": 1.2454664707183838,
|
|
"rewards/chosen": 0.6155564268613192,
|
|
"rewards/margins": 1.01636304186902,
|
|
"rewards/rejected": -0.4008066150077007,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 26.115161895751953,
|
|
"kl": 13.430798530578613,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -36929491.33956386,
|
|
"logits/rejected": -37107811.510971785,
|
|
"logps/chosen": -478.0388434579439,
|
|
"logps/rejected": -389.3616036442006,
|
|
"loss": 0.5142,
|
|
"loss/base_kto": 3.125993013381958,
|
|
"loss/total_with_kl": 4.113446235656738,
|
|
"metrics/advantage_var": 593.0652465820312,
|
|
"regularization/advantage_var": 593.0652465820312,
|
|
"regularization/kl": 0.9874534606933594,
|
|
"rewards/chosen": 0.5246434672227901,
|
|
"rewards/margins": 0.9730896154643653,
|
|
"rewards/rejected": -0.44844614824157525,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 16.133544921875,
|
|
"kl": 25.103172302246094,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -35453747.83800623,
|
|
"logits/rejected": -38156910.7460815,
|
|
"logps/chosen": -462.5388434579439,
|
|
"logps/rejected": -375.6218896943574,
|
|
"loss": 0.5507,
|
|
"loss/base_kto": 3.150460720062256,
|
|
"loss/total_with_kl": 4.40519905090332,
|
|
"metrics/advantage_var": 753.5966186523438,
|
|
"regularization/advantage_var": 753.5966186523438,
|
|
"regularization/kl": 1.254738450050354,
|
|
"rewards/chosen": 0.7627866275585329,
|
|
"rewards/margins": 0.8761208767380979,
|
|
"rewards/rejected": -0.11333424917956504,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 25.691125869750977,
|
|
"kl": 21.39615821838379,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -35257481.87594554,
|
|
"logits/rejected": -36158283.683360256,
|
|
"logps/chosen": -483.1625378214826,
|
|
"logps/rejected": -367.0164832390953,
|
|
"loss": 0.5266,
|
|
"loss/base_kto": 3.1581482887268066,
|
|
"loss/total_with_kl": 4.212549686431885,
|
|
"metrics/advantage_var": 633.2742309570312,
|
|
"regularization/advantage_var": 633.2742309570312,
|
|
"regularization/kl": 1.0544016361236572,
|
|
"rewards/chosen": 0.6843446855646038,
|
|
"rewards/margins": 0.9018967596950057,
|
|
"rewards/rejected": -0.21755207413040187,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 13.309680938720703,
|
|
"kl": 26.730859756469727,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -35458344.789074354,
|
|
"logits/rejected": -36895755.54267311,
|
|
"logps/chosen": -480.3571699544765,
|
|
"logps/rejected": -370.9915710547504,
|
|
"loss": 0.5117,
|
|
"loss/base_kto": 3.118666172027588,
|
|
"loss/total_with_kl": 4.093329906463623,
|
|
"metrics/advantage_var": 585.3839721679688,
|
|
"regularization/advantage_var": 585.3839721679688,
|
|
"regularization/kl": 0.9746643304824829,
|
|
"rewards/chosen": 0.8353993903523331,
|
|
"rewards/margins": 0.9118079915818427,
|
|
"rewards/rejected": -0.07640860122950949,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 17.09473991394043,
|
|
"kl": 20.014665603637695,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -36031626.19837398,
|
|
"logits/rejected": -37941383.506766915,
|
|
"logps/chosen": -502.64502032520323,
|
|
"logps/rejected": -370.1692434210526,
|
|
"loss": 0.5159,
|
|
"loss/base_kto": 3.173341751098633,
|
|
"loss/total_with_kl": 4.127114772796631,
|
|
"metrics/advantage_var": 572.8365478515625,
|
|
"regularization/advantage_var": 572.8365478515625,
|
|
"regularization/kl": 0.9537727236747742,
|
|
"rewards/chosen": 0.6165187928734757,
|
|
"rewards/margins": 0.9000808091813352,
|
|
"rewards/rejected": -0.2835620163078595,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 12.381759643554688,
|
|
"kl": 16.968212127685547,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -35665446.419601835,
|
|
"logits/rejected": -36283135.59170654,
|
|
"logps/chosen": -464.3429843032159,
|
|
"logps/rejected": -361.9744318181818,
|
|
"loss": 0.5261,
|
|
"loss/base_kto": 3.1213691234588623,
|
|
"loss/total_with_kl": 4.2086501121521,
|
|
"metrics/advantage_var": 653.021484375,
|
|
"regularization/advantage_var": 653.021484375,
|
|
"regularization/kl": 1.0872806310653687,
|
|
"rewards/chosen": 0.7053009079937548,
|
|
"rewards/margins": 0.9547515915875048,
|
|
"rewards/rejected": -0.24945068359375,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 13.494943618774414,
|
|
"kl": 19.32745933532715,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -35171909.10429448,
|
|
"logits/rejected": -36244995.26114649,
|
|
"logps/chosen": -493.63247699386505,
|
|
"logps/rejected": -384.27149681528664,
|
|
"loss": 0.5164,
|
|
"loss/base_kto": 3.1353840827941895,
|
|
"loss/total_with_kl": 4.131186485290527,
|
|
"metrics/advantage_var": 598.0792846679688,
|
|
"regularization/advantage_var": 598.0792846679688,
|
|
"regularization/kl": 0.9958020448684692,
|
|
"rewards/chosen": 0.7007261873022911,
|
|
"rewards/margins": 0.9272444416734968,
|
|
"rewards/rejected": -0.2265182543712057,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 15.036930084228516,
|
|
"kl": 17.484758377075195,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -34972631.81161696,
|
|
"logits/rejected": -36632652.44167963,
|
|
"logps/chosen": -489.1505592621664,
|
|
"logps/rejected": -392.01404548989115,
|
|
"loss": 0.532,
|
|
"loss/base_kto": 3.1289374828338623,
|
|
"loss/total_with_kl": 4.255650520324707,
|
|
"metrics/advantage_var": 676.7042236328125,
|
|
"regularization/advantage_var": 676.7042236328125,
|
|
"regularization/kl": 1.1267125606536865,
|
|
"rewards/chosen": 0.6656846266526443,
|
|
"rewards/margins": 0.9405176057985545,
|
|
"rewards/rejected": -0.27483297914591026,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 20.474241256713867,
|
|
"kl": 17.55146598815918,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -36360107.34499205,
|
|
"logits/rejected": -36947744.6390169,
|
|
"logps/chosen": -500.82571542130364,
|
|
"logps/rejected": -370.28729838709677,
|
|
"loss": 0.5275,
|
|
"loss/base_kto": 3.1530396938323975,
|
|
"loss/total_with_kl": 4.219779014587402,
|
|
"metrics/advantage_var": 640.68408203125,
|
|
"regularization/advantage_var": 640.68408203125,
|
|
"regularization/kl": 1.0667389631271362,
|
|
"rewards/chosen": 0.6676064689890948,
|
|
"rewards/margins": 0.9096627105885572,
|
|
"rewards/rejected": -0.24205624159946237,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 25.762794494628906,
|
|
"kl": 13.425375938415527,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -34769629.27444795,
|
|
"logits/rejected": -37366346.501547985,
|
|
"logps/chosen": -473.2747436908517,
|
|
"logps/rejected": -385.3550212848297,
|
|
"loss": 0.5154,
|
|
"loss/base_kto": 3.161378860473633,
|
|
"loss/total_with_kl": 4.123094081878662,
|
|
"metrics/advantage_var": 577.6064453125,
|
|
"regularization/advantage_var": 577.6064453125,
|
|
"regularization/kl": 0.9617147445678711,
|
|
"rewards/chosen": 0.5202091830762027,
|
|
"rewards/margins": 0.8678770543211234,
|
|
"rewards/rejected": -0.34766787124492066,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 18.46282958984375,
|
|
"kl": 20.11356544494629,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -34801996.63694268,
|
|
"logits/rejected": -36689806.9202454,
|
|
"logps/chosen": -471.41147492038215,
|
|
"logps/rejected": -363.62940950920245,
|
|
"loss": 0.5153,
|
|
"loss/base_kto": 3.1584091186523438,
|
|
"loss/total_with_kl": 4.122684478759766,
|
|
"metrics/advantage_var": 579.1444091796875,
|
|
"regularization/advantage_var": 579.1444091796875,
|
|
"regularization/kl": 0.9642754793167114,
|
|
"rewards/chosen": 0.6112483808189441,
|
|
"rewards/margins": 0.8804255426617116,
|
|
"rewards/rejected": -0.26917716184276747,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 19.694732666015625,
|
|
"kl": 14.28063678741455,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -34455708.59370315,
|
|
"logits/rejected": -36949474.76672105,
|
|
"logps/chosen": -458.2426443028486,
|
|
"logps/rejected": -374.86755709624794,
|
|
"loss": 0.511,
|
|
"loss/base_kto": 3.243755340576172,
|
|
"loss/total_with_kl": 4.088174343109131,
|
|
"metrics/advantage_var": 507.15875244140625,
|
|
"regularization/advantage_var": 507.15875244140625,
|
|
"regularization/kl": 0.8444193005561829,
|
|
"rewards/chosen": 0.5739662822397396,
|
|
"rewards/margins": 0.8146776126348889,
|
|
"rewards/rejected": -0.24071133039514936,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 14.32904052734375,
|
|
"kl": 27.570173263549805,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -35039963.202472955,
|
|
"logits/rejected": -36363459.740916274,
|
|
"logps/chosen": -499.9491885625966,
|
|
"logps/rejected": -394.2331161137441,
|
|
"loss": 0.5221,
|
|
"loss/base_kto": 3.1464409828186035,
|
|
"loss/total_with_kl": 4.176943302154541,
|
|
"metrics/advantage_var": 618.92041015625,
|
|
"regularization/advantage_var": 618.92041015625,
|
|
"regularization/kl": 1.030502438545227,
|
|
"rewards/chosen": 0.7834836791803517,
|
|
"rewards/margins": 0.9163908174202559,
|
|
"rewards/rejected": -0.13290713823990422,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 17.497726440429688,
|
|
"kl": 21.241546630859375,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -35421142.4,
|
|
"logits/rejected": -34407510.4,
|
|
"logps/chosen": -507.23505859375,
|
|
"logps/rejected": -381.8780517578125,
|
|
"loss": 0.505,
|
|
"loss/base_kto": 3.1952483654022217,
|
|
"loss/total_with_kl": 4.0400710105896,
|
|
"metrics/advantage_var": 507.4008483886719,
|
|
"regularization/advantage_var": 507.4008483886719,
|
|
"regularization/kl": 0.844822347164154,
|
|
"rewards/chosen": 0.62314772605896,
|
|
"rewards/margins": 0.827998161315918,
|
|
"rewards/rejected": -0.204850435256958,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 15.432272911071777,
|
|
"kl": 19.388975143432617,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -35294148.707355246,
|
|
"logits/rejected": -36516804.892355695,
|
|
"logps/chosen": -464.58939749608766,
|
|
"logps/rejected": -374.42921216848674,
|
|
"loss": 0.525,
|
|
"loss/base_kto": 3.2198193073272705,
|
|
"loss/total_with_kl": 4.199686527252197,
|
|
"metrics/advantage_var": 588.5087280273438,
|
|
"regularization/advantage_var": 588.5087280273438,
|
|
"regularization/kl": 0.9798671007156372,
|
|
"rewards/chosen": 0.6397384142092136,
|
|
"rewards/margins": 0.8534422779155197,
|
|
"rewards/rejected": -0.21370386370630606,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 19.87433624267578,
|
|
"kl": 15.814824104309082,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -35284534.0621118,
|
|
"logits/rejected": -36277512.05031446,
|
|
"logps/chosen": -463.0890916149068,
|
|
"logps/rejected": -383.2644948899371,
|
|
"loss": 0.508,
|
|
"loss/base_kto": 3.209702253341675,
|
|
"loss/total_with_kl": 4.06430196762085,
|
|
"metrics/advantage_var": 513.27294921875,
|
|
"regularization/advantage_var": 513.27294921875,
|
|
"regularization/kl": 0.8545995950698853,
|
|
"rewards/chosen": 0.517004072295953,
|
|
"rewards/margins": 0.8222768524204493,
|
|
"rewards/rejected": -0.3052727801244964,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 24.365325927734375,
|
|
"kl": 17.4028377532959,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -35636988.3943662,
|
|
"logits/rejected": -37049467.00780031,
|
|
"logps/chosen": -490.056338028169,
|
|
"logps/rejected": -376.13830928237127,
|
|
"loss": 0.5271,
|
|
"loss/base_kto": 3.215531587600708,
|
|
"loss/total_with_kl": 4.216752052307129,
|
|
"metrics/advantage_var": 601.33349609375,
|
|
"regularization/advantage_var": 601.33349609375,
|
|
"regularization/kl": 1.0012203454971313,
|
|
"rewards/chosen": 0.5895771629560348,
|
|
"rewards/margins": 0.8474491746138448,
|
|
"rewards/rejected": -0.25787201165781004,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 15.083908081054688,
|
|
"kl": 20.98103141784668,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -36386649.521472394,
|
|
"logits/rejected": -37119908.68789809,
|
|
"logps/chosen": -484.9796779141104,
|
|
"logps/rejected": -388.6567227308917,
|
|
"loss": 0.5327,
|
|
"loss/base_kto": 3.1443984508514404,
|
|
"loss/total_with_kl": 4.261448860168457,
|
|
"metrics/advantage_var": 670.9013061523438,
|
|
"regularization/advantage_var": 670.9013061523438,
|
|
"regularization/kl": 1.1170505285263062,
|
|
"rewards/chosen": 0.7250059162912194,
|
|
"rewards/margins": 0.9323071107919907,
|
|
"rewards/rejected": -0.2073011945007713,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 28.53917121887207,
|
|
"kl": 15.016886711120605,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -34886625.835658915,
|
|
"logits/rejected": -36964135.911811024,
|
|
"logps/chosen": -477.06671511627906,
|
|
"logps/rejected": -370.5987942913386,
|
|
"loss": 0.4977,
|
|
"loss/base_kto": 3.180560827255249,
|
|
"loss/total_with_kl": 3.981778860092163,
|
|
"metrics/advantage_var": 481.2121276855469,
|
|
"regularization/advantage_var": 481.2121276855469,
|
|
"regularization/kl": 0.8012181520462036,
|
|
"rewards/chosen": 0.6110617940740067,
|
|
"rewards/margins": 0.8825826940494004,
|
|
"rewards/rejected": -0.2715208999753937,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 18.6873779296875,
|
|
"kl": 15.860907554626465,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -35441551.25188537,
|
|
"logits/rejected": -36986567.98703404,
|
|
"logps/chosen": -488.82579185520365,
|
|
"logps/rejected": -361.5347953808752,
|
|
"loss": 0.5315,
|
|
"loss/base_kto": 3.235243558883667,
|
|
"loss/total_with_kl": 4.252362251281738,
|
|
"metrics/advantage_var": 610.8818359375,
|
|
"regularization/advantage_var": 610.8818359375,
|
|
"regularization/kl": 1.0171183347702026,
|
|
"rewards/chosen": 0.6360747508513622,
|
|
"rewards/margins": 0.7943206350835973,
|
|
"rewards/rejected": -0.1582458842322351,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 15.02067756652832,
|
|
"kl": 21.746383666992188,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -34904363.94330709,
|
|
"logits/rejected": -35648215.11937985,
|
|
"logps/chosen": -464.41087598425196,
|
|
"logps/rejected": -377.1945736434109,
|
|
"loss": 0.5064,
|
|
"loss/base_kto": 3.1952991485595703,
|
|
"loss/total_with_kl": 4.050997734069824,
|
|
"metrics/advantage_var": 513.9329223632812,
|
|
"regularization/advantage_var": 513.9329223632812,
|
|
"regularization/kl": 0.85569828748703,
|
|
"rewards/chosen": 0.6702821265994094,
|
|
"rewards/margins": 0.8468919879880965,
|
|
"rewards/rejected": -0.176609861388687,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 17.969749450683594,
|
|
"kl": 24.225019454956055,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -35991219.80472441,
|
|
"logits/rejected": -37809683.84496124,
|
|
"logps/chosen": -461.95132874015746,
|
|
"logps/rejected": -383.9342054263566,
|
|
"loss": 0.5032,
|
|
"loss/base_kto": 3.1875898838043213,
|
|
"loss/total_with_kl": 4.025256156921387,
|
|
"metrics/advantage_var": 503.1027526855469,
|
|
"regularization/advantage_var": 503.1027526855469,
|
|
"regularization/kl": 0.8376660346984863,
|
|
"rewards/chosen": 0.6945979561392717,
|
|
"rewards/margins": 0.8265683129252455,
|
|
"rewards/rejected": -0.13197035678597382,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 15.879536628723145,
|
|
"kl": 15.6066312789917,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -34675608.16224649,
|
|
"logits/rejected": -37605270.234741785,
|
|
"logps/chosen": -485.176872074883,
|
|
"logps/rejected": -378.03501564945225,
|
|
"loss": 0.5316,
|
|
"loss/base_kto": 3.154336452484131,
|
|
"loss/total_with_kl": 4.252406597137451,
|
|
"metrics/advantage_var": 659.5015869140625,
|
|
"regularization/advantage_var": 659.5015869140625,
|
|
"regularization/kl": 1.0980701446533203,
|
|
"rewards/chosen": 0.5974360092567765,
|
|
"rewards/margins": 0.9344268194063021,
|
|
"rewards/rejected": -0.33699081014952564,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 15.690549850463867,
|
|
"kl": 18.53376579284668,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -34943416.43010753,
|
|
"logits/rejected": -37584742.96979332,
|
|
"logps/chosen": -499.8222926267281,
|
|
"logps/rejected": -349.464626391097,
|
|
"loss": 0.5107,
|
|
"loss/base_kto": 3.266888380050659,
|
|
"loss/total_with_kl": 4.085919380187988,
|
|
"metrics/advantage_var": 491.91046142578125,
|
|
"regularization/advantage_var": 491.91046142578125,
|
|
"regularization/kl": 0.8190308809280396,
|
|
"rewards/chosen": 0.5777973227786578,
|
|
"rewards/margins": 0.7850164087146052,
|
|
"rewards/rejected": -0.20721908593594743,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 15.495351791381836,
|
|
"kl": 17.53782081604004,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -35821094.89612403,
|
|
"logits/rejected": -37500292.63622047,
|
|
"logps/chosen": -495.1854651162791,
|
|
"logps/rejected": -394.68700787401576,
|
|
"loss": 0.5088,
|
|
"loss/base_kto": 3.1480536460876465,
|
|
"loss/total_with_kl": 4.070526599884033,
|
|
"metrics/advantage_var": 554.0377197265625,
|
|
"regularization/advantage_var": 554.0377197265625,
|
|
"regularization/kl": 0.9224728941917419,
|
|
"rewards/chosen": 0.6472693628118944,
|
|
"rewards/margins": 0.8807732181191655,
|
|
"rewards/rejected": -0.23350385530727116,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 13.229820251464844,
|
|
"kl": 16.59683609008789,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -34546465.25195618,
|
|
"logits/rejected": -36228953.85959438,
|
|
"logps/chosen": -480.75821596244134,
|
|
"logps/rejected": -363.6736056942278,
|
|
"loss": 0.5108,
|
|
"loss/base_kto": 3.221752166748047,
|
|
"loss/total_with_kl": 4.086036682128906,
|
|
"metrics/advantage_var": 519.0896606445312,
|
|
"regularization/advantage_var": 519.0896606445312,
|
|
"regularization/kl": 0.8642842173576355,
|
|
"rewards/chosen": 0.5837155433141383,
|
|
"rewards/margins": 0.8153934749821863,
|
|
"rewards/rejected": -0.23167793166804798,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 12.983701705932617,
|
|
"kl": 13.62908935546875,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -35417064.54961832,
|
|
"logits/rejected": -38027205.0176,
|
|
"logps/chosen": -503.6783396946565,
|
|
"logps/rejected": -385.09245,
|
|
"loss": 0.5277,
|
|
"loss/base_kto": 3.24162220954895,
|
|
"loss/total_with_kl": 4.2219719886779785,
|
|
"metrics/advantage_var": 588.7986450195312,
|
|
"regularization/advantage_var": 588.7986450195312,
|
|
"regularization/kl": 0.9803497195243835,
|
|
"rewards/chosen": 0.5724618693344465,
|
|
"rewards/margins": 0.8077351115219465,
|
|
"rewards/rejected": -0.2352732421875,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 18.497446060180664,
|
|
"kl": 17.117921829223633,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -36815440.58505564,
|
|
"logits/rejected": -38032508.26420891,
|
|
"logps/chosen": -471.76008545310015,
|
|
"logps/rejected": -382.54637096774195,
|
|
"loss": 0.5189,
|
|
"loss/base_kto": 3.233093738555908,
|
|
"loss/total_with_kl": 4.151073455810547,
|
|
"metrics/advantage_var": 551.33935546875,
|
|
"regularization/advantage_var": 551.33935546875,
|
|
"regularization/kl": 0.9179800152778625,
|
|
"rewards/chosen": 0.5842545210651828,
|
|
"rewards/margins": 0.8223942698216032,
|
|
"rewards/rejected": -0.2381397487564204,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 14.194183349609375,
|
|
"kl": 12.717142105102539,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -36926536.112676054,
|
|
"logits/rejected": -37216898.19656786,
|
|
"logps/chosen": -477.30017605633805,
|
|
"logps/rejected": -376.520646450858,
|
|
"loss": 0.5011,
|
|
"loss/base_kto": 3.1532018184661865,
|
|
"loss/total_with_kl": 4.0084452629089355,
|
|
"metrics/advantage_var": 513.6597290039062,
|
|
"regularization/advantage_var": 513.6597290039062,
|
|
"regularization/kl": 0.8552435040473938,
|
|
"rewards/chosen": 0.5345801889243447,
|
|
"rewards/margins": 0.8961057974096216,
|
|
"rewards/rejected": -0.36152560848527693,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 13.919219017028809,
|
|
"kl": 12.122225761413574,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -35699408.0,
|
|
"logits/rejected": -36755974.4,
|
|
"logps/chosen": -514.2568359375,
|
|
"logps/rejected": -359.650341796875,
|
|
"loss": 0.5017,
|
|
"loss/base_kto": 3.237687826156616,
|
|
"loss/total_with_kl": 4.013680934906006,
|
|
"metrics/advantage_var": 466.0621032714844,
|
|
"regularization/advantage_var": 466.0621032714844,
|
|
"regularization/kl": 0.7759934663772583,
|
|
"rewards/chosen": 0.5108028411865234,
|
|
"rewards/margins": 0.8227104425430298,
|
|
"rewards/rejected": -0.31190760135650636,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 34.277565002441406,
|
|
"kl": 12.307581901550293,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -34852477.710569106,
|
|
"logits/rejected": -36199521.010526314,
|
|
"logps/chosen": -446.1870426829268,
|
|
"logps/rejected": -390.51139567669173,
|
|
"loss": 0.5223,
|
|
"loss/base_kto": 3.2325856685638428,
|
|
"loss/total_with_kl": 4.178535461425781,
|
|
"metrics/advantage_var": 568.1380004882812,
|
|
"regularization/advantage_var": 568.1380004882812,
|
|
"regularization/kl": 0.9459497332572937,
|
|
"rewards/chosen": 0.4276986471036585,
|
|
"rewards/margins": 0.8017381923779772,
|
|
"rewards/rejected": -0.3740395452743186,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 16.852924346923828,
|
|
"kl": 21.103397369384766,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -36382961.34680135,
|
|
"logits/rejected": -36860889.18950437,
|
|
"logps/chosen": -471.56691919191917,
|
|
"logps/rejected": -366.2282707725947,
|
|
"loss": 0.5039,
|
|
"loss/base_kto": 3.1697006225585938,
|
|
"loss/total_with_kl": 4.030932903289795,
|
|
"metrics/advantage_var": 517.2568359375,
|
|
"regularization/advantage_var": 517.2568359375,
|
|
"regularization/kl": 0.861232578754425,
|
|
"rewards/chosen": 0.5889803183199179,
|
|
"rewards/margins": 0.8769434292409046,
|
|
"rewards/rejected": -0.2879631109209867,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 13.533108711242676,
|
|
"kl": 10.815665245056152,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -35762853.48710167,
|
|
"logits/rejected": -35424846.32528181,
|
|
"logps/chosen": -451.8063353566009,
|
|
"logps/rejected": -362.4895833333333,
|
|
"loss": 0.5185,
|
|
"loss/base_kto": 3.2216908931732178,
|
|
"loss/total_with_kl": 4.148040771484375,
|
|
"metrics/advantage_var": 556.3663940429688,
|
|
"regularization/advantage_var": 556.3663940429688,
|
|
"regularization/kl": 0.9263500571250916,
|
|
"rewards/chosen": 0.4788356659444945,
|
|
"rewards/margins": 0.8566194243813026,
|
|
"rewards/rejected": -0.37778375843680806,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 21.33864402770996,
|
|
"kl": 21.08111572265625,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -35531581.40809969,
|
|
"logits/rejected": -36369761.10344828,
|
|
"logps/chosen": -463.61453465732086,
|
|
"logps/rejected": -363.862019984326,
|
|
"loss": 0.4965,
|
|
"loss/base_kto": 3.2017879486083984,
|
|
"loss/total_with_kl": 3.971954345703125,
|
|
"metrics/advantage_var": 462.5621032714844,
|
|
"regularization/advantage_var": 462.5621032714844,
|
|
"regularization/kl": 0.7701658606529236,
|
|
"rewards/chosen": 0.5949206010574863,
|
|
"rewards/margins": 0.8511713711719187,
|
|
"rewards/rejected": -0.2562507701144323,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 15.033001899719238,
|
|
"kl": 18.862932205200195,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -36394802.18780889,
|
|
"logits/rejected": -37580526.121842496,
|
|
"logps/chosen": -500.09071252059306,
|
|
"logps/rejected": -373.67106240713224,
|
|
"loss": 0.5066,
|
|
"loss/base_kto": 3.241631269454956,
|
|
"loss/total_with_kl": 4.0531744956970215,
|
|
"metrics/advantage_var": 487.41314697265625,
|
|
"regularization/advantage_var": 487.41314697265625,
|
|
"regularization/kl": 0.8115429282188416,
|
|
"rewards/chosen": 0.5605515003989909,
|
|
"rewards/margins": 0.7824891589271531,
|
|
"rewards/rejected": -0.22193765852816214,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 7.838743209838867,
|
|
"kl": 10.29575252532959,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -35350278.56410257,
|
|
"logits/rejected": -37508124.18348624,
|
|
"logps/chosen": -493.4591846955128,
|
|
"logps/rejected": -375.8194762996942,
|
|
"loss": 0.4779,
|
|
"loss/base_kto": 3.150137424468994,
|
|
"loss/total_with_kl": 3.823232650756836,
|
|
"metrics/advantage_var": 404.2614440917969,
|
|
"regularization/advantage_var": 404.2614440917969,
|
|
"regularization/kl": 0.6730953454971313,
|
|
"rewards/chosen": 0.44738153310922474,
|
|
"rewards/margins": 0.8603632255487794,
|
|
"rewards/rejected": -0.4129816924395547,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 9.149922370910645,
|
|
"kl": 10.108046531677246,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -35104825.33125972,
|
|
"logits/rejected": -36478493.73940346,
|
|
"logps/chosen": -472.9266621306376,
|
|
"logps/rejected": -363.74850372841445,
|
|
"loss": 0.4587,
|
|
"loss/base_kto": 3.1205315589904785,
|
|
"loss/total_with_kl": 3.6693668365478516,
|
|
"metrics/advantage_var": 329.63067626953125,
|
|
"regularization/advantage_var": 329.63067626953125,
|
|
"regularization/kl": 0.5488350987434387,
|
|
"rewards/chosen": 0.5208696570033048,
|
|
"rewards/margins": 0.9014249581126308,
|
|
"rewards/rejected": -0.38055530110932595,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 9.750592231750488,
|
|
"kl": 14.340692520141602,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -37244958.21639344,
|
|
"logits/rejected": -37628062.94925373,
|
|
"logps/chosen": -486.4201844262295,
|
|
"logps/rejected": -376.6964552238806,
|
|
"loss": 0.4783,
|
|
"loss/base_kto": 3.090118885040283,
|
|
"loss/total_with_kl": 3.826637029647827,
|
|
"metrics/advantage_var": 442.35333251953125,
|
|
"regularization/advantage_var": 442.35333251953125,
|
|
"regularization/kl": 0.7365182638168335,
|
|
"rewards/chosen": 0.6377988721503586,
|
|
"rewards/margins": 0.9401026760070517,
|
|
"rewards/rejected": -0.3023038038566931,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 10.538399696350098,
|
|
"kl": 14.86938190460205,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -36523571.98769231,
|
|
"logits/rejected": -36889148.13968254,
|
|
"logps/chosen": -465.06745192307693,
|
|
"logps/rejected": -375.3545882936508,
|
|
"loss": 0.4612,
|
|
"loss/base_kto": 3.1292314529418945,
|
|
"loss/total_with_kl": 3.6898274421691895,
|
|
"metrics/advantage_var": 336.6940612792969,
|
|
"regularization/advantage_var": 336.6940612792969,
|
|
"regularization/kl": 0.5605956315994263,
|
|
"rewards/chosen": 0.5273802771935097,
|
|
"rewards/margins": 0.8657501492715727,
|
|
"rewards/rejected": -0.338369872078063,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 9.5286283493042,
|
|
"kl": 20.716459274291992,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -33676433.83281734,
|
|
"logits/rejected": -34795135.59621451,
|
|
"logps/chosen": -479.11629256965944,
|
|
"logps/rejected": -356.22289037854887,
|
|
"loss": 0.4582,
|
|
"loss/base_kto": 3.143415927886963,
|
|
"loss/total_with_kl": 3.6656646728515625,
|
|
"metrics/advantage_var": 313.6628112792969,
|
|
"regularization/advantage_var": 313.6628112792969,
|
|
"regularization/kl": 0.5222485065460205,
|
|
"rewards/chosen": 0.6794614998548761,
|
|
"rewards/margins": 0.8385073480394557,
|
|
"rewards/rejected": -0.15904584818457956,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 10.756758689880371,
|
|
"kl": 20.486328125,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -36256672.55608215,
|
|
"logits/rejected": -38048123.84544049,
|
|
"logps/chosen": -471.6715047393365,
|
|
"logps/rejected": -360.400550618238,
|
|
"loss": 0.4548,
|
|
"loss/base_kto": 3.101207971572876,
|
|
"loss/total_with_kl": 3.638510227203369,
|
|
"metrics/advantage_var": 322.7040100097656,
|
|
"regularization/advantage_var": 322.7040100097656,
|
|
"regularization/kl": 0.5373021364212036,
|
|
"rewards/chosen": 0.6829303663112164,
|
|
"rewards/margins": 0.8889523193456542,
|
|
"rewards/rejected": -0.2060219530344378,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 13.509828567504883,
|
|
"kl": 14.168840408325195,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -34525097.3776435,
|
|
"logits/rejected": -35924650.666666664,
|
|
"logps/chosen": -448.12858761329306,
|
|
"logps/rejected": -381.19584344660194,
|
|
"loss": 0.4559,
|
|
"loss/base_kto": 3.1329665184020996,
|
|
"loss/total_with_kl": 3.6471784114837646,
|
|
"metrics/advantage_var": 308.8360290527344,
|
|
"regularization/advantage_var": 308.8360290527344,
|
|
"regularization/kl": 0.5142119526863098,
|
|
"rewards/chosen": 0.5891165373188492,
|
|
"rewards/margins": 0.853250472154635,
|
|
"rewards/rejected": -0.2641339348357858,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 12.57075023651123,
|
|
"kl": 19.2509822845459,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -35237894.29185868,
|
|
"logits/rejected": -37095485.863275036,
|
|
"logps/chosen": -482.6663786482335,
|
|
"logps/rejected": -384.49781399046105,
|
|
"loss": 0.4528,
|
|
"loss/base_kto": 3.0966010093688965,
|
|
"loss/total_with_kl": 3.622481107711792,
|
|
"metrics/advantage_var": 315.8439025878906,
|
|
"regularization/advantage_var": 315.8439025878906,
|
|
"regularization/kl": 0.5258800983428955,
|
|
"rewards/chosen": 0.6583768079907114,
|
|
"rewards/margins": 0.8704186901402667,
|
|
"rewards/rejected": -0.21204188214955536,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 15.50874137878418,
|
|
"kl": 21.606769561767578,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -35078353.7152,
|
|
"logits/rejected": -37200575.51145038,
|
|
"logps/chosen": -468.7221,
|
|
"logps/rejected": -377.8267175572519,
|
|
"loss": 0.4726,
|
|
"loss/base_kto": 3.1125903129577637,
|
|
"loss/total_with_kl": 3.7810211181640625,
|
|
"metrics/advantage_var": 401.46002197265625,
|
|
"regularization/advantage_var": 401.46002197265625,
|
|
"regularization/kl": 0.6684309244155884,
|
|
"rewards/chosen": 0.660563623046875,
|
|
"rewards/margins": 0.8704991680582062,
|
|
"rewards/rejected": -0.2099355450113311,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 11.897238731384277,
|
|
"kl": 16.70973014831543,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -35422536.184899844,
|
|
"logits/rejected": -36982170.57369255,
|
|
"logps/chosen": -507.70406394453005,
|
|
"logps/rejected": -369.6924524564184,
|
|
"loss": 0.4566,
|
|
"loss/base_kto": 3.113170623779297,
|
|
"loss/total_with_kl": 3.652764081954956,
|
|
"metrics/advantage_var": 324.0802917480469,
|
|
"regularization/advantage_var": 324.0802917480469,
|
|
"regularization/kl": 0.5395936965942383,
|
|
"rewards/chosen": 0.6239382327980547,
|
|
"rewards/margins": 0.8614468222146554,
|
|
"rewards/rejected": -0.23750858941660064,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 12.369135856628418,
|
|
"kl": 18.403594970703125,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -35873037.560192615,
|
|
"logits/rejected": -36774252.71232877,
|
|
"logps/chosen": -447.79995987158907,
|
|
"logps/rejected": -388.8015125570776,
|
|
"loss": 0.458,
|
|
"loss/base_kto": 3.0503218173980713,
|
|
"loss/total_with_kl": 3.664189100265503,
|
|
"metrics/advantage_var": 368.6891784667969,
|
|
"regularization/advantage_var": 368.6891784667969,
|
|
"regularization/kl": 0.613867461681366,
|
|
"rewards/chosen": 0.6563940155372191,
|
|
"rewards/margins": 0.9242874888445107,
|
|
"rewards/rejected": -0.2678934733072917,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 11.39638900756836,
|
|
"kl": 15.031561851501465,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -35858396.63422292,
|
|
"logits/rejected": -36195485.66096423,
|
|
"logps/chosen": -476.51609105180535,
|
|
"logps/rejected": -379.79335147744945,
|
|
"loss": 0.4714,
|
|
"loss/base_kto": 3.1057381629943848,
|
|
"loss/total_with_kl": 3.770984649658203,
|
|
"metrics/advantage_var": 399.547607421875,
|
|
"regularization/advantage_var": 399.547607421875,
|
|
"regularization/kl": 0.6652467846870422,
|
|
"rewards/chosen": 0.5956329393611411,
|
|
"rewards/margins": 0.9121395224251477,
|
|
"rewards/rejected": -0.3165065830640066,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 8.565475463867188,
|
|
"kl": 19.21080207824707,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -36174090.33183856,
|
|
"logits/rejected": -37466929.85924713,
|
|
"logps/chosen": -479.5201326606876,
|
|
"logps/rejected": -382.2530943126023,
|
|
"loss": 0.4596,
|
|
"loss/base_kto": 3.106900453567505,
|
|
"loss/total_with_kl": 3.6769168376922607,
|
|
"metrics/advantage_var": 342.35211181640625,
|
|
"regularization/advantage_var": 342.35211181640625,
|
|
"regularization/kl": 0.5700162649154663,
|
|
"rewards/chosen": 0.6672370123756306,
|
|
"rewards/margins": 0.8702982176511459,
|
|
"rewards/rejected": -0.2030612052755153,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 19.47868537902832,
|
|
"kl": 20.330358505249023,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -35070985.81469648,
|
|
"logits/rejected": -37145806.67889908,
|
|
"logps/chosen": -476.7040734824281,
|
|
"logps/rejected": -343.0444619648318,
|
|
"loss": 0.4547,
|
|
"loss/base_kto": 3.101200819015503,
|
|
"loss/total_with_kl": 3.637281894683838,
|
|
"metrics/advantage_var": 321.97064208984375,
|
|
"regularization/advantage_var": 321.97064208984375,
|
|
"regularization/kl": 0.5360811352729797,
|
|
"rewards/chosen": 0.6737556823145467,
|
|
"rewards/margins": 0.8761985164512296,
|
|
"rewards/rejected": -0.2024428341366829,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 14.665727615356445,
|
|
"kl": 17.39240264892578,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -35536572.115015976,
|
|
"logits/rejected": -36869580.33027523,
|
|
"logps/chosen": -501.66703274760386,
|
|
"logps/rejected": -349.67870795107035,
|
|
"loss": 0.4646,
|
|
"loss/base_kto": 3.1088297367095947,
|
|
"loss/total_with_kl": 3.7167680263519287,
|
|
"metrics/advantage_var": 365.1280822753906,
|
|
"regularization/advantage_var": 365.1280822753906,
|
|
"regularization/kl": 0.6079382300376892,
|
|
"rewards/chosen": 0.6022115591615914,
|
|
"rewards/margins": 0.8850808806522379,
|
|
"rewards/rejected": -0.2828693214906465,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 12.126596450805664,
|
|
"kl": 15.947830200195312,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -34450717.69278997,
|
|
"logits/rejected": -36153755.51401869,
|
|
"logps/chosen": -484.55329153605015,
|
|
"logps/rejected": -371.44813570872276,
|
|
"loss": 0.4584,
|
|
"loss/base_kto": 3.0944137573242188,
|
|
"loss/total_with_kl": 3.6672418117523193,
|
|
"metrics/advantage_var": 344.0411071777344,
|
|
"regularization/advantage_var": 344.0411071777344,
|
|
"regularization/kl": 0.5728284120559692,
|
|
"rewards/chosen": 0.5509435001959248,
|
|
"rewards/margins": 0.8927254491530412,
|
|
"rewards/rejected": -0.34178194895711644,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 13.250471115112305,
|
|
"kl": 15.709235191345215,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -35802305.11568938,
|
|
"logits/rejected": -37203478.87827427,
|
|
"logps/chosen": -487.08755942947704,
|
|
"logps/rejected": -402.92218798151004,
|
|
"loss": 0.4637,
|
|
"loss/base_kto": 3.1344103813171387,
|
|
"loss/total_with_kl": 3.7092621326446533,
|
|
"metrics/advantage_var": 345.2561950683594,
|
|
"regularization/advantage_var": 345.2561950683594,
|
|
"regularization/kl": 0.5748516321182251,
|
|
"rewards/chosen": 0.5793548922531201,
|
|
"rewards/margins": 0.8618215514463501,
|
|
"rewards/rejected": -0.28246665919323,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 8.29250717163086,
|
|
"kl": 16.85877799987793,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -35182345.45104334,
|
|
"logits/rejected": -36199771.56773212,
|
|
"logps/chosen": -488.061797752809,
|
|
"logps/rejected": -378.22590848554034,
|
|
"loss": 0.4607,
|
|
"loss/base_kto": 3.071286916732788,
|
|
"loss/total_with_kl": 3.6855340003967285,
|
|
"metrics/advantage_var": 368.9174499511719,
|
|
"regularization/advantage_var": 368.9174499511719,
|
|
"regularization/kl": 0.6142475008964539,
|
|
"rewards/chosen": 0.6008935625250803,
|
|
"rewards/margins": 0.9226538591652297,
|
|
"rewards/rejected": -0.3217602966401494,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 12.62736701965332,
|
|
"kl": 15.312149047851562,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -35044054.30745814,
|
|
"logits/rejected": -36081128.16693419,
|
|
"logps/chosen": -474.1341799847793,
|
|
"logps/rejected": -392.05372191011236,
|
|
"loss": 0.4641,
|
|
"loss/base_kto": 3.102219343185425,
|
|
"loss/total_with_kl": 3.7128524780273438,
|
|
"metrics/advantage_var": 366.7467041015625,
|
|
"regularization/advantage_var": 366.7467041015625,
|
|
"regularization/kl": 0.6106332540512085,
|
|
"rewards/chosen": 0.611444249726503,
|
|
"rewards/margins": 0.9139893471633007,
|
|
"rewards/rejected": -0.30254509743679775,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 11.171317100524902,
|
|
"kl": 13.967089653015137,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -36518208.41290323,
|
|
"logits/rejected": -37216280.82424243,
|
|
"logps/chosen": -484.270564516129,
|
|
"logps/rejected": -372.72211174242426,
|
|
"loss": 0.4651,
|
|
"loss/base_kto": 3.111393451690674,
|
|
"loss/total_with_kl": 3.7210137844085693,
|
|
"metrics/advantage_var": 366.1383972167969,
|
|
"regularization/advantage_var": 366.1383972167969,
|
|
"regularization/kl": 0.6096203923225403,
|
|
"rewards/chosen": 0.6078026063980595,
|
|
"rewards/margins": 0.9022674806656377,
|
|
"rewards/rejected": -0.2944648742675781,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 15.535805702209473,
|
|
"kl": 13.67083740234375,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -35986181.827010624,
|
|
"logits/rejected": -37283889.46859904,
|
|
"logps/chosen": -491.7790686646434,
|
|
"logps/rejected": -382.84329710144925,
|
|
"loss": 0.4581,
|
|
"loss/base_kto": 3.0956504344940186,
|
|
"loss/total_with_kl": 3.6650466918945312,
|
|
"metrics/advantage_var": 341.979736328125,
|
|
"regularization/advantage_var": 341.979736328125,
|
|
"regularization/kl": 0.5693961977958679,
|
|
"rewards/chosen": 0.5942041978850531,
|
|
"rewards/margins": 0.9113392086335308,
|
|
"rewards/rejected": -0.3171350107484778,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 7.794953346252441,
|
|
"kl": 17.230022430419922,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -35545908.50238474,
|
|
"logits/rejected": -37066632.4546851,
|
|
"logps/chosen": -482.3726152623211,
|
|
"logps/rejected": -350.3778801843318,
|
|
"loss": 0.4576,
|
|
"loss/base_kto": 3.122943639755249,
|
|
"loss/total_with_kl": 3.6607768535614014,
|
|
"metrics/advantage_var": 323.0230407714844,
|
|
"regularization/advantage_var": 323.0230407714844,
|
|
"regularization/kl": 0.5378333330154419,
|
|
"rewards/chosen": 0.6392264313083764,
|
|
"rewards/margins": 0.8757048002989918,
|
|
"rewards/rejected": -0.2364783689906154,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 14.496573448181152,
|
|
"kl": 14.244569778442383,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -35272454.91891892,
|
|
"logits/rejected": -38282980.48208469,
|
|
"logps/chosen": -483.8616741741742,
|
|
"logps/rejected": -365.0823493485342,
|
|
"loss": 0.4661,
|
|
"loss/base_kto": 3.119757890701294,
|
|
"loss/total_with_kl": 3.7290408611297607,
|
|
"metrics/advantage_var": 365.9358825683594,
|
|
"regularization/advantage_var": 365.9358825683594,
|
|
"regularization/kl": 0.6092831492424011,
|
|
"rewards/chosen": 0.5961507161458334,
|
|
"rewards/margins": 0.8685713943000984,
|
|
"rewards/rejected": -0.2724206781542651,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 8.606616973876953,
|
|
"kl": 17.470041275024414,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -35759817.3126935,
|
|
"logits/rejected": -36109596.26498423,
|
|
"logps/chosen": -474.74076044891643,
|
|
"logps/rejected": -400.95859621451103,
|
|
"loss": 0.4785,
|
|
"loss/base_kto": 3.094226598739624,
|
|
"loss/total_with_kl": 3.827843427658081,
|
|
"metrics/advantage_var": 440.6109313964844,
|
|
"regularization/advantage_var": 440.6109313964844,
|
|
"regularization/kl": 0.7336172461509705,
|
|
"rewards/chosen": 0.6469605625967493,
|
|
"rewards/margins": 0.9237381008120735,
|
|
"rewards/rejected": -0.2767775382153243,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 9.362181663513184,
|
|
"kl": 16.609148025512695,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -35475751.20720721,
|
|
"logits/rejected": -36011905.25081433,
|
|
"logps/chosen": -477.34834834834834,
|
|
"logps/rejected": -375.93322475570034,
|
|
"loss": 0.4774,
|
|
"loss/base_kto": 3.1456475257873535,
|
|
"loss/total_with_kl": 3.8194503784179688,
|
|
"metrics/advantage_var": 404.6862487792969,
|
|
"regularization/advantage_var": 404.6862487792969,
|
|
"regularization/kl": 0.6738025546073914,
|
|
"rewards/chosen": 0.6248625792540587,
|
|
"rewards/margins": 0.8333165870768969,
|
|
"rewards/rejected": -0.2084540078228382,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 9.95600414276123,
|
|
"kl": 20.428028106689453,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -34650066.1248,
|
|
"logits/rejected": -36858165.54503817,
|
|
"logps/chosen": -500.63025,
|
|
"logps/rejected": -372.54942748091605,
|
|
"loss": 0.4551,
|
|
"loss/base_kto": 3.068481922149658,
|
|
"loss/total_with_kl": 3.640624761581421,
|
|
"metrics/advantage_var": 343.62945556640625,
|
|
"regularization/advantage_var": 343.62945556640625,
|
|
"regularization/kl": 0.572143018245697,
|
|
"rewards/chosen": 0.65843291015625,
|
|
"rewards/margins": 0.9051499642175573,
|
|
"rewards/rejected": -0.24671705406130726,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 11.976444244384766,
|
|
"kl": 20.151103973388672,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -35797414.233766235,
|
|
"logits/rejected": -37080107.18072289,
|
|
"logps/chosen": -504.4137581168831,
|
|
"logps/rejected": -370.0573701054217,
|
|
"loss": 0.4601,
|
|
"loss/base_kto": 3.0977590084075928,
|
|
"loss/total_with_kl": 3.6808815002441406,
|
|
"metrics/advantage_var": 350.2239685058594,
|
|
"regularization/advantage_var": 350.2239685058594,
|
|
"regularization/kl": 0.5831229090690613,
|
|
"rewards/chosen": 0.668527974710836,
|
|
"rewards/margins": 0.8835734210837299,
|
|
"rewards/rejected": -0.2150454463728939,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 11.191988945007324,
|
|
"kl": 17.3702449798584,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -35625021.08634223,
|
|
"logits/rejected": -37539808.14930016,
|
|
"logps/chosen": -478.78806907378333,
|
|
"logps/rejected": -379.2405715396579,
|
|
"loss": 0.4617,
|
|
"loss/base_kto": 3.1444108486175537,
|
|
"loss/total_with_kl": 3.6934609413146973,
|
|
"metrics/advantage_var": 329.759765625,
|
|
"regularization/advantage_var": 329.759765625,
|
|
"regularization/kl": 0.5490500330924988,
|
|
"rewards/chosen": 0.6028379143875098,
|
|
"rewards/margins": 0.840053066255888,
|
|
"rewards/rejected": -0.23721515186837822,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 10.4734468460083,
|
|
"kl": 17.648040771484375,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -34946985.977382876,
|
|
"logits/rejected": -34789041.379727684,
|
|
"logps/chosen": -463.60213045234246,
|
|
"logps/rejected": -385.6117861195159,
|
|
"loss": 0.46,
|
|
"loss/base_kto": 3.097611427307129,
|
|
"loss/total_with_kl": 3.679656982421875,
|
|
"metrics/advantage_var": 349.5768127441406,
|
|
"regularization/advantage_var": 349.5768127441406,
|
|
"regularization/kl": 0.5820453763008118,
|
|
"rewards/chosen": 0.6198318308890347,
|
|
"rewards/margins": 0.8890758647717645,
|
|
"rewards/rejected": -0.26924403388272977,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 10.661663055419922,
|
|
"kl": 15.899003982543945,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -34278258.54654655,
|
|
"logits/rejected": -36539315.283387624,
|
|
"logps/chosen": -484.5742304804805,
|
|
"logps/rejected": -353.02107084690556,
|
|
"loss": 0.454,
|
|
"loss/base_kto": 3.0870721340179443,
|
|
"loss/total_with_kl": 3.6318938732147217,
|
|
"metrics/advantage_var": 327.2203674316406,
|
|
"regularization/advantage_var": 327.2203674316406,
|
|
"regularization/kl": 0.5448219180107117,
|
|
"rewards/chosen": 0.6431778458145646,
|
|
"rewards/margins": 0.9113439063709825,
|
|
"rewards/rejected": -0.26816606055641795,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 10.554672241210938,
|
|
"kl": 14.993478775024414,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -35443723.1131783,
|
|
"logits/rejected": -36313168.62992126,
|
|
"logps/chosen": -494.4032945736434,
|
|
"logps/rejected": -364.5371062992126,
|
|
"loss": 0.4589,
|
|
"loss/base_kto": 3.143242359161377,
|
|
"loss/total_with_kl": 3.670926809310913,
|
|
"metrics/advantage_var": 316.927490234375,
|
|
"regularization/advantage_var": 316.927490234375,
|
|
"regularization/kl": 0.5276843309402466,
|
|
"rewards/chosen": 0.5962972005208333,
|
|
"rewards/margins": 0.8600629591253486,
|
|
"rewards/rejected": -0.26376575860451523,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 11.917473793029785,
|
|
"kl": 16.714590072631836,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -35588741.882725835,
|
|
"logits/rejected": -36025733.719568565,
|
|
"logps/chosen": -475.3946117274168,
|
|
"logps/rejected": -381.23353235747305,
|
|
"loss": 0.4643,
|
|
"loss/base_kto": 3.10038685798645,
|
|
"loss/total_with_kl": 3.714500904083252,
|
|
"metrics/advantage_var": 368.8370056152344,
|
|
"regularization/advantage_var": 368.8370056152344,
|
|
"regularization/kl": 0.6141135692596436,
|
|
"rewards/chosen": 0.6494020682696118,
|
|
"rewards/margins": 0.9073708004010761,
|
|
"rewards/rejected": -0.25796873213146426,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 11.250147819519043,
|
|
"kl": 16.610803604125977,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -35299404.086687304,
|
|
"logits/rejected": -36673949.476340696,
|
|
"logps/chosen": -490.515576625387,
|
|
"logps/rejected": -370.5534305993691,
|
|
"loss": 0.4572,
|
|
"loss/base_kto": 3.090590238571167,
|
|
"loss/total_with_kl": 3.6578400135040283,
|
|
"metrics/advantage_var": 340.69073486328125,
|
|
"regularization/advantage_var": 340.69073486328125,
|
|
"regularization/kl": 0.5672500729560852,
|
|
"rewards/chosen": 0.6346463891374806,
|
|
"rewards/margins": 0.8842525246532411,
|
|
"rewards/rejected": -0.24960613551576055,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 12.6209077835083,
|
|
"kl": 16.491117477416992,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -36563779.53374233,
|
|
"logits/rejected": -38137869.04458599,
|
|
"logps/chosen": -496.8289877300613,
|
|
"logps/rejected": -397.1479896496815,
|
|
"loss": 0.4686,
|
|
"loss/base_kto": 3.15112566947937,
|
|
"loss/total_with_kl": 3.7484970092773438,
|
|
"metrics/advantage_var": 358.781494140625,
|
|
"regularization/advantage_var": 358.781494140625,
|
|
"regularization/kl": 0.5973712205886841,
|
|
"rewards/chosen": 0.6082396243978863,
|
|
"rewards/margins": 0.8536871598216025,
|
|
"rewards/rejected": -0.24544753542371617,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 9.102306365966797,
|
|
"kl": 14.926358222961426,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -34773597.575037144,
|
|
"logits/rejected": -36620473.56836903,
|
|
"logps/chosen": -451.94650817236254,
|
|
"logps/rejected": -371.92668863261946,
|
|
"loss": 0.4715,
|
|
"loss/base_kto": 3.195483922958374,
|
|
"loss/total_with_kl": 3.7720258235931396,
|
|
"metrics/advantage_var": 346.2713623046875,
|
|
"regularization/advantage_var": 346.2713623046875,
|
|
"regularization/kl": 0.5765418410301208,
|
|
"rewards/chosen": 0.5847326678294483,
|
|
"rewards/margins": 0.7950258296395801,
|
|
"rewards/rejected": -0.2102931618101318,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 20.292781829833984,
|
|
"kl": 17.15540885925293,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -34452366.222222224,
|
|
"logits/rejected": -36262594.09672387,
|
|
"logps/chosen": -485.53257042253523,
|
|
"logps/rejected": -369.1083268330733,
|
|
"loss": 0.4608,
|
|
"loss/base_kto": 3.0643093585968018,
|
|
"loss/total_with_kl": 3.6860809326171875,
|
|
"metrics/advantage_var": 373.4364929199219,
|
|
"regularization/advantage_var": 373.4364929199219,
|
|
"regularization/kl": 0.6217717528343201,
|
|
"rewards/chosen": 0.6874183332416374,
|
|
"rewards/margins": 0.9498664819244207,
|
|
"rewards/rejected": -0.26244814868278327,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 7.896585464477539,
|
|
"kl": 19.631961822509766,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -34915289.82820097,
|
|
"logits/rejected": -35333362.48567119,
|
|
"logps/chosen": -496.1209481361426,
|
|
"logps/rejected": -379.11625659879337,
|
|
"loss": 0.456,
|
|
"loss/base_kto": 3.134779930114746,
|
|
"loss/total_with_kl": 3.6479270458221436,
|
|
"metrics/advantage_var": 308.1963806152344,
|
|
"regularization/advantage_var": 308.1963806152344,
|
|
"regularization/kl": 0.5131470561027527,
|
|
"rewards/chosen": 0.5945063611097042,
|
|
"rewards/margins": 0.8418014358027425,
|
|
"rewards/rejected": -0.24729507469303827,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 6.725947380065918,
|
|
"kl": 15.279718399047852,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -36518858.60122699,
|
|
"logits/rejected": -36968134.929936305,
|
|
"logps/chosen": -467.9155962423313,
|
|
"logps/rejected": -390.60415007961785,
|
|
"loss": 0.4587,
|
|
"loss/base_kto": 3.129828691482544,
|
|
"loss/total_with_kl": 3.6699371337890625,
|
|
"metrics/advantage_var": 324.3893127441406,
|
|
"regularization/advantage_var": 324.3893127441406,
|
|
"regularization/kl": 0.5401082038879395,
|
|
"rewards/chosen": 0.6043563561936829,
|
|
"rewards/margins": 0.8610317119470045,
|
|
"rewards/rejected": -0.25667535575332157,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.98294195064275e+17,
|
|
"train_loss": 0.5012723423656404,
|
|
"train_runtime": 10990.0555,
|
|
"train_samples_per_second": 13.487,
|
|
"train_steps_per_second": 0.211
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.98294195064275e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|