2229 lines
81 KiB
JSON
2229 lines
81 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 3.0,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 2316,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"epoch": 0.025906735751295335,
|
||
|
|
"grad_norm": 15.130857467651367,
|
||
|
|
"kl": 4.761574745178223,
|
||
|
|
"learning_rate": 1.8999999999999998e-07,
|
||
|
|
"logits/chosen": -36942093.653333336,
|
||
|
|
"logits/rejected": -38905217.50588235,
|
||
|
|
"logps/chosen": -484.46708333333333,
|
||
|
|
"logps/rejected": -355.75847886029413,
|
||
|
|
"loss": 0.5879,
|
||
|
|
"loss/base_kto": 3.8887081146240234,
|
||
|
|
"metrics/advantage_var": 190.82591247558594,
|
||
|
|
"regularization/mmd": 0.8144394159317017,
|
||
|
|
"regularization/rkhs_norm": 156.92474365234375,
|
||
|
|
"rewards/chosen": -0.08358226140340169,
|
||
|
|
"rewards/margins": 0.07920884487675686,
|
||
|
|
"rewards/rejected": -0.16279110628015855,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.05181347150259067,
|
||
|
|
"grad_norm": 15.646539688110352,
|
||
|
|
"kl": 3.6444268226623535,
|
||
|
|
"learning_rate": 3.8999999999999997e-07,
|
||
|
|
"logits/chosen": -35316123.529042386,
|
||
|
|
"logits/rejected": -36554541.78538103,
|
||
|
|
"logps/chosen": -464.786204866562,
|
||
|
|
"logps/rejected": -379.72438763608085,
|
||
|
|
"loss": 0.5807,
|
||
|
|
"loss/base_kto": 3.8761627674102783,
|
||
|
|
"metrics/advantage_var": 168.8544921875,
|
||
|
|
"regularization/mmd": 0.7692461013793945,
|
||
|
|
"regularization/rkhs_norm": 148.21697998046875,
|
||
|
|
"rewards/chosen": 0.044457020901810244,
|
||
|
|
"rewards/margins": 0.1253069962029803,
|
||
|
|
"rewards/rejected": -0.08084997530117005,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07772020725388601,
|
||
|
|
"grad_norm": 18.42901611328125,
|
||
|
|
"kl": 5.139020919799805,
|
||
|
|
"learning_rate": 5.9e-07,
|
||
|
|
"logits/chosen": -34244575.38853503,
|
||
|
|
"logits/rejected": -37065247.41104294,
|
||
|
|
"logps/chosen": -481.8531548566879,
|
||
|
|
"logps/rejected": -378.44581575920245,
|
||
|
|
"loss": 0.576,
|
||
|
|
"loss/base_kto": 3.8305325508117676,
|
||
|
|
"metrics/advantage_var": 168.36598205566406,
|
||
|
|
"regularization/mmd": 0.7773937582969666,
|
||
|
|
"regularization/rkhs_norm": 149.78684997558594,
|
||
|
|
"rewards/chosen": 0.030986403204073573,
|
||
|
|
"rewards/margins": 0.15914004619783673,
|
||
|
|
"rewards/rejected": -0.12815364299376317,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10362694300518134,
|
||
|
|
"grad_norm": 15.9312162399292,
|
||
|
|
"kl": 6.56032133102417,
|
||
|
|
"learning_rate": 7.9e-07,
|
||
|
|
"logits/chosen": -35905693.2864,
|
||
|
|
"logits/rejected": -37326103.84122138,
|
||
|
|
"logps/chosen": -478.7005,
|
||
|
|
"logps/rejected": -371.0121898854962,
|
||
|
|
"loss": 0.5894,
|
||
|
|
"loss/base_kto": 3.8666043281555176,
|
||
|
|
"metrics/advantage_var": 211.9551544189453,
|
||
|
|
"regularization/mmd": 0.8487977981567383,
|
||
|
|
"regularization/rkhs_norm": 163.5448455810547,
|
||
|
|
"rewards/chosen": 0.10424482421875,
|
||
|
|
"rewards/margins": 0.12079478056230618,
|
||
|
|
"rewards/rejected": -0.016549956343556178,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.12953367875647667,
|
||
|
|
"grad_norm": 20.496784210205078,
|
||
|
|
"kl": 4.3715033531188965,
|
||
|
|
"learning_rate": 9.9e-07,
|
||
|
|
"logits/chosen": -34802072.27228525,
|
||
|
|
"logits/rejected": -36334751.08295626,
|
||
|
|
"logps/chosen": -447.65169165316047,
|
||
|
|
"logps/rejected": -361.70769702111613,
|
||
|
|
"loss": 0.574,
|
||
|
|
"loss/base_kto": 3.859165906906128,
|
||
|
|
"metrics/advantage_var": 153.92991638183594,
|
||
|
|
"regularization/mmd": 0.7332141995429993,
|
||
|
|
"regularization/rkhs_norm": 141.2744140625,
|
||
|
|
"rewards/chosen": 0.028494527312886195,
|
||
|
|
"rewards/margins": 0.1332735938561201,
|
||
|
|
"rewards/rejected": -0.10477906654323388,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.15544041450777202,
|
||
|
|
"grad_norm": 17.79025650024414,
|
||
|
|
"kl": 15.81823444366455,
|
||
|
|
"learning_rate": 9.998186234298189e-07,
|
||
|
|
"logits/chosen": -35241291.294117644,
|
||
|
|
"logits/rejected": -36149821.44,
|
||
|
|
"logps/chosen": -481.5256893382353,
|
||
|
|
"logps/rejected": -367.18734375,
|
||
|
|
"loss": 0.586,
|
||
|
|
"loss/base_kto": 3.828622579574585,
|
||
|
|
"metrics/advantage_var": 208.0641632080078,
|
||
|
|
"regularization/mmd": 0.8597148060798645,
|
||
|
|
"regularization/rkhs_norm": 165.64834594726562,
|
||
|
|
"rewards/chosen": 0.24961013793945314,
|
||
|
|
"rewards/margins": 0.17664976755777995,
|
||
|
|
"rewards/rejected": 0.07296037038167318,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18134715025906736,
|
||
|
|
"grad_norm": 21.106868743896484,
|
||
|
|
"kl": 16.922224044799805,
|
||
|
|
"learning_rate": 9.992359552107714e-07,
|
||
|
|
"logits/chosen": -34983266.579104476,
|
||
|
|
"logits/rejected": -36376092.537704915,
|
||
|
|
"logps/chosen": -474.43222947761194,
|
||
|
|
"logps/rejected": -355.95435450819673,
|
||
|
|
"loss": 0.5752,
|
||
|
|
"loss/base_kto": 3.780517578125,
|
||
|
|
"metrics/advantage_var": 188.562744140625,
|
||
|
|
"regularization/mmd": 0.8209512829780579,
|
||
|
|
"regularization/rkhs_norm": 158.179443359375,
|
||
|
|
"rewards/chosen": 0.3078866531599813,
|
||
|
|
"rewards/margins": 0.21121301914678972,
|
||
|
|
"rewards/rejected": 0.0966736340131916,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20725388601036268,
|
||
|
|
"grad_norm": 15.32691478729248,
|
||
|
|
"kl": 12.154338836669922,
|
||
|
|
"learning_rate": 9.982519612796161e-07,
|
||
|
|
"logits/chosen": -36591796.327044025,
|
||
|
|
"logits/rejected": -36384440.44720497,
|
||
|
|
"logps/chosen": -473.89087067610063,
|
||
|
|
"logps/rejected": -364.2359035326087,
|
||
|
|
"loss": 0.5765,
|
||
|
|
"loss/base_kto": 3.747577667236328,
|
||
|
|
"metrics/advantage_var": 216.19285583496094,
|
||
|
|
"regularization/mmd": 0.8640968203544617,
|
||
|
|
"regularization/rkhs_norm": 166.49266052246094,
|
||
|
|
"rewards/chosen": 0.2337539960753243,
|
||
|
|
"rewards/margins": 0.236531285234874,
|
||
|
|
"rewards/rejected": -0.0027772891595496895,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.23316062176165803,
|
||
|
|
"grad_norm": 14.877063751220703,
|
||
|
|
"kl": 7.616835117340088,
|
||
|
|
"learning_rate": 9.968674326492213e-07,
|
||
|
|
"logits/chosen": -35663145.394034535,
|
||
|
|
"logits/rejected": -36818721.04510109,
|
||
|
|
"logps/chosen": -472.18789246467816,
|
||
|
|
"logps/rejected": -349.0640066096423,
|
||
|
|
"loss": 0.5783,
|
||
|
|
"loss/base_kto": 3.768833875656128,
|
||
|
|
"metrics/advantage_var": 209.60205078125,
|
||
|
|
"regularization/mmd": 0.8572441339492798,
|
||
|
|
"regularization/rkhs_norm": 165.17227172851562,
|
||
|
|
"rewards/chosen": 0.11851194851813923,
|
||
|
|
"rewards/margins": 0.21512558183562686,
|
||
|
|
"rewards/rejected": -0.09661363331748761,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.25906735751295334,
|
||
|
|
"grad_norm": 13.783124923706055,
|
||
|
|
"kl": 11.596030235290527,
|
||
|
|
"learning_rate": 9.950834823142198e-07,
|
||
|
|
"logits/chosen": -37818205.16207951,
|
||
|
|
"logits/rejected": -37808514.044728436,
|
||
|
|
"logps/chosen": -509.60311544342505,
|
||
|
|
"logps/rejected": -362.6952875399361,
|
||
|
|
"loss": 0.5794,
|
||
|
|
"loss/base_kto": 3.737436056137085,
|
||
|
|
"metrics/advantage_var": 217.01065063476562,
|
||
|
|
"regularization/mmd": 0.8980569839477539,
|
||
|
|
"regularization/rkhs_norm": 173.03604125976562,
|
||
|
|
"rewards/chosen": 0.25883346230976445,
|
||
|
|
"rewards/margins": 0.2537100382367676,
|
||
|
|
"rewards/rejected": 0.0051234240729968765,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2849740932642487,
|
||
|
|
"grad_norm": 15.125517845153809,
|
||
|
|
"kl": 9.459787368774414,
|
||
|
|
"learning_rate": 9.929015443562942e-07,
|
||
|
|
"logits/chosen": -36797581.40212443,
|
||
|
|
"logits/rejected": -38671823.355877616,
|
||
|
|
"logps/chosen": -498.74411987860395,
|
||
|
|
"logps/rejected": -376.55427234299515,
|
||
|
|
"loss": 0.5865,
|
||
|
|
"loss/base_kto": 3.8149335384368896,
|
||
|
|
"metrics/advantage_var": 254.8901824951172,
|
||
|
|
"regularization/mmd": 0.8767148852348328,
|
||
|
|
"regularization/rkhs_norm": 168.92385864257812,
|
||
|
|
"rewards/chosen": 0.20460924718738144,
|
||
|
|
"rewards/margins": 0.16430174302794856,
|
||
|
|
"rewards/rejected": 0.04030750415943287,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.31088082901554404,
|
||
|
|
"grad_norm": 14.51710319519043,
|
||
|
|
"kl": 11.137063980102539,
|
||
|
|
"learning_rate": 9.90323372791347e-07,
|
||
|
|
"logits/chosen": -35898331.371695176,
|
||
|
|
"logits/rejected": -36782517.24960753,
|
||
|
|
"logps/chosen": -475.05214813374806,
|
||
|
|
"logps/rejected": -366.24526589481945,
|
||
|
|
"loss": 0.5812,
|
||
|
|
"loss/base_kto": 3.800611972808838,
|
||
|
|
"metrics/advantage_var": 206.521240234375,
|
||
|
|
"regularization/mmd": 0.8491171002388,
|
||
|
|
"regularization/rkhs_norm": 163.6063995361328,
|
||
|
|
"rewards/chosen": 0.16592986253705774,
|
||
|
|
"rewards/margins": 0.17666043527199232,
|
||
|
|
"rewards/rejected": -0.010730572734934569,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.33678756476683935,
|
||
|
|
"grad_norm": 13.338346481323242,
|
||
|
|
"kl": 6.600700378417969,
|
||
|
|
"learning_rate": 9.87351040159484e-07,
|
||
|
|
"logits/chosen": -36512637.37952756,
|
||
|
|
"logits/rejected": -37593764.31627907,
|
||
|
|
"logps/chosen": -487.8697834645669,
|
||
|
|
"logps/rejected": -380.8578246124031,
|
||
|
|
"loss": 0.585,
|
||
|
|
"loss/base_kto": 3.7840969562530518,
|
||
|
|
"metrics/advantage_var": 230.69541931152344,
|
||
|
|
"regularization/mmd": 0.8959032297134399,
|
||
|
|
"regularization/rkhs_norm": 172.6210479736328,
|
||
|
|
"rewards/chosen": 0.06875443946658157,
|
||
|
|
"rewards/margins": 0.20920915611205154,
|
||
|
|
"rewards/rejected": -0.14045471664546996,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3626943005181347,
|
||
|
|
"grad_norm": 16.90336036682129,
|
||
|
|
"kl": 9.412796020507812,
|
||
|
|
"learning_rate": 9.839869358589396e-07,
|
||
|
|
"logits/chosen": -34535846.7072,
|
||
|
|
"logits/rejected": -36815961.11145038,
|
||
|
|
"logps/chosen": -478.9129,
|
||
|
|
"logps/rejected": -384.14408396946567,
|
||
|
|
"loss": 0.5829,
|
||
|
|
"loss/base_kto": 3.7607433795928955,
|
||
|
|
"metrics/advantage_var": 231.4911651611328,
|
||
|
|
"regularization/mmd": 0.9025770425796509,
|
||
|
|
"regularization/rkhs_norm": 173.9069366455078,
|
||
|
|
"rewards/chosen": 0.1127895263671875,
|
||
|
|
"rewards/margins": 0.19504585515378997,
|
||
|
|
"rewards/rejected": -0.08225632878660245,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.38860103626943004,
|
||
|
|
"grad_norm": 16.822004318237305,
|
||
|
|
"kl": 10.808634757995605,
|
||
|
|
"learning_rate": 9.80233764225289e-07,
|
||
|
|
"logits/chosen": -35965313.58024691,
|
||
|
|
"logits/rejected": -37074036.658227846,
|
||
|
|
"logps/chosen": -481.5387249228395,
|
||
|
|
"logps/rejected": -390.07004054588606,
|
||
|
|
"loss": 0.5872,
|
||
|
|
"loss/base_kto": 3.78863525390625,
|
||
|
|
"metrics/advantage_var": 229.93997192382812,
|
||
|
|
"regularization/mmd": 0.9090096354484558,
|
||
|
|
"regularization/rkhs_norm": 175.14639282226562,
|
||
|
|
"rewards/chosen": 0.20611553427613813,
|
||
|
|
"rewards/margins": 0.18817446376778482,
|
||
|
|
"rewards/rejected": 0.017941070508353318,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.41450777202072536,
|
||
|
|
"grad_norm": 14.761191368103027,
|
||
|
|
"kl": 13.942374229431152,
|
||
|
|
"learning_rate": 9.760945423574868e-07,
|
||
|
|
"logits/chosen": -37473318.4962406,
|
||
|
|
"logits/rejected": -37717673.001626015,
|
||
|
|
"logps/chosen": -469.2546052631579,
|
||
|
|
"logps/rejected": -388.0675304878049,
|
||
|
|
"loss": 0.5779,
|
||
|
|
"loss/base_kto": 3.696047306060791,
|
||
|
|
"metrics/advantage_var": 240.49853515625,
|
||
|
|
"regularization/mmd": 0.9273321032524109,
|
||
|
|
"regularization/rkhs_norm": 178.67672729492188,
|
||
|
|
"rewards/chosen": 0.3169454244742716,
|
||
|
|
"rewards/margins": 0.27461627825190116,
|
||
|
|
"rewards/rejected": 0.042329146222370426,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.44041450777202074,
|
||
|
|
"grad_norm": 16.781259536743164,
|
||
|
|
"kl": 10.861861228942871,
|
||
|
|
"learning_rate": 9.71572597692482e-07,
|
||
|
|
"logits/chosen": -37333167.72542902,
|
||
|
|
"logits/rejected": -37195577.28951487,
|
||
|
|
"logps/chosen": -509.5145280811232,
|
||
|
|
"logps/rejected": -366.51457355242565,
|
||
|
|
"loss": 0.5783,
|
||
|
|
"loss/base_kto": 3.7383134365081787,
|
||
|
|
"metrics/advantage_var": 223.09609985351562,
|
||
|
|
"regularization/mmd": 0.8883072137832642,
|
||
|
|
"regularization/rkhs_norm": 171.157470703125,
|
||
|
|
"rewards/chosen": 0.17960316007855157,
|
||
|
|
"rewards/margins": 0.25466792516101683,
|
||
|
|
"rewards/rejected": -0.07506476508246528,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.46632124352331605,
|
||
|
|
"grad_norm": 16.52114486694336,
|
||
|
|
"kl": 10.722800254821777,
|
||
|
|
"learning_rate": 9.666715653303588e-07,
|
||
|
|
"logits/chosen": -34513129.7057057,
|
||
|
|
"logits/rejected": -35481946.89250814,
|
||
|
|
"logps/chosen": -490.37509384384384,
|
||
|
|
"logps/rejected": -388.6891286644951,
|
||
|
|
"loss": 0.5835,
|
||
|
|
"loss/base_kto": 3.733602285385132,
|
||
|
|
"metrics/advantage_var": 251.0590057373047,
|
||
|
|
"regularization/mmd": 0.9347845315933228,
|
||
|
|
"regularization/rkhs_norm": 180.1126251220703,
|
||
|
|
"rewards/chosen": 0.2069030395141235,
|
||
|
|
"rewards/margins": 0.2647182557773882,
|
||
|
|
"rewards/rejected": -0.05781521626326471,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.49222797927461137,
|
||
|
|
"grad_norm": 15.33375072479248,
|
||
|
|
"kl": 9.749954223632812,
|
||
|
|
"learning_rate": 9.613953851121528e-07,
|
||
|
|
"logits/chosen": -33689994.335403726,
|
||
|
|
"logits/rejected": -36951816.05031446,
|
||
|
|
"logps/chosen": -496.167701863354,
|
||
|
|
"logps/rejected": -357.282822327044,
|
||
|
|
"loss": 0.5827,
|
||
|
|
"loss/base_kto": 3.7767181396484375,
|
||
|
|
"metrics/advantage_var": 226.6070556640625,
|
||
|
|
"regularization/mmd": 0.8848592042922974,
|
||
|
|
"regularization/rkhs_norm": 170.4930877685547,
|
||
|
|
"rewards/chosen": 0.15638849009638248,
|
||
|
|
"rewards/margins": 0.21226255288044546,
|
||
|
|
"rewards/rejected": -0.05587406278406299,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5181347150259067,
|
||
|
|
"grad_norm": 14.226479530334473,
|
||
|
|
"kl": 13.073600769042969,
|
||
|
|
"learning_rate": 9.557482984526924e-07,
|
||
|
|
"logits/chosen": -35692129.29595016,
|
||
|
|
"logits/rejected": -37533185.60501567,
|
||
|
|
"logps/chosen": -478.4701129283489,
|
||
|
|
"logps/rejected": -379.8528115203762,
|
||
|
|
"loss": 0.5828,
|
||
|
|
"loss/base_kto": 3.7585251331329346,
|
||
|
|
"metrics/advantage_var": 226.40322875976562,
|
||
|
|
"regularization/mmd": 0.9036926627159119,
|
||
|
|
"regularization/rkhs_norm": 174.1219024658203,
|
||
|
|
"rewards/chosen": 0.20966519744968118,
|
||
|
|
"rewards/margins": 0.23187254278439492,
|
||
|
|
"rewards/rejected": -0.022207345334713736,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5440414507772021,
|
||
|
|
"grad_norm": 17.16434669494629,
|
||
|
|
"kl": 20.28582763671875,
|
||
|
|
"learning_rate": 9.497348449310107e-07,
|
||
|
|
"logits/chosen": -36566540.641975306,
|
||
|
|
"logits/rejected": -36928382.37974683,
|
||
|
|
"logps/chosen": -483.3317901234568,
|
||
|
|
"logps/rejected": -381.178945806962,
|
||
|
|
"loss": 0.5811,
|
||
|
|
"loss/base_kto": 3.742854356765747,
|
||
|
|
"metrics/advantage_var": 239.9812469482422,
|
||
|
|
"regularization/mmd": 0.9056269526481628,
|
||
|
|
"regularization/rkhs_norm": 174.49461364746094,
|
||
|
|
"rewards/chosen": 0.32063430032612367,
|
||
|
|
"rewards/margins": 0.19578205948603267,
|
||
|
|
"rewards/rejected": 0.12485224084009099,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5699481865284974,
|
||
|
|
"grad_norm": 15.937045097351074,
|
||
|
|
"kl": 4.6071085929870605,
|
||
|
|
"learning_rate": 9.433598586410701e-07,
|
||
|
|
"logits/chosen": -37060291.43217666,
|
||
|
|
"logits/rejected": -38281992.718266256,
|
||
|
|
"logps/chosen": -487.60636829653,
|
||
|
|
"logps/rejected": -415.375386996904,
|
||
|
|
"loss": 0.5897,
|
||
|
|
"loss/base_kto": 3.7728347778320312,
|
||
|
|
"metrics/advantage_var": 303.0648498535156,
|
||
|
|
"regularization/mmd": 0.9448938369750977,
|
||
|
|
"regularization/rkhs_norm": 182.06048583984375,
|
||
|
|
"rewards/chosen": -0.03856106285793172,
|
||
|
|
"rewards/margins": 0.20288829912627399,
|
||
|
|
"rewards/rejected": -0.2414493619842057,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5958549222797928,
|
||
|
|
"grad_norm": 14.645048141479492,
|
||
|
|
"kl": 11.784172058105469,
|
||
|
|
"learning_rate": 9.366284643057313e-07,
|
||
|
|
"logits/chosen": -34455175.50554675,
|
||
|
|
"logits/rejected": -36888973.60862866,
|
||
|
|
"logps/chosen": -501.4950475435816,
|
||
|
|
"logps/rejected": -367.9532935285054,
|
||
|
|
"loss": 0.586,
|
||
|
|
"loss/base_kto": 3.7162437438964844,
|
||
|
|
"metrics/advantage_var": 269.9901428222656,
|
||
|
|
"regularization/mmd": 0.9714435935020447,
|
||
|
|
"regularization/rkhs_norm": 187.17601013183594,
|
||
|
|
"rewards/chosen": 0.20885306719554775,
|
||
|
|
"rewards/margins": 0.2564124497473552,
|
||
|
|
"rewards/rejected": -0.047559382551807465,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6217616580310881,
|
||
|
|
"grad_norm": 14.725089073181152,
|
||
|
|
"kl": 21.3284969329834,
|
||
|
|
"learning_rate": 9.295460731570917e-07,
|
||
|
|
"logits/chosen": -37677663.61445783,
|
||
|
|
"logits/rejected": -38149319.48051948,
|
||
|
|
"logps/chosen": -484.6805346385542,
|
||
|
|
"logps/rejected": -376.2382305194805,
|
||
|
|
"loss": 0.5839,
|
||
|
|
"loss/base_kto": 3.740070343017578,
|
||
|
|
"metrics/advantage_var": 249.39854431152344,
|
||
|
|
"regularization/mmd": 0.9308084845542908,
|
||
|
|
"regularization/rkhs_norm": 179.34652709960938,
|
||
|
|
"rewards/chosen": 0.28971419276961363,
|
||
|
|
"rewards/margins": 0.22355909971279864,
|
||
|
|
"rewards/rejected": 0.06615509305681501,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6476683937823834,
|
||
|
|
"grad_norm": 15.315522193908691,
|
||
|
|
"kl": 15.955157279968262,
|
||
|
|
"learning_rate": 9.221183785865056e-07,
|
||
|
|
"logits/chosen": -36865574.65639445,
|
||
|
|
"logits/rejected": -38866720.05071315,
|
||
|
|
"logps/chosen": -507.76213405238826,
|
||
|
|
"logps/rejected": -371.5848851030111,
|
||
|
|
"loss": 0.5823,
|
||
|
|
"loss/base_kto": 3.7561357021331787,
|
||
|
|
"metrics/advantage_var": 229.5916748046875,
|
||
|
|
"regularization/mmd": 0.9023582339286804,
|
||
|
|
"regularization/rkhs_norm": 173.8647918701172,
|
||
|
|
"rewards/chosen": 0.2609805913845821,
|
||
|
|
"rewards/margins": 0.19861362819603587,
|
||
|
|
"rewards/rejected": 0.062366963188546204,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6735751295336787,
|
||
|
|
"grad_norm": 17.44818115234375,
|
||
|
|
"kl": 13.880945205688477,
|
||
|
|
"learning_rate": 9.143513515677817e-07,
|
||
|
|
"logits/chosen": -36378504.11707317,
|
||
|
|
"logits/rejected": -39158961.082706764,
|
||
|
|
"logps/chosen": -504.4064024390244,
|
||
|
|
"logps/rejected": -345.63677161654135,
|
||
|
|
"loss": 0.575,
|
||
|
|
"loss/base_kto": 3.706279754638672,
|
||
|
|
"metrics/advantage_var": 242.1262664794922,
|
||
|
|
"regularization/mmd": 0.8934823870658875,
|
||
|
|
"regularization/rkhs_norm": 172.1546173095703,
|
||
|
|
"rewards/chosen": 0.18307581955824442,
|
||
|
|
"rewards/margins": 0.22600504872093385,
|
||
|
|
"rewards/rejected": -0.04292922916268944,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6994818652849741,
|
||
|
|
"grad_norm": 15.78923511505127,
|
||
|
|
"kl": 10.004231452941895,
|
||
|
|
"learning_rate": 9.062512358572373e-07,
|
||
|
|
"logits/chosen": -36161229.58167939,
|
||
|
|
"logits/rejected": -36614009.6512,
|
||
|
|
"logps/chosen": -469.91140267175575,
|
||
|
|
"logps/rejected": -394.6747,
|
||
|
|
"loss": 0.5811,
|
||
|
|
"loss/base_kto": 3.7478103637695312,
|
||
|
|
"metrics/advantage_var": 233.16355895996094,
|
||
|
|
"regularization/mmd": 0.901284396648407,
|
||
|
|
"regularization/rkhs_norm": 173.6578826904297,
|
||
|
|
"rewards/chosen": 0.12188799909052958,
|
||
|
|
"rewards/margins": 0.2359503770202171,
|
||
|
|
"rewards/rejected": -0.1140623779296875,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7253886010362695,
|
||
|
|
"grad_norm": 13.183453559875488,
|
||
|
|
"kl": 9.830798149108887,
|
||
|
|
"learning_rate": 8.978245429744691e-07,
|
||
|
|
"logits/chosen": -35139613.77705977,
|
||
|
|
"logits/rejected": -37245392.750378214,
|
||
|
|
"logps/chosen": -483.6814923263328,
|
||
|
|
"logps/rejected": -362.81415468986387,
|
||
|
|
"loss": 0.5809,
|
||
|
|
"loss/base_kto": 3.744293689727783,
|
||
|
|
"metrics/advantage_var": 230.5392303466797,
|
||
|
|
"regularization/mmd": 0.903019905090332,
|
||
|
|
"regularization/rkhs_norm": 173.9922637939453,
|
||
|
|
"rewards/chosen": 0.14337463871920436,
|
||
|
|
"rewards/margins": 0.2262163070140908,
|
||
|
|
"rewards/rejected": -0.08284166829488641,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7512953367875648,
|
||
|
|
"grad_norm": 18.788639068603516,
|
||
|
|
"kl": 11.212885856628418,
|
||
|
|
"learning_rate": 8.890780469678738e-07,
|
||
|
|
"logits/chosen": -35674151.76699029,
|
||
|
|
"logits/rejected": -36726842.77945619,
|
||
|
|
"logps/chosen": -481.4205097087379,
|
||
|
|
"logps/rejected": -357.83846299093653,
|
||
|
|
"loss": 0.5732,
|
||
|
|
"loss/base_kto": 3.6835312843322754,
|
||
|
|
"metrics/advantage_var": 223.0271453857422,
|
||
|
|
"regularization/mmd": 0.9017006754875183,
|
||
|
|
"regularization/rkhs_norm": 173.7380828857422,
|
||
|
|
"rewards/chosen": 0.14063882673442557,
|
||
|
|
"rewards/margins": 0.2636975261434725,
|
||
|
|
"rewards/rejected": -0.12305869940904693,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7772020725388601,
|
||
|
|
"grad_norm": 14.359786987304688,
|
||
|
|
"kl": 14.062952041625977,
|
||
|
|
"learning_rate": 8.800187789691269e-07,
|
||
|
|
"logits/chosen": -35991686.65443425,
|
||
|
|
"logits/rejected": -37016252.115015976,
|
||
|
|
"logps/chosen": -487.0045871559633,
|
||
|
|
"logps/rejected": -382.6799121405751,
|
||
|
|
"loss": 0.5835,
|
||
|
|
"loss/base_kto": 3.7117087841033936,
|
||
|
|
"metrics/advantage_var": 255.1334991455078,
|
||
|
|
"regularization/mmd": 0.9565412402153015,
|
||
|
|
"regularization/rkhs_norm": 184.3046875,
|
||
|
|
"rewards/chosen": 0.20807981345267107,
|
||
|
|
"rewards/margins": 0.2725569417953696,
|
||
|
|
"rewards/rejected": -0.06447712834269856,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8031088082901554,
|
||
|
|
"grad_norm": 19.544111251831055,
|
||
|
|
"kl": 11.21495532989502,
|
||
|
|
"learning_rate": 8.706540215409981e-07,
|
||
|
|
"logits/chosen": -35774285.04854369,
|
||
|
|
"logits/rejected": -37030008.652567975,
|
||
|
|
"logps/chosen": -495.9582322006473,
|
||
|
|
"logps/rejected": -385.1773508308157,
|
||
|
|
"loss": 0.5783,
|
||
|
|
"loss/base_kto": 3.706045627593994,
|
||
|
|
"metrics/advantage_var": 244.302734375,
|
||
|
|
"regularization/mmd": 0.9202253222465515,
|
||
|
|
"regularization/rkhs_norm": 177.30740356445312,
|
||
|
|
"rewards/chosen": 0.1879773186248483,
|
||
|
|
"rewards/margins": 0.26639470767916107,
|
||
|
|
"rewards/rejected": -0.07841738905431281,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8290155440414507,
|
||
|
|
"grad_norm": 16.922754287719727,
|
||
|
|
"kl": 10.054945945739746,
|
||
|
|
"learning_rate": 8.609913028230462e-07,
|
||
|
|
"logits/chosen": -36098540.517503805,
|
||
|
|
"logits/rejected": -37637108.494382024,
|
||
|
|
"logps/chosen": -464.3390410958904,
|
||
|
|
"logps/rejected": -373.63859349919744,
|
||
|
|
"loss": 0.5812,
|
||
|
|
"loss/base_kto": 3.7624480724334717,
|
||
|
|
"metrics/advantage_var": 218.2515106201172,
|
||
|
|
"regularization/mmd": 0.887394905090332,
|
||
|
|
"regularization/rkhs_norm": 170.98167419433594,
|
||
|
|
"rewards/chosen": 0.10013762638086235,
|
||
|
|
"rewards/margins": 0.22314770149231272,
|
||
|
|
"rewards/rejected": -0.12301007511145039,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8549222797927462,
|
||
|
|
"grad_norm": 19.089248657226562,
|
||
|
|
"kl": 12.878872871398926,
|
||
|
|
"learning_rate": 8.510383904798994e-07,
|
||
|
|
"logits/chosen": -36169354.78101072,
|
||
|
|
"logits/rejected": -36968578.65390749,
|
||
|
|
"logps/chosen": -496.31919984686067,
|
||
|
|
"logps/rejected": -365.0413177830941,
|
||
|
|
"loss": 0.5829,
|
||
|
|
"loss/base_kto": 3.7469775676727295,
|
||
|
|
"metrics/advantage_var": 235.8196258544922,
|
||
|
|
"regularization/mmd": 0.91584312915802,
|
||
|
|
"regularization/rkhs_norm": 176.4630126953125,
|
||
|
|
"rewards/chosen": 0.20673404243784696,
|
||
|
|
"rewards/margins": 0.21996827066630495,
|
||
|
|
"rewards/rejected": -0.013234228228457996,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8808290155440415,
|
||
|
|
"grad_norm": 19.181142807006836,
|
||
|
|
"kl": 11.222612380981445,
|
||
|
|
"learning_rate": 8.408032854569865e-07,
|
||
|
|
"logits/chosen": -36186652.981132075,
|
||
|
|
"logits/rejected": -36443841.98757764,
|
||
|
|
"logps/chosen": -485.46098663522014,
|
||
|
|
"logps/rejected": -365.9584384704969,
|
||
|
|
"loss": 0.5754,
|
||
|
|
"loss/base_kto": 3.7289528846740723,
|
||
|
|
"metrics/advantage_var": 215.1337890625,
|
||
|
|
"regularization/mmd": 0.8744221925735474,
|
||
|
|
"regularization/rkhs_norm": 168.4821014404297,
|
||
|
|
"rewards/chosen": 0.1499655381688532,
|
||
|
|
"rewards/margins": 0.24829562719298826,
|
||
|
|
"rewards/rejected": -0.09833008902413505,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9067357512953368,
|
||
|
|
"grad_norm": 13.612163543701172,
|
||
|
|
"kl": 8.981104850769043,
|
||
|
|
"learning_rate": 8.302942155487377e-07,
|
||
|
|
"logits/chosen": -37335498.06158833,
|
||
|
|
"logits/rejected": -38093476.48868778,
|
||
|
|
"logps/chosen": -490.0928383306321,
|
||
|
|
"logps/rejected": -389.31033182503774,
|
||
|
|
"loss": 0.5832,
|
||
|
|
"loss/base_kto": 3.7175660133361816,
|
||
|
|
"metrics/advantage_var": 278.3384704589844,
|
||
|
|
"regularization/mmd": 0.9483335614204407,
|
||
|
|
"regularization/rkhs_norm": 182.72325134277344,
|
||
|
|
"rewards/chosen": 0.11168753501854994,
|
||
|
|
"rewards/margins": 0.2646296343812067,
|
||
|
|
"rewards/rejected": -0.1529420993626567,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9326424870466321,
|
||
|
|
"grad_norm": 13.148715019226074,
|
||
|
|
"kl": 11.871718406677246,
|
||
|
|
"learning_rate": 8.195196287844278e-07,
|
||
|
|
"logits/chosen": -34459446.35076923,
|
||
|
|
"logits/rejected": -37675073.015873015,
|
||
|
|
"logps/chosen": -461.98649038461537,
|
||
|
|
"logps/rejected": -380.78224206349205,
|
||
|
|
"loss": 0.5842,
|
||
|
|
"loss/base_kto": 3.7333762645721436,
|
||
|
|
"metrics/advantage_var": 250.69200134277344,
|
||
|
|
"regularization/mmd": 0.939959704875946,
|
||
|
|
"regularization/rkhs_norm": 181.1097869873047,
|
||
|
|
"rewards/chosen": 0.21714160625751203,
|
||
|
|
"rewards/margins": 0.20143503380054667,
|
||
|
|
"rewards/rejected": 0.01570657245696537,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9585492227979274,
|
||
|
|
"grad_norm": 14.055625915527344,
|
||
|
|
"kl": 11.139016151428223,
|
||
|
|
"learning_rate": 8.08488186636975e-07,
|
||
|
|
"logits/chosen": -35397003.779141106,
|
||
|
|
"logits/rejected": -37095710.98089172,
|
||
|
|
"logps/chosen": -496.9721050613497,
|
||
|
|
"logps/rejected": -362.6318421576433,
|
||
|
|
"loss": 0.5852,
|
||
|
|
"loss/base_kto": 3.7339775562286377,
|
||
|
|
"metrics/advantage_var": 297.9362487792969,
|
||
|
|
"regularization/mmd": 0.9475700259208679,
|
||
|
|
"regularization/rkhs_norm": 182.5761260986328,
|
||
|
|
"rewards/chosen": 0.17294377051979487,
|
||
|
|
"rewards/margins": 0.21707383463975344,
|
||
|
|
"rewards/rejected": -0.044130064119958574,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9844559585492227,
|
||
|
|
"grad_norm": 14.052969932556152,
|
||
|
|
"kl": 11.488138198852539,
|
||
|
|
"learning_rate": 7.972087570601576e-07,
|
||
|
|
"logits/chosen": -34332419.73419773,
|
||
|
|
"logits/rejected": -35945729.9306184,
|
||
|
|
"logps/chosen": -500.45613857374394,
|
||
|
|
"logps/rejected": -374.53893288084464,
|
||
|
|
"loss": 0.5881,
|
||
|
|
"loss/base_kto": 3.779372453689575,
|
||
|
|
"metrics/advantage_var": 240.10044860839844,
|
||
|
|
"regularization/mmd": 0.9253013730049133,
|
||
|
|
"regularization/rkhs_norm": 178.28543090820312,
|
||
|
|
"rewards/chosen": 0.10344443792655364,
|
||
|
|
"rewards/margins": 0.178723691027471,
|
||
|
|
"rewards/rejected": -0.07527925310091735,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0103626943005182,
|
||
|
|
"grad_norm": 14.608016014099121,
|
||
|
|
"kl": 8.775541305541992,
|
||
|
|
"learning_rate": 7.856904073598458e-07,
|
||
|
|
"logits/chosen": -35473447.63870968,
|
||
|
|
"logits/rejected": -36471443.84194529,
|
||
|
|
"logps/chosen": -517.9454637096774,
|
||
|
|
"logps/rejected": -377.43075607902733,
|
||
|
|
"loss": 0.5784,
|
||
|
|
"loss/base_kto": 3.49992299079895,
|
||
|
|
"metrics/advantage_var": 381.3582458496094,
|
||
|
|
"regularization/mmd": 1.1271765232086182,
|
||
|
|
"regularization/rkhs_norm": 217.18235778808594,
|
||
|
|
"rewards/chosen": 0.1366151625110257,
|
||
|
|
"rewards/margins": 0.5295594984485257,
|
||
|
|
"rewards/rejected": -0.3929443359375,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0362694300518134,
|
||
|
|
"grad_norm": 14.597328186035156,
|
||
|
|
"kl": 7.6913065910339355,
|
||
|
|
"learning_rate": 7.739423969049761e-07,
|
||
|
|
"logits/chosen": -36081680.12598425,
|
||
|
|
"logits/rejected": -36631226.54263566,
|
||
|
|
"logps/chosen": -481.8291338582677,
|
||
|
|
"logps/rejected": -379.1112403100775,
|
||
|
|
"loss": 0.5746,
|
||
|
|
"loss/base_kto": 3.2306125164031982,
|
||
|
|
"metrics/advantage_var": 599.6570434570312,
|
||
|
|
"regularization/mmd": 1.3662827014923096,
|
||
|
|
"regularization/rkhs_norm": 263.2529296875,
|
||
|
|
"rewards/chosen": 0.32224049004982774,
|
||
|
|
"rewards/margins": 0.8935758257147988,
|
||
|
|
"rewards/rejected": -0.571335335664971,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0621761658031088,
|
||
|
|
"grad_norm": 13.44502067565918,
|
||
|
|
"kl": 6.010591506958008,
|
||
|
|
"learning_rate": 7.619741696841322e-07,
|
||
|
|
"logits/chosen": -34946848.58181818,
|
||
|
|
"logits/rejected": -36547967.174193546,
|
||
|
|
"logps/chosen": -480.3958806818182,
|
||
|
|
"logps/rejected": -369.7039314516129,
|
||
|
|
"loss": 0.5776,
|
||
|
|
"loss/base_kto": 3.225162982940674,
|
||
|
|
"metrics/advantage_var": 677.2144165039062,
|
||
|
|
"regularization/mmd": 1.395242691040039,
|
||
|
|
"regularization/rkhs_norm": 268.8329162597656,
|
||
|
|
"rewards/chosen": 0.5056861646247632,
|
||
|
|
"rewards/margins": 0.895577438369408,
|
||
|
|
"rewards/rejected": -0.38989127374464466,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0880829015544042,
|
||
|
|
"grad_norm": 13.183380126953125,
|
||
|
|
"kl": 14.039239883422852,
|
||
|
|
"learning_rate": 7.497953467137135e-07,
|
||
|
|
"logits/chosen": -35152786.57735247,
|
||
|
|
"logits/rejected": -36525863.595712095,
|
||
|
|
"logps/chosen": -478.68052232854865,
|
||
|
|
"logps/rejected": -420.7020960949464,
|
||
|
|
"loss": 0.5678,
|
||
|
|
"loss/base_kto": 3.1883442401885986,
|
||
|
|
"metrics/advantage_var": 562.2630004882812,
|
||
|
|
"regularization/mmd": 1.3538860082626343,
|
||
|
|
"regularization/rkhs_norm": 260.8643493652344,
|
||
|
|
"rewards/chosen": 0.4896285499682267,
|
||
|
|
"rewards/margins": 0.8697491238445069,
|
||
|
|
"rewards/rejected": -0.3801205738762802,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1139896373056994,
|
||
|
|
"grad_norm": 12.155740737915039,
|
||
|
|
"kl": 9.601874351501465,
|
||
|
|
"learning_rate": 7.37415718303793e-07,
|
||
|
|
"logits/chosen": -36049119.5377207,
|
||
|
|
"logits/rejected": -36852672.09741248,
|
||
|
|
"logps/chosen": -459.9356942215088,
|
||
|
|
"logps/rejected": -382.8304794520548,
|
||
|
|
"loss": 0.5667,
|
||
|
|
"loss/base_kto": 3.194615364074707,
|
||
|
|
"metrics/advantage_var": 583.6920166015625,
|
||
|
|
"regularization/mmd": 1.3391153812408447,
|
||
|
|
"regularization/rkhs_norm": 258.01837158203125,
|
||
|
|
"rewards/chosen": 0.46039249426289125,
|
||
|
|
"rewards/margins": 0.9328118962707157,
|
||
|
|
"rewards/rejected": -0.4724194020078244,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1398963730569949,
|
||
|
|
"grad_norm": 15.561748504638672,
|
||
|
|
"kl": 11.203712463378906,
|
||
|
|
"learning_rate": 7.248452361878855e-07,
|
||
|
|
"logits/chosen": -36341135.081571,
|
||
|
|
"logits/rejected": -37225206.88673139,
|
||
|
|
"logps/chosen": -475.67909743202415,
|
||
|
|
"logps/rejected": -379.53094660194176,
|
||
|
|
"loss": 0.569,
|
||
|
|
"loss/base_kto": 3.2016074657440186,
|
||
|
|
"metrics/advantage_var": 608.2510375976562,
|
||
|
|
"regularization/mmd": 1.3505550622940063,
|
||
|
|
"regularization/rkhs_norm": 260.2225341796875,
|
||
|
|
"rewards/chosen": 0.4458746319451001,
|
||
|
|
"rewards/margins": 0.9215175059622167,
|
||
|
|
"rewards/rejected": -0.4756428740171167,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.16580310880829,
|
||
|
|
"grad_norm": 16.223711013793945,
|
||
|
|
"kl": 10.957860946655273,
|
||
|
|
"learning_rate": 7.120940055229497e-07,
|
||
|
|
"logits/chosen": -35770295.42064715,
|
||
|
|
"logits/rejected": -37127124.18383518,
|
||
|
|
"logps/chosen": -480.1949634052388,
|
||
|
|
"logps/rejected": -367.9017432646593,
|
||
|
|
"loss": 0.5646,
|
||
|
|
"loss/base_kto": 3.2067558765411377,
|
||
|
|
"metrics/advantage_var": 560.8873291015625,
|
||
|
|
"regularization/mmd": 1.3101739883422852,
|
||
|
|
"regularization/rkhs_norm": 252.44200134277344,
|
||
|
|
"rewards/chosen": 0.5309256390540735,
|
||
|
|
"rewards/margins": 0.8729425936725487,
|
||
|
|
"rewards/rejected": -0.34201695461847514,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1917098445595855,
|
||
|
|
"grad_norm": 14.827778816223145,
|
||
|
|
"kl": 14.446756362915039,
|
||
|
|
"learning_rate": 6.991722767660556e-07,
|
||
|
|
"logits/chosen": -36632868.571428575,
|
||
|
|
"logits/rejected": -38392632.93312597,
|
||
|
|
"logps/chosen": -484.8189266091052,
|
||
|
|
"logps/rejected": -372.49951399688956,
|
||
|
|
"loss": 0.5655,
|
||
|
|
"loss/base_kto": 3.1839141845703125,
|
||
|
|
"metrics/advantage_var": 580.0728759765625,
|
||
|
|
"regularization/mmd": 1.340128779411316,
|
||
|
|
"regularization/rkhs_norm": 258.2136535644531,
|
||
|
|
"rewards/chosen": 0.534545323538069,
|
||
|
|
"rewards/margins": 0.8950570677272944,
|
||
|
|
"rewards/rejected": -0.3605117441892253,
|
||
|
|
"step": 920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2176165803108807,
|
||
|
|
"grad_norm": 16.71170997619629,
|
||
|
|
"kl": 13.128976821899414,
|
||
|
|
"learning_rate": 6.860904374342499e-07,
|
||
|
|
"logits/chosen": -35226307.43217666,
|
||
|
|
"logits/rejected": -35626532.45820434,
|
||
|
|
"logps/chosen": -490.3660291798107,
|
||
|
|
"logps/rejected": -375.5688370743034,
|
||
|
|
"loss": 0.5745,
|
||
|
|
"loss/base_kto": 3.200298309326172,
|
||
|
|
"metrics/advantage_var": 622.9153442382812,
|
||
|
|
"regularization/mmd": 1.3953193426132202,
|
||
|
|
"regularization/rkhs_norm": 268.8476867675781,
|
||
|
|
"rewards/chosen": 0.5524515819850404,
|
||
|
|
"rewards/margins": 0.9107014943060543,
|
||
|
|
"rewards/rejected": -0.35824991232101394,
|
||
|
|
"step": 940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2435233160621761,
|
||
|
|
"grad_norm": 12.138189315795898,
|
||
|
|
"kl": 11.546676635742188,
|
||
|
|
"learning_rate": 6.7285900375424e-07,
|
||
|
|
"logits/chosen": -35727222.184663534,
|
||
|
|
"logits/rejected": -36486733.47893916,
|
||
|
|
"logps/chosen": -447.2831572769953,
|
||
|
|
"logps/rejected": -371.9009360374415,
|
||
|
|
"loss": 0.5629,
|
||
|
|
"loss/base_kto": 3.164229393005371,
|
||
|
|
"metrics/advantage_var": 560.22900390625,
|
||
|
|
"regularization/mmd": 1.338631510734558,
|
||
|
|
"regularization/rkhs_norm": 257.9251403808594,
|
||
|
|
"rewards/chosen": 0.4996206555194885,
|
||
|
|
"rewards/margins": 0.9092353494334169,
|
||
|
|
"rewards/rejected": -0.4096146939139284,
|
||
|
|
"step": 960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2694300518134716,
|
||
|
|
"grad_norm": 15.995272636413574,
|
||
|
|
"kl": 8.912761688232422,
|
||
|
|
"learning_rate": 6.594886122086123e-07,
|
||
|
|
"logits/chosen": -34464722.05128205,
|
||
|
|
"logits/rejected": -35713276.87804878,
|
||
|
|
"logps/chosen": -498.4085036057692,
|
||
|
|
"logps/rejected": -369.6967654344512,
|
||
|
|
"loss": 0.5657,
|
||
|
|
"loss/base_kto": 3.212599277496338,
|
||
|
|
"metrics/advantage_var": 523.2824096679688,
|
||
|
|
"regularization/mmd": 1.3127588033676147,
|
||
|
|
"regularization/rkhs_norm": 252.94004821777344,
|
||
|
|
"rewards/chosen": 0.4278733669183193,
|
||
|
|
"rewards/margins": 0.8709167733350495,
|
||
|
|
"rewards/rejected": -0.4430434064167302,
|
||
|
|
"step": 980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2953367875647668,
|
||
|
|
"grad_norm": 14.5647611618042,
|
||
|
|
"kl": 15.449335098266602,
|
||
|
|
"learning_rate": 6.459900109853766e-07,
|
||
|
|
"logits/chosen": -35486392.93134329,
|
||
|
|
"logits/rejected": -36879403.64590164,
|
||
|
|
"logps/chosen": -495.8786380597015,
|
||
|
|
"logps/rejected": -389.3173155737705,
|
||
|
|
"loss": 0.5711,
|
||
|
|
"loss/base_kto": 3.253002882003784,
|
||
|
|
"metrics/advantage_var": 533.5360717773438,
|
||
|
|
"regularization/mmd": 1.31588876247406,
|
||
|
|
"regularization/rkhs_norm": 253.5431365966797,
|
||
|
|
"rewards/chosen": 0.5541762622434702,
|
||
|
|
"rewards/margins": 0.7733676414779092,
|
||
|
|
"rewards/rejected": -0.21919137923443904,
|
||
|
|
"step": 1000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3212435233160622,
|
||
|
|
"grad_norm": 11.143204689025879,
|
||
|
|
"kl": 6.068701267242432,
|
||
|
|
"learning_rate": 6.323740513377171e-07,
|
||
|
|
"logits/chosen": -34305420.28482972,
|
||
|
|
"logits/rejected": -36479954.776025236,
|
||
|
|
"logps/chosen": -470.2604972910217,
|
||
|
|
"logps/rejected": -367.3026666009464,
|
||
|
|
"loss": 0.5654,
|
||
|
|
"loss/base_kto": 3.286928653717041,
|
||
|
|
"metrics/advantage_var": 501.8526306152344,
|
||
|
|
"regularization/mmd": 1.2363615036010742,
|
||
|
|
"regularization/rkhs_norm": 238.219970703125,
|
||
|
|
"rewards/chosen": 0.3501069154532701,
|
||
|
|
"rewards/margins": 0.8025213468597665,
|
||
|
|
"rewards/rejected": -0.45241443140649645,
|
||
|
|
"step": 1020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3471502590673574,
|
||
|
|
"grad_norm": 15.586249351501465,
|
||
|
|
"kl": 15.317721366882324,
|
||
|
|
"learning_rate": 6.186516788608865e-07,
|
||
|
|
"logits/chosen": -35702237.86666667,
|
||
|
|
"logits/rejected": -35553676.38709678,
|
||
|
|
"logps/chosen": -501.6375946969697,
|
||
|
|
"logps/rejected": -366.27615927419356,
|
||
|
|
"loss": 0.5631,
|
||
|
|
"loss/base_kto": 3.1214816570281982,
|
||
|
|
"metrics/advantage_var": 617.6486206054688,
|
||
|
|
"regularization/mmd": 1.3834471702575684,
|
||
|
|
"regularization/rkhs_norm": 266.5601501464844,
|
||
|
|
"rewards/chosen": 0.6193990071614583,
|
||
|
|
"rewards/margins": 0.9554227439306116,
|
||
|
|
"rewards/rejected": -0.3360237367691532,
|
||
|
|
"step": 1040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3730569948186528,
|
||
|
|
"grad_norm": 13.858671188354492,
|
||
|
|
"kl": 16.675804138183594,
|
||
|
|
"learning_rate": 6.048339246932652e-07,
|
||
|
|
"logits/chosen": -35363947.70486656,
|
||
|
|
"logits/rejected": -37440886.24572317,
|
||
|
|
"logps/chosen": -506.073489010989,
|
||
|
|
"logps/rejected": -372.9756026438569,
|
||
|
|
"loss": 0.5719,
|
||
|
|
"loss/base_kto": 3.223862409591675,
|
||
|
|
"metrics/advantage_var": 569.9444580078125,
|
||
|
|
"regularization/mmd": 1.3512192964553833,
|
||
|
|
"regularization/rkhs_norm": 260.35052490234375,
|
||
|
|
"rewards/chosen": 0.5281324641109448,
|
||
|
|
"rewards/margins": 0.8282414730898523,
|
||
|
|
"rewards/rejected": -0.30010900897890747,
|
||
|
|
"step": 1060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3989637305699483,
|
||
|
|
"grad_norm": 13.420823097229004,
|
||
|
|
"kl": 15.193489074707031,
|
||
|
|
"learning_rate": 5.909318966486494e-07,
|
||
|
|
"logits/chosen": -34836834.52241113,
|
||
|
|
"logits/rejected": -36498155.37440758,
|
||
|
|
"logps/chosen": -470.3858191653787,
|
||
|
|
"logps/rejected": -374.23514020537124,
|
||
|
|
"loss": 0.5633,
|
||
|
|
"loss/base_kto": 3.220278263092041,
|
||
|
|
"metrics/advantage_var": 524.41015625,
|
||
|
|
"regularization/mmd": 1.2857328653335571,
|
||
|
|
"regularization/rkhs_norm": 247.7327423095703,
|
||
|
|
"rewards/chosen": 0.5431538836849159,
|
||
|
|
"rewards/margins": 0.8286780802192761,
|
||
|
|
"rewards/rejected": -0.2855241965343602,
|
||
|
|
"step": 1080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4248704663212435,
|
||
|
|
"grad_norm": 11.169465065002441,
|
||
|
|
"kl": 8.387614250183105,
|
||
|
|
"learning_rate": 5.769567702869052e-07,
|
||
|
|
"logits/chosen": -34593283.22012579,
|
||
|
|
"logits/rejected": -35831591.7515528,
|
||
|
|
"logps/chosen": -466.9943003144654,
|
||
|
|
"logps/rejected": -374.63994565217394,
|
||
|
|
"loss": 0.5679,
|
||
|
|
"loss/base_kto": 3.2649879455566406,
|
||
|
|
"metrics/advantage_var": 483.4437561035156,
|
||
|
|
"regularization/mmd": 1.2778500318527222,
|
||
|
|
"regularization/rkhs_norm": 246.2138671875,
|
||
|
|
"rewards/chosen": 0.4452309278572131,
|
||
|
|
"rewards/margins": 0.8021823393236338,
|
||
|
|
"rewards/rejected": -0.3569514114664208,
|
||
|
|
"step": 1100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.450777202072539,
|
||
|
|
"grad_norm": 12.55168342590332,
|
||
|
|
"kl": 11.815613746643066,
|
||
|
|
"learning_rate": 5.629197799301614e-07,
|
||
|
|
"logits/chosen": -34086599.155905515,
|
||
|
|
"logits/rejected": -35980746.81550387,
|
||
|
|
"logps/chosen": -475.75600393700785,
|
||
|
|
"logps/rejected": -380.1451065891473,
|
||
|
|
"loss": 0.569,
|
||
|
|
"loss/base_kto": 3.2363297939300537,
|
||
|
|
"metrics/advantage_var": 604.4345092773438,
|
||
|
|
"regularization/mmd": 1.3159685134887695,
|
||
|
|
"regularization/rkhs_norm": 253.5584716796875,
|
||
|
|
"rewards/chosen": 0.44834589920644685,
|
||
|
|
"rewards/margins": 0.8243316879509963,
|
||
|
|
"rewards/rejected": -0.3759857887445494,
|
||
|
|
"step": 1120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4766839378238341,
|
||
|
|
"grad_norm": 15.806407928466797,
|
||
|
|
"kl": 8.803001403808594,
|
||
|
|
"learning_rate": 5.488322096317633e-07,
|
||
|
|
"logits/chosen": -34337923.11183144,
|
||
|
|
"logits/rejected": -37926173.73152338,
|
||
|
|
"logps/chosen": -480.3818881685575,
|
||
|
|
"logps/rejected": -365.2533700980392,
|
||
|
|
"loss": 0.5594,
|
||
|
|
"loss/base_kto": 3.1876957416534424,
|
||
|
|
"metrics/advantage_var": 515.0336303710938,
|
||
|
|
"regularization/mmd": 1.2876020669937134,
|
||
|
|
"regularization/rkhs_norm": 248.0928955078125,
|
||
|
|
"rewards/chosen": 0.4363956296849676,
|
||
|
|
"rewards/margins": 0.8824412420029115,
|
||
|
|
"rewards/rejected": -0.446045612317944,
|
||
|
|
"step": 1140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5025906735751295,
|
||
|
|
"grad_norm": 13.409736633300781,
|
||
|
|
"kl": 11.580401420593262,
|
||
|
|
"learning_rate": 5.347053841052518e-07,
|
||
|
|
"logits/chosen": -34767339.25445705,
|
||
|
|
"logits/rejected": -35640914.63046757,
|
||
|
|
"logps/chosen": -489.12662074554294,
|
||
|
|
"logps/rejected": -372.7496700603318,
|
||
|
|
"loss": 0.5609,
|
||
|
|
"loss/base_kto": 3.2315571308135986,
|
||
|
|
"metrics/advantage_var": 491.1260681152344,
|
||
|
|
"regularization/mmd": 1.2559025287628174,
|
||
|
|
"regularization/rkhs_norm": 241.9850616455078,
|
||
|
|
"rewards/chosen": 0.48537050615060273,
|
||
|
|
"rewards/margins": 0.8391282234615356,
|
||
|
|
"rewards/rejected": -0.3537577173109328,
|
||
|
|
"step": 1160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.528497409326425,
|
||
|
|
"grad_norm": 14.48072624206543,
|
||
|
|
"kl": 10.842148780822754,
|
||
|
|
"learning_rate": 5.205506596206531e-07,
|
||
|
|
"logits/chosen": -34132429.74233129,
|
||
|
|
"logits/rejected": -36569577.171974525,
|
||
|
|
"logps/chosen": -489.0885736196319,
|
||
|
|
"logps/rejected": -384.1238057324841,
|
||
|
|
"loss": 0.5747,
|
||
|
|
"loss/base_kto": 3.2510056495666504,
|
||
|
|
"metrics/advantage_var": 608.0940551757812,
|
||
|
|
"regularization/mmd": 1.3465982675552368,
|
||
|
|
"regularization/rkhs_norm": 259.4601745605469,
|
||
|
|
"rewards/chosen": 0.4769764999670485,
|
||
|
|
"rewards/margins": 0.817667404329235,
|
||
|
|
"rewards/rejected": -0.3406909043621865,
|
||
|
|
"step": 1180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5544041450777202,
|
||
|
|
"grad_norm": 13.27139663696289,
|
||
|
|
"kl": 9.682214736938477,
|
||
|
|
"learning_rate": 5.063794148754032e-07,
|
||
|
|
"logits/chosen": -34232928.9514867,
|
||
|
|
"logits/rejected": -35218769.07332293,
|
||
|
|
"logps/chosen": -471.99897300469485,
|
||
|
|
"logps/rejected": -385.6353110374415,
|
||
|
|
"loss": 0.5618,
|
||
|
|
"loss/base_kto": 3.20005202293396,
|
||
|
|
"metrics/advantage_var": 541.8212890625,
|
||
|
|
"regularization/mmd": 1.2941473722457886,
|
||
|
|
"regularization/rkhs_norm": 249.35403442382812,
|
||
|
|
"rewards/chosen": 0.4603751933257531,
|
||
|
|
"rewards/margins": 0.8822661900593725,
|
||
|
|
"rewards/rejected": -0.42189099673361935,
|
||
|
|
"step": 1200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5803108808290154,
|
||
|
|
"grad_norm": 10.266549110412598,
|
||
|
|
"kl": 8.075851440429688,
|
||
|
|
"learning_rate": 4.922030418472422e-07,
|
||
|
|
"logits/chosen": -34897622.81388012,
|
||
|
|
"logits/rejected": -35043112.421052635,
|
||
|
|
"logps/chosen": -473.31254929022083,
|
||
|
|
"logps/rejected": -412.2219427244582,
|
||
|
|
"loss": 0.571,
|
||
|
|
"loss/base_kto": 3.2248382568359375,
|
||
|
|
"metrics/advantage_var": 638.4703369140625,
|
||
|
|
"regularization/mmd": 1.3433607816696167,
|
||
|
|
"regularization/rkhs_norm": 258.8363952636719,
|
||
|
|
"rewards/chosen": 0.40739460171961256,
|
||
|
|
"rewards/margins": 0.8605293333312127,
|
||
|
|
"rewards/rejected": -0.4531347316116002,
|
||
|
|
"step": 1220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6062176165803108,
|
||
|
|
"grad_norm": 9.42730712890625,
|
||
|
|
"kl": 9.281641960144043,
|
||
|
|
"learning_rate": 4.780329366364336e-07,
|
||
|
|
"logits/chosen": -35156001.96351575,
|
||
|
|
"logits/rejected": -36589816.05908419,
|
||
|
|
"logps/chosen": -495.7744092039801,
|
||
|
|
"logps/rejected": -352.54712887740027,
|
||
|
|
"loss": 0.568,
|
||
|
|
"loss/base_kto": 3.2109742164611816,
|
||
|
|
"metrics/advantage_var": 651.4511108398438,
|
||
|
|
"regularization/mmd": 1.3327293395996094,
|
||
|
|
"regularization/rkhs_norm": 256.7879333496094,
|
||
|
|
"rewards/chosen": 0.383702798466975,
|
||
|
|
"rewards/margins": 0.861857216150113,
|
||
|
|
"rewards/rejected": -0.47815441768313793,
|
||
|
|
"step": 1240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6321243523316062,
|
||
|
|
"grad_norm": 15.535483360290527,
|
||
|
|
"kl": 9.601252555847168,
|
||
|
|
"learning_rate": 4.638804903046684e-07,
|
||
|
|
"logits/chosen": -34077212.746585734,
|
||
|
|
"logits/rejected": -33392696.06441224,
|
||
|
|
"logps/chosen": -507.0222875569044,
|
||
|
|
"logps/rejected": -412.6364985909823,
|
||
|
|
"loss": 0.5663,
|
||
|
|
"loss/base_kto": 3.186985492706299,
|
||
|
|
"metrics/advantage_var": 593.4682006835938,
|
||
|
|
"regularization/mmd": 1.3430308103561401,
|
||
|
|
"regularization/rkhs_norm": 258.7727966308594,
|
||
|
|
"rewards/chosen": 0.4009834717908289,
|
||
|
|
"rewards/margins": 0.904778855999766,
|
||
|
|
"rewards/rejected": -0.5037953842089372,
|
||
|
|
"step": 1260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6580310880829017,
|
||
|
|
"grad_norm": 12.554241180419922,
|
||
|
|
"kl": 11.170215606689453,
|
||
|
|
"learning_rate": 4.497570797180217e-07,
|
||
|
|
"logits/chosen": -34870943.34281201,
|
||
|
|
"logits/rejected": -35789442.57187017,
|
||
|
|
"logps/chosen": -494.6497334123223,
|
||
|
|
"logps/rejected": -384.8082496136012,
|
||
|
|
"loss": 0.5706,
|
||
|
|
"loss/base_kto": 3.247715473175049,
|
||
|
|
"metrics/advantage_var": 565.27490234375,
|
||
|
|
"regularization/mmd": 1.3169947862625122,
|
||
|
|
"regularization/rkhs_norm": 253.75621032714844,
|
||
|
|
"rewards/chosen": 0.446243069183205,
|
||
|
|
"rewards/margins": 0.8005949324252113,
|
||
|
|
"rewards/rejected": -0.35435186324200635,
|
||
|
|
"step": 1280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6839378238341969,
|
||
|
|
"grad_norm": 10.809260368347168,
|
||
|
|
"kl": 11.35492992401123,
|
||
|
|
"learning_rate": 4.3567405840131853e-07,
|
||
|
|
"logits/chosen": -34217530.95757576,
|
||
|
|
"logits/rejected": -36857063.22580645,
|
||
|
|
"logps/chosen": -499.0260890151515,
|
||
|
|
"logps/rejected": -359.84964717741934,
|
||
|
|
"loss": 0.5584,
|
||
|
|
"loss/base_kto": 3.232714891433716,
|
||
|
|
"metrics/advantage_var": 445.6983337402344,
|
||
|
|
"regularization/mmd": 1.234392523765564,
|
||
|
|
"regularization/rkhs_norm": 237.840576171875,
|
||
|
|
"rewards/chosen": 0.4778497869318182,
|
||
|
|
"rewards/margins": 0.8051091705948726,
|
||
|
|
"rewards/rejected": -0.32725938366305446,
|
||
|
|
"step": 1300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.709844559585492,
|
||
|
|
"grad_norm": 13.519683837890625,
|
||
|
|
"kl": 18.970869064331055,
|
||
|
|
"learning_rate": 4.2164274741126506e-07,
|
||
|
|
"logits/chosen": -32272567.87616099,
|
||
|
|
"logits/rejected": -35914635.70977918,
|
||
|
|
"logps/chosen": -501.64014125386996,
|
||
|
|
"logps/rejected": -358.21978509463725,
|
||
|
|
"loss": 0.5773,
|
||
|
|
"loss/base_kto": 3.1998229026794434,
|
||
|
|
"metrics/advantage_var": 968.603515625,
|
||
|
|
"regularization/mmd": 1.4182510375976562,
|
||
|
|
"regularization/rkhs_norm": 273.26611328125,
|
||
|
|
"rewards/chosen": 0.6546994613789183,
|
||
|
|
"rewards/margins": 0.8154330062430313,
|
||
|
|
"rewards/rejected": -0.160733544864113,
|
||
|
|
"step": 1320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7357512953367875,
|
||
|
|
"grad_norm": 11.478642463684082,
|
||
|
|
"kl": 11.4176025390625,
|
||
|
|
"learning_rate": 4.0767442623567856e-07,
|
||
|
|
"logits/chosen": -33646541.587692305,
|
||
|
|
"logits/rejected": -36106067.70793651,
|
||
|
|
"logps/chosen": -487.5895192307692,
|
||
|
|
"logps/rejected": -366.80401785714287,
|
||
|
|
"loss": 0.5602,
|
||
|
|
"loss/base_kto": 3.179579496383667,
|
||
|
|
"metrics/advantage_var": 534.3556518554688,
|
||
|
|
"regularization/mmd": 1.302286982536316,
|
||
|
|
"regularization/rkhs_norm": 250.92234802246094,
|
||
|
|
"rewards/chosen": 0.5370872614933895,
|
||
|
|
"rewards/margins": 0.8848675194298974,
|
||
|
|
"rewards/rejected": -0.34778025793650796,
|
||
|
|
"step": 1340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.761658031088083,
|
||
|
|
"grad_norm": 14.537354469299316,
|
||
|
|
"kl": 11.186471939086914,
|
||
|
|
"learning_rate": 3.937803237261357e-07,
|
||
|
|
"logits/chosen": -35295880.26456693,
|
||
|
|
"logits/rejected": -36673574.10232558,
|
||
|
|
"logps/chosen": -475.64936023622045,
|
||
|
|
"logps/rejected": -375.23214631782946,
|
||
|
|
"loss": 0.565,
|
||
|
|
"loss/base_kto": 3.2556397914886475,
|
||
|
|
"metrics/advantage_var": 500.22979736328125,
|
||
|
|
"regularization/mmd": 1.2640533447265625,
|
||
|
|
"regularization/rkhs_norm": 243.55557250976562,
|
||
|
|
"rewards/chosen": 0.43695981483759844,
|
||
|
|
"rewards/margins": 0.8345817621335044,
|
||
|
|
"rewards/rejected": -0.397621947295906,
|
||
|
|
"step": 1360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7875647668393784,
|
||
|
|
"grad_norm": 14.746607780456543,
|
||
|
|
"kl": 16.326108932495117,
|
||
|
|
"learning_rate": 3.7997160907132456e-07,
|
||
|
|
"logits/chosen": -35028851.63862928,
|
||
|
|
"logits/rejected": -35513321.52978057,
|
||
|
|
"logps/chosen": -484.29994158878503,
|
||
|
|
"logps/rejected": -359.69080623040753,
|
||
|
|
"loss": 0.5635,
|
||
|
|
"loss/base_kto": 3.145965576171875,
|
||
|
|
"metrics/advantage_var": 570.2919311523438,
|
||
|
|
"regularization/mmd": 1.3619108200073242,
|
||
|
|
"regularization/rkhs_norm": 262.41058349609375,
|
||
|
|
"rewards/chosen": 0.6024724181938765,
|
||
|
|
"rewards/margins": 0.941774935597506,
|
||
|
|
"rewards/rejected": -0.3393025174036295,
|
||
|
|
"step": 1380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8134715025906736,
|
||
|
|
"grad_norm": 10.006362915039062,
|
||
|
|
"kl": 11.556924819946289,
|
||
|
|
"learning_rate": 3.662593828183601e-07,
|
||
|
|
"logits/chosen": -34959923.36569579,
|
||
|
|
"logits/rejected": -35546882.32024169,
|
||
|
|
"logps/chosen": -483.86792071197414,
|
||
|
|
"logps/rejected": -388.843773602719,
|
||
|
|
"loss": 0.5568,
|
||
|
|
"loss/base_kto": 3.178264617919922,
|
||
|
|
"metrics/advantage_var": 510.99249267578125,
|
||
|
|
"regularization/mmd": 1.2762376070022583,
|
||
|
|
"regularization/rkhs_norm": 245.90322875976562,
|
||
|
|
"rewards/chosen": 0.48347492897009003,
|
||
|
|
"rewards/margins": 0.8942438354856206,
|
||
|
|
"rewards/rejected": -0.4107689065155306,
|
||
|
|
"step": 1400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8393782383419688,
|
||
|
|
"grad_norm": 14.284242630004883,
|
||
|
|
"kl": 13.512750625610352,
|
||
|
|
"learning_rate": 3.5265466794927725e-07,
|
||
|
|
"logits/chosen": -33485100.710973725,
|
||
|
|
"logits/rejected": -33434664.44233807,
|
||
|
|
"logps/chosen": -455.9562403400309,
|
||
|
|
"logps/rejected": -382.1572126777251,
|
||
|
|
"loss": 0.5555,
|
||
|
|
"loss/base_kto": 3.187201738357544,
|
||
|
|
"metrics/advantage_var": 497.49542236328125,
|
||
|
|
"regularization/mmd": 1.256669282913208,
|
||
|
|
"regularization/rkhs_norm": 242.1328125,
|
||
|
|
"rewards/chosen": 0.4644394746335249,
|
||
|
|
"rewards/margins": 0.8592076818159646,
|
||
|
|
"rewards/rejected": -0.39476820718243977,
|
||
|
|
"step": 1420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8652849740932642,
|
||
|
|
"grad_norm": 21.990886688232422,
|
||
|
|
"kl": 13.477426528930664,
|
||
|
|
"learning_rate": 3.3916840101987887e-07,
|
||
|
|
"logits/chosen": -35016842.24803767,
|
||
|
|
"logits/rejected": -36510769.36858476,
|
||
|
|
"logps/chosen": -492.2745290423862,
|
||
|
|
"logps/rejected": -378.68818040435457,
|
||
|
|
"loss": 0.5612,
|
||
|
|
"loss/base_kto": 3.203294515609741,
|
||
|
|
"metrics/advantage_var": 535.4396362304688,
|
||
|
|
"regularization/mmd": 1.2866798639297485,
|
||
|
|
"regularization/rkhs_norm": 247.9152069091797,
|
||
|
|
"rewards/chosen": 0.5291542999411303,
|
||
|
|
"rewards/margins": 0.8422502232862896,
|
||
|
|
"rewards/rejected": -0.3130959233451594,
|
||
|
|
"step": 1440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8911917098445596,
|
||
|
|
"grad_norm": 14.122488975524902,
|
||
|
|
"kl": 14.161651611328125,
|
||
|
|
"learning_rate": 3.258114233680583e-07,
|
||
|
|
"logits/chosen": -34305510.4,
|
||
|
|
"logits/rejected": -35749548.8,
|
||
|
|
"logps/chosen": -478.6404296875,
|
||
|
|
"logps/rejected": -374.73798828125,
|
||
|
|
"loss": 0.5718,
|
||
|
|
"loss/base_kto": 3.222114324569702,
|
||
|
|
"metrics/advantage_var": 584.7927856445312,
|
||
|
|
"regularization/mmd": 1.3520044088363647,
|
||
|
|
"regularization/rkhs_norm": 260.5018005371094,
|
||
|
|
"rewards/chosen": 0.48429231643676757,
|
||
|
|
"rewards/margins": 0.8469077110290527,
|
||
|
|
"rewards/rejected": -0.36261539459228515,
|
||
|
|
"step": 1460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.917098445595855,
|
||
|
|
"grad_norm": 11.841097831726074,
|
||
|
|
"kl": 16.10405921936035,
|
||
|
|
"learning_rate": 3.1259447239866796e-07,
|
||
|
|
"logits/chosen": -35767966.02469136,
|
||
|
|
"logits/rejected": -35548892.35443038,
|
||
|
|
"logps/chosen": -462.54591049382714,
|
||
|
|
"logps/rejected": -370.9122577136076,
|
||
|
|
"loss": 0.5642,
|
||
|
|
"loss/base_kto": 3.18354868888855,
|
||
|
|
"metrics/advantage_var": 535.4041748046875,
|
||
|
|
"regularization/mmd": 1.3298248052597046,
|
||
|
|
"regularization/rkhs_norm": 256.228271484375,
|
||
|
|
"rewards/chosen": 0.5445183177053192,
|
||
|
|
"rewards/margins": 0.877083842615389,
|
||
|
|
"rewards/rejected": -0.33256552491006974,
|
||
|
|
"step": 1480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9430051813471503,
|
||
|
|
"grad_norm": 11.20692253112793,
|
||
|
|
"kl": 14.479681968688965,
|
||
|
|
"learning_rate": 2.9952817295193435e-07,
|
||
|
|
"logits/chosen": -33587540.29268292,
|
||
|
|
"logits/rejected": -34997740.307692304,
|
||
|
|
"logps/chosen": -472.9408346036585,
|
||
|
|
"logps/rejected": -375.51016626602564,
|
||
|
|
"loss": 0.5548,
|
||
|
|
"loss/base_kto": 3.1923911571502686,
|
||
|
|
"metrics/advantage_var": 489.254638671875,
|
||
|
|
"regularization/mmd": 1.2462199926376343,
|
||
|
|
"regularization/rkhs_norm": 240.1194305419922,
|
||
|
|
"rewards/chosen": 0.6118030082888719,
|
||
|
|
"rewards/margins": 0.8673452498988854,
|
||
|
|
"rewards/rejected": -0.25554224161001354,
|
||
|
|
"step": 1500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9689119170984455,
|
||
|
|
"grad_norm": 11.861700057983398,
|
||
|
|
"kl": 12.703452110290527,
|
||
|
|
"learning_rate": 2.866230287623651e-07,
|
||
|
|
"logits/chosen": -34238559.639291465,
|
||
|
|
"logits/rejected": -36946728.0121396,
|
||
|
|
"logps/chosen": -497.42340982286635,
|
||
|
|
"logps/rejected": -356.5920428679818,
|
||
|
|
"loss": 0.5542,
|
||
|
|
"loss/base_kto": 3.2080001831054688,
|
||
|
|
"metrics/advantage_var": 460.76611328125,
|
||
|
|
"regularization/mmd": 1.2256393432617188,
|
||
|
|
"regularization/rkhs_norm": 236.154052734375,
|
||
|
|
"rewards/chosen": 0.526352149852808,
|
||
|
|
"rewards/margins": 0.8376644186786806,
|
||
|
|
"rewards/rejected": -0.31131226882587254,
|
||
|
|
"step": 1520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.994818652849741,
|
||
|
|
"grad_norm": 10.996031761169434,
|
||
|
|
"kl": 11.139498710632324,
|
||
|
|
"learning_rate": 2.738894140150075e-07,
|
||
|
|
"logits/chosen": -33521878.913580246,
|
||
|
|
"logits/rejected": -34998809.92405064,
|
||
|
|
"logps/chosen": -442.2758487654321,
|
||
|
|
"logps/rejected": -375.3325257120253,
|
||
|
|
"loss": 0.5695,
|
||
|
|
"loss/base_kto": 3.2456352710723877,
|
||
|
|
"metrics/advantage_var": 592.4486083984375,
|
||
|
|
"regularization/mmd": 1.3107349872589111,
|
||
|
|
"regularization/rkhs_norm": 252.5500946044922,
|
||
|
|
"rewards/chosen": 0.4672075907389323,
|
||
|
|
"rewards/margins": 0.8383880164552842,
|
||
|
|
"rewards/rejected": -0.37118042571635185,
|
||
|
|
"step": 1540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0207253886010363,
|
||
|
|
"grad_norm": 12.017375946044922,
|
||
|
|
"kl": 14.11705493927002,
|
||
|
|
"learning_rate": 2.6133756500585156e-07,
|
||
|
|
"logits/chosen": -34685571.75244299,
|
||
|
|
"logits/rejected": -34988796.915662654,
|
||
|
|
"logps/chosen": -450.93622760586317,
|
||
|
|
"logps/rejected": -354.99595256024094,
|
||
|
|
"loss": 0.5464,
|
||
|
|
"loss/base_kto": 3.1940011978149414,
|
||
|
|
"metrics/advantage_var": 723.1547241210938,
|
||
|
|
"regularization/mmd": 1.1770918369293213,
|
||
|
|
"regularization/rkhs_norm": 226.79995727539062,
|
||
|
|
"rewards/chosen": 0.4711724197437398,
|
||
|
|
"rewards/margins": 0.7959710496883228,
|
||
|
|
"rewards/rejected": -0.32479862994458303,
|
||
|
|
"step": 1560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0466321243523318,
|
||
|
|
"grad_norm": 10.234877586364746,
|
||
|
|
"kl": 8.539804458618164,
|
||
|
|
"learning_rate": 2.489775719130767e-07,
|
||
|
|
"logits/chosen": -34224039.09457365,
|
||
|
|
"logits/rejected": -35279921.99055118,
|
||
|
|
"logps/chosen": -482.2390503875969,
|
||
|
|
"logps/rejected": -387.03179133858265,
|
||
|
|
"loss": 0.5327,
|
||
|
|
"loss/base_kto": 3.124056577682495,
|
||
|
|
"metrics/advantage_var": 364.8231201171875,
|
||
|
|
"regularization/mmd": 1.1374704837799072,
|
||
|
|
"regularization/rkhs_norm": 219.16580200195312,
|
||
|
|
"rewards/chosen": 0.47338261567344964,
|
||
|
|
"rewards/margins": 0.930632102401427,
|
||
|
|
"rewards/rejected": -0.4572494867279774,
|
||
|
|
"step": 1580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0725388601036268,
|
||
|
|
"grad_norm": 9.447223663330078,
|
||
|
|
"kl": 7.858912944793701,
|
||
|
|
"learning_rate": 2.3681937068576303e-07,
|
||
|
|
"logits/chosen": -34184481.320634924,
|
||
|
|
"logits/rejected": -36332840.17230769,
|
||
|
|
"logps/chosen": -487.9683531746032,
|
||
|
|
"logps/rejected": -358.2747355769231,
|
||
|
|
"loss": 0.5312,
|
||
|
|
"loss/base_kto": 3.1154003143310547,
|
||
|
|
"metrics/advantage_var": 365.167724609375,
|
||
|
|
"regularization/mmd": 1.1340020895004272,
|
||
|
|
"regularization/rkhs_norm": 218.4975128173828,
|
||
|
|
"rewards/chosen": 0.4642588781932044,
|
||
|
|
"rewards/margins": 0.921900385855464,
|
||
|
|
"rewards/rejected": -0.45764150766225964,
|
||
|
|
"step": 1600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.098445595854922,
|
||
|
|
"grad_norm": 11.296177864074707,
|
||
|
|
"kl": 8.442159652709961,
|
||
|
|
"learning_rate": 2.2487273505658e-07,
|
||
|
|
"logits/chosen": -34541226.139103554,
|
||
|
|
"logits/rejected": -35661176.92259084,
|
||
|
|
"logps/chosen": -465.1571193972179,
|
||
|
|
"logps/rejected": -385.75661532385465,
|
||
|
|
"loss": 0.5339,
|
||
|
|
"loss/base_kto": 3.1327946186065674,
|
||
|
|
"metrics/advantage_var": 385.8008728027344,
|
||
|
|
"regularization/mmd": 1.1381915807724,
|
||
|
|
"regularization/rkhs_norm": 219.3047332763672,
|
||
|
|
"rewards/chosen": 0.4784832678769803,
|
||
|
|
"rewards/margins": 0.914495331046535,
|
||
|
|
"rewards/rejected": -0.4360120631695547,
|
||
|
|
"step": 1620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1243523316062176,
|
||
|
|
"grad_norm": 9.192686080932617,
|
||
|
|
"kl": 9.473230361938477,
|
||
|
|
"learning_rate": 2.131472686848817e-07,
|
||
|
|
"logits/chosen": -34542063.1880597,
|
||
|
|
"logits/rejected": -35455271.44918033,
|
||
|
|
"logps/chosen": -486.3051305970149,
|
||
|
|
"logps/rejected": -387.897925204918,
|
||
|
|
"loss": 0.5317,
|
||
|
|
"loss/base_kto": 3.0982487201690674,
|
||
|
|
"metrics/advantage_var": 408.5754089355469,
|
||
|
|
"regularization/mmd": 1.1551413536071777,
|
||
|
|
"regularization/rkhs_norm": 222.5706024169922,
|
||
|
|
"rewards/chosen": 0.4941988816901819,
|
||
|
|
"rewards/margins": 0.9388071260389268,
|
||
|
|
"rewards/rejected": -0.44460824434874485,
|
||
|
|
"step": 1640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.150259067357513,
|
||
|
|
"grad_norm": 8.646968841552734,
|
||
|
|
"kl": 15.39623737335205,
|
||
|
|
"learning_rate": 2.016523974365188e-07,
|
||
|
|
"logits/chosen": -32979905.840255592,
|
||
|
|
"logits/rejected": -36205458.39755352,
|
||
|
|
"logps/chosen": -466.7436102236422,
|
||
|
|
"logps/rejected": -366.65125668960246,
|
||
|
|
"loss": 0.5299,
|
||
|
|
"loss/base_kto": 3.0637404918670654,
|
||
|
|
"metrics/advantage_var": 391.736572265625,
|
||
|
|
"regularization/mmd": 1.1757006645202637,
|
||
|
|
"regularization/rkhs_norm": 226.5319061279297,
|
||
|
|
"rewards/chosen": 0.6177439339244708,
|
||
|
|
"rewards/margins": 0.9501236629358432,
|
||
|
|
"rewards/rejected": -0.3323797290113723,
|
||
|
|
"step": 1660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1761658031088085,
|
||
|
|
"grad_norm": 11.392359733581543,
|
||
|
|
"kl": 12.8080472946167,
|
||
|
|
"learning_rate": 1.9039736180657372e-07,
|
||
|
|
"logits/chosen": -34391432.91259259,
|
||
|
|
"logits/rejected": -34560064.31735537,
|
||
|
|
"logps/chosen": -487.67699074074073,
|
||
|
|
"logps/rejected": -379.25206611570246,
|
||
|
|
"loss": 0.5266,
|
||
|
|
"loss/base_kto": 3.076906204223633,
|
||
|
|
"metrics/advantage_var": 373.26593017578125,
|
||
|
|
"regularization/mmd": 1.1360769271850586,
|
||
|
|
"regularization/rkhs_norm": 218.8972625732422,
|
||
|
|
"rewards/chosen": 0.5498810944733796,
|
||
|
|
"rewards/margins": 0.9218989470043714,
|
||
|
|
"rewards/rejected": -0.37201785253099173,
|
||
|
|
"step": 1680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2020725388601035,
|
||
|
|
"grad_norm": 12.47360897064209,
|
||
|
|
"kl": 16.97549819946289,
|
||
|
|
"learning_rate": 1.7939120949111498e-07,
|
||
|
|
"logits/chosen": -34634272.405063294,
|
||
|
|
"logits/rejected": -35292596.14814815,
|
||
|
|
"logps/chosen": -485.03060719936707,
|
||
|
|
"logps/rejected": -370.6174527391975,
|
||
|
|
"loss": 0.5237,
|
||
|
|
"loss/base_kto": 3.0509536266326904,
|
||
|
|
"metrics/advantage_var": 388.0455322265625,
|
||
|
|
"regularization/mmd": 1.138675332069397,
|
||
|
|
"regularization/rkhs_norm": 219.39794921875,
|
||
|
|
"rewards/chosen": 0.6157732130605963,
|
||
|
|
"rewards/margins": 0.9683371417353648,
|
||
|
|
"rewards/rejected": -0.35256392867476855,
|
||
|
|
"step": 1700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.227979274611399,
|
||
|
|
"grad_norm": 9.188547134399414,
|
||
|
|
"kl": 11.431300163269043,
|
||
|
|
"learning_rate": 1.6864278811393523e-07,
|
||
|
|
"logits/chosen": -33846622.01589825,
|
||
|
|
"logits/rejected": -36931958.3655914,
|
||
|
|
"logps/chosen": -496.81508346581876,
|
||
|
|
"logps/rejected": -351.578125,
|
||
|
|
"loss": 0.5248,
|
||
|
|
"loss/base_kto": 3.1210286617279053,
|
||
|
|
"metrics/advantage_var": 336.357666015625,
|
||
|
|
"regularization/mmd": 1.077267050743103,
|
||
|
|
"regularization/rkhs_norm": 207.56591796875,
|
||
|
|
"rewards/chosen": 0.49792203918359995,
|
||
|
|
"rewards/margins": 0.8834565805817494,
|
||
|
|
"rewards/rejected": -0.38553454139814947,
|
||
|
|
"step": 1720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2538860103626943,
|
||
|
|
"grad_norm": 9.713398933410645,
|
||
|
|
"kl": 11.883028984069824,
|
||
|
|
"learning_rate": 1.5816073811412584e-07,
|
||
|
|
"logits/chosen": -33889109.0719755,
|
||
|
|
"logits/rejected": -34294579.85326954,
|
||
|
|
"logps/chosen": -480.3625574272588,
|
||
|
|
"logps/rejected": -387.71730462519935,
|
||
|
|
"loss": 0.5246,
|
||
|
|
"loss/base_kto": 3.0617434978485107,
|
||
|
|
"metrics/advantage_var": 368.1744689941406,
|
||
|
|
"regularization/mmd": 1.1350570917129517,
|
||
|
|
"regularization/rkhs_norm": 218.70079040527344,
|
||
|
|
"rewards/chosen": 0.5901486453747129,
|
||
|
|
"rewards/margins": 0.9540277868096212,
|
||
|
|
"rewards/rejected": -0.3638791414349083,
|
||
|
|
"step": 1740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2797927461139897,
|
||
|
|
"grad_norm": 11.076713562011719,
|
||
|
|
"kl": 12.0452880859375,
|
||
|
|
"learning_rate": 1.4795348580020207e-07,
|
||
|
|
"logits/chosen": -33272275.2,
|
||
|
|
"logits/rejected": -35163008.0,
|
||
|
|
"logps/chosen": -503.425390625,
|
||
|
|
"logps/rejected": -401.28505859375,
|
||
|
|
"loss": 0.5246,
|
||
|
|
"loss/base_kto": 3.06697940826416,
|
||
|
|
"metrics/advantage_var": 361.0828857421875,
|
||
|
|
"regularization/mmd": 1.1299915313720703,
|
||
|
|
"regularization/rkhs_norm": 217.72476196289062,
|
||
|
|
"rewards/chosen": 0.5445161819458008,
|
||
|
|
"rewards/margins": 0.9467146396636963,
|
||
|
|
"rewards/rejected": -0.40219845771789553,
|
||
|
|
"step": 1760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.305699481865285,
|
||
|
|
"grad_norm": 9.097661972045898,
|
||
|
|
"kl": 14.0631742477417,
|
||
|
|
"learning_rate": 1.3802923657636355e-07,
|
||
|
|
"logits/chosen": -35050913.42675159,
|
||
|
|
"logits/rejected": -35578766.9202454,
|
||
|
|
"logps/chosen": -496.2229796974522,
|
||
|
|
"logps/rejected": -370.4764426763804,
|
||
|
|
"loss": 0.5253,
|
||
|
|
"loss/base_kto": 3.037888288497925,
|
||
|
|
"metrics/advantage_var": 412.54736328125,
|
||
|
|
"regularization/mmd": 1.1644972562789917,
|
||
|
|
"regularization/rkhs_norm": 224.37327575683594,
|
||
|
|
"rewards/chosen": 0.5647124272243232,
|
||
|
|
"rewards/margins": 0.9833096895541737,
|
||
|
|
"rewards/rejected": -0.41859726232985045,
|
||
|
|
"step": 1780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.33160621761658,
|
||
|
|
"grad_norm": 10.079657554626465,
|
||
|
|
"kl": 13.203523635864258,
|
||
|
|
"learning_rate": 1.28395968346336e-07,
|
||
|
|
"logits/chosen": -33831678.38993711,
|
||
|
|
"logits/rejected": -36057851.229813665,
|
||
|
|
"logps/chosen": -480.44192216981133,
|
||
|
|
"logps/rejected": -372.98456909937886,
|
||
|
|
"loss": 0.5271,
|
||
|
|
"loss/base_kto": 3.1184487342834473,
|
||
|
|
"metrics/advantage_var": 335.4075012207031,
|
||
|
|
"regularization/mmd": 1.0987217426300049,
|
||
|
|
"regularization/rkhs_norm": 211.69973754882812,
|
||
|
|
"rewards/chosen": 0.5416550546322229,
|
||
|
|
"rewards/margins": 0.8963459381028411,
|
||
|
|
"rewards/rejected": -0.3546908834706182,
|
||
|
|
"step": 1800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.3575129533678756,
|
||
|
|
"grad_norm": 8.813055992126465,
|
||
|
|
"kl": 14.720227241516113,
|
||
|
|
"learning_rate": 1.1906142510009415e-07,
|
||
|
|
"logits/chosen": -33851448.79608483,
|
||
|
|
"logits/rejected": -36096592.59970015,
|
||
|
|
"logps/chosen": -462.68943719412727,
|
||
|
|
"logps/rejected": -380.2259651424288,
|
||
|
|
"loss": 0.5219,
|
||
|
|
"loss/base_kto": 3.045072317123413,
|
||
|
|
"metrics/advantage_var": 368.2608947753906,
|
||
|
|
"regularization/mmd": 1.1300747394561768,
|
||
|
|
"regularization/rkhs_norm": 217.7407684326172,
|
||
|
|
"rewards/chosen": 0.5551960433287877,
|
||
|
|
"rewards/margins": 0.9650930540519418,
|
||
|
|
"rewards/rejected": -0.40989701072315404,
|
||
|
|
"step": 1820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.383419689119171,
|
||
|
|
"grad_norm": 7.799367904663086,
|
||
|
|
"kl": 13.704968452453613,
|
||
|
|
"learning_rate": 1.1003311068862547e-07,
|
||
|
|
"logits/chosen": -33670361.403076924,
|
||
|
|
"logits/rejected": -35210587.83492064,
|
||
|
|
"logps/chosen": -476.4866826923077,
|
||
|
|
"logps/rejected": -369.2220238095238,
|
||
|
|
"loss": 0.5257,
|
||
|
|
"loss/base_kto": 3.0902602672576904,
|
||
|
|
"metrics/advantage_var": 345.37432861328125,
|
||
|
|
"regularization/mmd": 1.1155065298080444,
|
||
|
|
"regularization/rkhs_norm": 214.933837890625,
|
||
|
|
"rewards/chosen": 0.5661159104567308,
|
||
|
|
"rewards/margins": 0.9428646645644937,
|
||
|
|
"rewards/rejected": -0.3767487541077629,
|
||
|
|
"step": 1840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4093264248704664,
|
||
|
|
"grad_norm": 10.140314102172852,
|
||
|
|
"kl": 13.442036628723145,
|
||
|
|
"learning_rate": 1.0131828279173588e-07,
|
||
|
|
"logits/chosen": -33691455.426865675,
|
||
|
|
"logits/rejected": -36494064.05245902,
|
||
|
|
"logps/chosen": -480.81194029850747,
|
||
|
|
"logps/rejected": -392.9032786885246,
|
||
|
|
"loss": 0.5286,
|
||
|
|
"loss/base_kto": 3.090826988220215,
|
||
|
|
"metrics/advantage_var": 369.5147399902344,
|
||
|
|
"regularization/mmd": 1.1379073858261108,
|
||
|
|
"regularization/rkhs_norm": 219.2499542236328,
|
||
|
|
"rewards/chosen": 0.5957055846256997,
|
||
|
|
"rewards/margins": 0.9382153702622262,
|
||
|
|
"rewards/rejected": -0.3425097856365266,
|
||
|
|
"step": 1860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4352331606217614,
|
||
|
|
"grad_norm": 10.38778305053711,
|
||
|
|
"kl": 17.994678497314453,
|
||
|
|
"learning_rate": 9.292394708374596e-08,
|
||
|
|
"logits/chosen": -33399107.53870458,
|
||
|
|
"logits/rejected": -34088193.978361666,
|
||
|
|
"logps/chosen": -497.71800947867297,
|
||
|
|
"logps/rejected": -372.41344667697064,
|
||
|
|
"loss": 0.5246,
|
||
|
|
"loss/base_kto": 3.0352418422698975,
|
||
|
|
"metrics/advantage_var": 384.9737548828125,
|
||
|
|
"regularization/mmd": 1.1616432666778564,
|
||
|
|
"regularization/rkhs_norm": 223.82334899902344,
|
||
|
|
"rewards/chosen": 0.641728646758985,
|
||
|
|
"rewards/margins": 0.9633187106883585,
|
||
|
|
"rewards/rejected": -0.32159006392937356,
|
||
|
|
"step": 1880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.461139896373057,
|
||
|
|
"grad_norm": 10.558876037597656,
|
||
|
|
"kl": 10.543036460876465,
|
||
|
|
"learning_rate": 8.48568516017727e-08,
|
||
|
|
"logits/chosen": -34688667.82608695,
|
||
|
|
"logits/rejected": -34654745.89233279,
|
||
|
|
"logps/chosen": -485.84623313343326,
|
||
|
|
"logps/rejected": -389.13116843393146,
|
||
|
|
"loss": 0.5343,
|
||
|
|
"loss/base_kto": 3.105327844619751,
|
||
|
|
"metrics/advantage_var": 405.47283935546875,
|
||
|
|
"regularization/mmd": 1.168981909751892,
|
||
|
|
"regularization/rkhs_norm": 225.2373809814453,
|
||
|
|
"rewards/chosen": 0.576544399919181,
|
||
|
|
"rewards/margins": 0.94182358049881,
|
||
|
|
"rewards/rejected": -0.3652791805796289,
|
||
|
|
"step": 1900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4870466321243523,
|
||
|
|
"grad_norm": 10.592312812805176,
|
||
|
|
"kl": 11.316069602966309,
|
||
|
|
"learning_rate": 7.71234813211169e-08,
|
||
|
|
"logits/chosen": -33639102.45410628,
|
||
|
|
"logits/rejected": -35165608.20637329,
|
||
|
|
"logps/chosen": -465.92391304347825,
|
||
|
|
"logps/rejected": -374.7020106221548,
|
||
|
|
"loss": 0.5247,
|
||
|
|
"loss/base_kto": 3.09100341796875,
|
||
|
|
"metrics/advantage_var": 351.59423828125,
|
||
|
|
"regularization/mmd": 1.106646180152893,
|
||
|
|
"regularization/rkhs_norm": 213.2266387939453,
|
||
|
|
"rewards/chosen": 0.5346256077961453,
|
||
|
|
"rewards/margins": 0.9385559013872873,
|
||
|
|
"rewards/rejected": -0.4039302935911419,
|
||
|
|
"step": 1920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5129533678756477,
|
||
|
|
"grad_norm": 10.205567359924316,
|
||
|
|
"kl": 11.071531295776367,
|
||
|
|
"learning_rate": 6.973005294212353e-08,
|
||
|
|
"logits/chosen": -35240677.3006135,
|
||
|
|
"logits/rejected": -35846088.560509555,
|
||
|
|
"logps/chosen": -466.9605061349693,
|
||
|
|
"logps/rejected": -395.4638236464968,
|
||
|
|
"loss": 0.5236,
|
||
|
|
"loss/base_kto": 3.1130788326263428,
|
||
|
|
"metrics/advantage_var": 335.3868713378906,
|
||
|
|
"regularization/mmd": 1.075851321220398,
|
||
|
|
"regularization/rkhs_norm": 207.29312133789062,
|
||
|
|
"rewards/chosen": 0.48531617708732744,
|
||
|
|
"rewards/margins": 0.9047721244773552,
|
||
|
|
"rewards/rejected": -0.41945594739002784,
|
||
|
|
"step": 1940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.538860103626943,
|
||
|
|
"grad_norm": 10.455536842346191,
|
||
|
|
"kl": 12.735307693481445,
|
||
|
|
"learning_rate": 6.268250989270102e-08,
|
||
|
|
"logits/chosen": -35146073.681528665,
|
||
|
|
"logits/rejected": -36103990.969325155,
|
||
|
|
"logps/chosen": -486.9891520700637,
|
||
|
|
"logps/rejected": -376.17575728527606,
|
||
|
|
"loss": 0.5325,
|
||
|
|
"loss/base_kto": 3.1449241638183594,
|
||
|
|
"metrics/advantage_var": 377.4090270996094,
|
||
|
|
"regularization/mmd": 1.1152721643447876,
|
||
|
|
"regularization/rkhs_norm": 214.888671875,
|
||
|
|
"rewards/chosen": 0.5238154708959495,
|
||
|
|
"rewards/margins": 0.8699280088650589,
|
||
|
|
"rewards/rejected": -0.34611253796910946,
|
||
|
|
"step": 1960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5647668393782386,
|
||
|
|
"grad_norm": 8.617021560668945,
|
||
|
|
"kl": 11.457000732421875,
|
||
|
|
"learning_rate": 5.598651755051975e-08,
|
||
|
|
"logits/chosen": -34510007.63525836,
|
||
|
|
"logits/rejected": -36185081.414790995,
|
||
|
|
"logps/chosen": -488.3011968085106,
|
||
|
|
"logps/rejected": -383.1957897909968,
|
||
|
|
"loss": 0.5276,
|
||
|
|
"loss/base_kto": 3.0414016246795654,
|
||
|
|
"metrics/advantage_var": 402.95172119140625,
|
||
|
|
"regularization/mmd": 1.1797621250152588,
|
||
|
|
"regularization/rkhs_norm": 227.314453125,
|
||
|
|
"rewards/chosen": 0.55588737882017,
|
||
|
|
"rewards/margins": 0.9902694341702253,
|
||
|
|
"rewards/rejected": -0.43438205535005525,
|
||
|
|
"step": 1980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5906735751295336,
|
||
|
|
"grad_norm": 16.12910270690918,
|
||
|
|
"kl": 10.265693664550781,
|
||
|
|
"learning_rate": 4.964745868872933e-08,
|
||
|
|
"logits/chosen": -33452939.14714715,
|
||
|
|
"logits/rejected": -35558836.95114007,
|
||
|
|
"logps/chosen": -485.3864958708709,
|
||
|
|
"logps/rejected": -371.22546824104234,
|
||
|
|
"loss": 0.5336,
|
||
|
|
"loss/base_kto": 3.081040382385254,
|
||
|
|
"metrics/advantage_var": 421.461181640625,
|
||
|
|
"regularization/mmd": 1.1879180669784546,
|
||
|
|
"regularization/rkhs_norm": 228.8859405517578,
|
||
|
|
"rewards/chosen": 0.5901449850729635,
|
||
|
|
"rewards/margins": 0.9668989257173675,
|
||
|
|
"rewards/rejected": -0.376753940644404,
|
||
|
|
"step": 2000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.616580310880829,
|
||
|
|
"grad_norm": 9.409523010253906,
|
||
|
|
"kl": 12.09470272064209,
|
||
|
|
"learning_rate": 4.3670429148855493e-08,
|
||
|
|
"logits/chosen": -34416567.86523736,
|
||
|
|
"logits/rejected": -35170372.59330144,
|
||
|
|
"logps/chosen": -488.333221669219,
|
||
|
|
"logps/rejected": -369.500274122807,
|
||
|
|
"loss": 0.531,
|
||
|
|
"loss/base_kto": 3.099996328353882,
|
||
|
|
"metrics/advantage_var": 377.51416015625,
|
||
|
|
"regularization/mmd": 1.1477069854736328,
|
||
|
|
"regularization/rkhs_norm": 221.13816833496094,
|
||
|
|
"rewards/chosen": 0.6344299783750479,
|
||
|
|
"rewards/margins": 0.9355695057156829,
|
||
|
|
"rewards/rejected": -0.301139527340635,
|
||
|
|
"step": 2020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.6424870466321244,
|
||
|
|
"grad_norm": 9.471223831176758,
|
||
|
|
"kl": 13.312454223632812,
|
||
|
|
"learning_rate": 3.806023374435663e-08,
|
||
|
|
"logits/chosen": -33918623.34281201,
|
||
|
|
"logits/rejected": -35609706.040185474,
|
||
|
|
"logps/chosen": -454.03495260663504,
|
||
|
|
"logps/rejected": -376.96215707109735,
|
||
|
|
"loss": 0.5267,
|
||
|
|
"loss/base_kto": 3.1084702014923096,
|
||
|
|
"metrics/advantage_var": 353.3669738769531,
|
||
|
|
"regularization/mmd": 1.1049410104751587,
|
||
|
|
"regularization/rkhs_norm": 212.89810180664062,
|
||
|
|
"rewards/chosen": 0.5582775821053021,
|
||
|
|
"rewards/margins": 0.9037094449850596,
|
||
|
|
"rewards/rejected": -0.3454318628797575,
|
||
|
|
"step": 2040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.66839378238342,
|
||
|
|
"grad_norm": 7.567631244659424,
|
||
|
|
"kl": 13.232315063476562,
|
||
|
|
"learning_rate": 3.282138239813037e-08,
|
||
|
|
"logits/chosen": -34503498.34850863,
|
||
|
|
"logits/rejected": -36607984.07465008,
|
||
|
|
"logps/chosen": -482.61651295133436,
|
||
|
|
"logps/rejected": -384.65163297045103,
|
||
|
|
"loss": 0.5249,
|
||
|
|
"loss/base_kto": 3.0587663650512695,
|
||
|
|
"metrics/advantage_var": 373.1220397949219,
|
||
|
|
"regularization/mmd": 1.1407493352890015,
|
||
|
|
"regularization/rkhs_norm": 219.7975616455078,
|
||
|
|
"rewards/chosen": 0.5657442533052884,
|
||
|
|
"rewards/margins": 0.9622465923851022,
|
||
|
|
"rewards/rejected": -0.39650233907981386,
|
||
|
|
"step": 2060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.694300518134715,
|
||
|
|
"grad_norm": 9.058245658874512,
|
||
|
|
"kl": 12.501319885253906,
|
||
|
|
"learning_rate": 2.795808651707793e-08,
|
||
|
|
"logits/chosen": -32961729.115689382,
|
||
|
|
"logits/rejected": -36043304.23420647,
|
||
|
|
"logps/chosen": -492.27496038034866,
|
||
|
|
"logps/rejected": -365.89519934514635,
|
||
|
|
"loss": 0.5275,
|
||
|
|
"loss/base_kto": 3.0900230407714844,
|
||
|
|
"metrics/advantage_var": 369.5855407714844,
|
||
|
|
"regularization/mmd": 1.1300495862960815,
|
||
|
|
"regularization/rkhs_norm": 217.73594665527344,
|
||
|
|
"rewards/chosen": 0.5546478900214193,
|
||
|
|
"rewards/margins": 0.9458008112072581,
|
||
|
|
"rewards/rejected": -0.3911529211858388,
|
||
|
|
"step": 2080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7202072538860103,
|
||
|
|
"grad_norm": 9.40098762512207,
|
||
|
|
"kl": 14.867446899414062,
|
||
|
|
"learning_rate": 2.3474255606639293e-08,
|
||
|
|
"logits/chosen": -34542913.19254658,
|
||
|
|
"logits/rejected": -35755442.716981135,
|
||
|
|
"logps/chosen": -482.073612189441,
|
||
|
|
"logps/rejected": -384.23368710691824,
|
||
|
|
"loss": 0.5284,
|
||
|
|
"loss/base_kto": 3.0799896717071533,
|
||
|
|
"metrics/advantage_var": 391.1192626953125,
|
||
|
|
"regularization/mmd": 1.1469415426254272,
|
||
|
|
"regularization/rkhs_norm": 220.9906768798828,
|
||
|
|
"rewards/chosen": 0.5572285148667993,
|
||
|
|
"rewards/margins": 0.9512824718642687,
|
||
|
|
"rewards/rejected": -0.39405395699746953,
|
||
|
|
"step": 2100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7461139896373057,
|
||
|
|
"grad_norm": 9.91256332397461,
|
||
|
|
"kl": 12.974084854125977,
|
||
|
|
"learning_rate": 1.9373494128020195e-08,
|
||
|
|
"logits/chosen": -34190179.17117117,
|
||
|
|
"logits/rejected": -36306333.602605864,
|
||
|
|
"logps/chosen": -487.9260041291291,
|
||
|
|
"logps/rejected": -366.8720480456026,
|
||
|
|
"loss": 0.5278,
|
||
|
|
"loss/base_kto": 3.0544116497039795,
|
||
|
|
"metrics/advantage_var": 399.2501525878906,
|
||
|
|
"regularization/mmd": 1.1683543920516968,
|
||
|
|
"regularization/rkhs_norm": 225.116455078125,
|
||
|
|
"rewards/chosen": 0.6304746977201812,
|
||
|
|
"rewards/margins": 0.9665527326585337,
|
||
|
|
"rewards/rejected": -0.3360780349383525,
|
||
|
|
"step": 2120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.772020725388601,
|
||
|
|
"grad_norm": 13.506525993347168,
|
||
|
|
"kl": 13.567715644836426,
|
||
|
|
"learning_rate": 1.5659098600638798e-08,
|
||
|
|
"logits/chosen": -33345278.348387096,
|
||
|
|
"logits/rejected": -35806884.46060606,
|
||
|
|
"logps/chosen": -486.77671370967744,
|
||
|
|
"logps/rejected": -382.6580965909091,
|
||
|
|
"loss": 0.5283,
|
||
|
|
"loss/base_kto": 3.064450263977051,
|
||
|
|
"metrics/advantage_var": 396.38427734375,
|
||
|
|
"regularization/mmd": 1.1618027687072754,
|
||
|
|
"regularization/rkhs_norm": 223.85411071777344,
|
||
|
|
"rewards/chosen": 0.5827009631741431,
|
||
|
|
"rewards/margins": 0.959142017084832,
|
||
|
|
"rewards/rejected": -0.37644105391068894,
|
||
|
|
"step": 2140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7979274611398965,
|
||
|
|
"grad_norm": 9.991613388061523,
|
||
|
|
"kl": 12.376852989196777,
|
||
|
|
"learning_rate": 1.2334054952120143e-08,
|
||
|
|
"logits/chosen": -34209263.78280543,
|
||
|
|
"logits/rejected": -36075496.764991894,
|
||
|
|
"logps/chosen": -491.5558069381599,
|
||
|
|
"logps/rejected": -388.61960595623987,
|
||
|
|
"loss": 0.5296,
|
||
|
|
"loss/base_kto": 3.091087818145752,
|
||
|
|
"metrics/advantage_var": 365.6158447265625,
|
||
|
|
"regularization/mmd": 1.1457853317260742,
|
||
|
|
"regularization/rkhs_norm": 220.7678680419922,
|
||
|
|
"rewards/chosen": 0.5611746257246889,
|
||
|
|
"rewards/margins": 0.922801001933917,
|
||
|
|
"rewards/rejected": -0.3616263762092281,
|
||
|
|
"step": 2160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.823834196891192,
|
||
|
|
"grad_norm": 15.1278076171875,
|
||
|
|
"kl": 15.186076164245605,
|
||
|
|
"learning_rate": 9.401036117969108e-09,
|
||
|
|
"logits/chosen": -35485506.85350318,
|
||
|
|
"logits/rejected": -36513471.607361965,
|
||
|
|
"logps/chosen": -491.0250298566879,
|
||
|
|
"logps/rejected": -370.42800996932516,
|
||
|
|
"loss": 0.5273,
|
||
|
|
"loss/base_kto": 3.1048686504364014,
|
||
|
|
"metrics/advantage_var": 359.6988220214844,
|
||
|
|
"regularization/mmd": 1.113534688949585,
|
||
|
|
"regularization/rkhs_norm": 214.5538787841797,
|
||
|
|
"rewards/chosen": 0.572250949349373,
|
||
|
|
"rewards/margins": 0.9058157544562557,
|
||
|
|
"rewards/rejected": -0.3335648051068827,
|
||
|
|
"step": 2180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.849740932642487,
|
||
|
|
"grad_norm": 10.609084129333496,
|
||
|
|
"kl": 11.621691703796387,
|
||
|
|
"learning_rate": 6.8623998928517555e-09,
|
||
|
|
"logits/chosen": -35603950.344827585,
|
||
|
|
"logits/rejected": -36339788.56074766,
|
||
|
|
"logps/chosen": -483.99715909090907,
|
||
|
|
"logps/rejected": -366.5739875389408,
|
||
|
|
"loss": 0.5252,
|
||
|
|
"loss/base_kto": 3.114879608154297,
|
||
|
|
"metrics/advantage_var": 351.2439880371094,
|
||
|
|
"regularization/mmd": 1.086438536643982,
|
||
|
|
"regularization/rkhs_norm": 209.3330535888672,
|
||
|
|
"rewards/chosen": 0.5589293476929859,
|
||
|
|
"rewards/margins": 0.9029869953029936,
|
||
|
|
"rewards/rejected": -0.3440576476100078,
|
||
|
|
"step": 2200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.8756476683937824,
|
||
|
|
"grad_norm": 10.167004585266113,
|
||
|
|
"kl": 15.326950073242188,
|
||
|
|
"learning_rate": 4.72018703521132e-09,
|
||
|
|
"logits/chosen": -33601136.953301124,
|
||
|
|
"logits/rejected": -34893876.83156297,
|
||
|
|
"logps/chosen": -489.0647141706924,
|
||
|
|
"logps/rejected": -383.3451726100152,
|
||
|
|
"loss": 0.5281,
|
||
|
|
"loss/base_kto": 3.048800230026245,
|
||
|
|
"metrics/advantage_var": 396.35748291015625,
|
||
|
|
"regularization/mmd": 1.1763771772384644,
|
||
|
|
"regularization/rkhs_norm": 226.6622772216797,
|
||
|
|
"rewards/chosen": 0.604445747707201,
|
||
|
|
"rewards/margins": 0.9652159659963346,
|
||
|
|
"rewards/rejected": -0.36077021828913364,
|
||
|
|
"step": 2220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.901554404145078,
|
||
|
|
"grad_norm": 16.41950035095215,
|
||
|
|
"kl": 15.674674034118652,
|
||
|
|
"learning_rate": 2.976119626744267e-09,
|
||
|
|
"logits/chosen": -34605490.122977346,
|
||
|
|
"logits/rejected": -34796435.72205438,
|
||
|
|
"logps/chosen": -454.2214805825243,
|
||
|
|
"logps/rejected": -389.7560422960725,
|
||
|
|
"loss": 0.5324,
|
||
|
|
"loss/base_kto": 3.0943081378936768,
|
||
|
|
"metrics/advantage_var": 403.62054443359375,
|
||
|
|
"regularization/mmd": 1.1649839878082275,
|
||
|
|
"regularization/rkhs_norm": 224.467041015625,
|
||
|
|
"rewards/chosen": 0.5575224743691849,
|
||
|
|
"rewards/margins": 0.9312573317497551,
|
||
|
|
"rewards/rejected": -0.3737348573805702,
|
||
|
|
"step": 2240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.927461139896373,
|
||
|
|
"grad_norm": 12.384098052978516,
|
||
|
|
"kl": 16.30813980102539,
|
||
|
|
"learning_rate": 1.631599688052765e-09,
|
||
|
|
"logits/chosen": -34040730.10114192,
|
||
|
|
"logits/rejected": -35910092.56971514,
|
||
|
|
"logps/chosen": -469.271207177814,
|
||
|
|
"logps/rejected": -351.0214580209895,
|
||
|
|
"loss": 0.5226,
|
||
|
|
"loss/base_kto": 3.0762746334075928,
|
||
|
|
"metrics/advantage_var": 347.0081481933594,
|
||
|
|
"regularization/mmd": 1.104848861694336,
|
||
|
|
"regularization/rkhs_norm": 212.8803253173828,
|
||
|
|
"rewards/chosen": 0.5972424455801386,
|
||
|
|
"rewards/margins": 0.9287062150381831,
|
||
|
|
"rewards/rejected": -0.3314637694580444,
|
||
|
|
"step": 2260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.9533678756476682,
|
||
|
|
"grad_norm": 8.210082054138184,
|
||
|
|
"kl": 13.462668418884277,
|
||
|
|
"learning_rate": 6.877080515894085e-10,
|
||
|
|
"logits/chosen": -35319547.288343556,
|
||
|
|
"logits/rejected": -33990199.439490445,
|
||
|
|
"logps/chosen": -443.042321702454,
|
||
|
|
"logps/rejected": -357.6137042197452,
|
||
|
|
"loss": 0.5229,
|
||
|
|
"loss/base_kto": 3.1298470497131348,
|
||
|
|
"metrics/advantage_var": 311.5273132324219,
|
||
|
|
"regularization/mmd": 1.0533379316329956,
|
||
|
|
"regularization/rkhs_norm": 202.95530700683594,
|
||
|
|
"rewards/chosen": 0.5258189008279812,
|
||
|
|
"rewards/margins": 0.8736475897228484,
|
||
|
|
"rewards/rejected": -0.3478286888948671,
|
||
|
|
"step": 2280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.9792746113989637,
|
||
|
|
"grad_norm": 9.271958351135254,
|
||
|
|
"kl": 14.206778526306152,
|
||
|
|
"learning_rate": 1.4520349279739663e-10,
|
||
|
|
"logits/chosen": -34512060.674015746,
|
||
|
|
"logits/rejected": -36050931.29922481,
|
||
|
|
"logps/chosen": -468.143405511811,
|
||
|
|
"logps/rejected": -376.12604166666665,
|
||
|
|
"loss": 0.5234,
|
||
|
|
"loss/base_kto": 3.1001479625701904,
|
||
|
|
"metrics/advantage_var": 337.7193908691406,
|
||
|
|
"regularization/mmd": 1.0868895053863525,
|
||
|
|
"regularization/rkhs_norm": 209.41995239257812,
|
||
|
|
"rewards/chosen": 0.5491407980130414,
|
||
|
|
"rewards/margins": 0.9166954552320337,
|
||
|
|
"rewards/rejected": -0.3675546572189923,
|
||
|
|
"step": 2300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 3.0,
|
||
|
|
"step": 2316,
|
||
|
|
"total_flos": 9.97585917447635e+17,
|
||
|
|
"train_loss": 0.5585705229243673,
|
||
|
|
"train_runtime": 11048.9523,
|
||
|
|
"train_samples_per_second": 13.415,
|
||
|
|
"train_steps_per_second": 0.21
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 20,
|
||
|
|
"max_steps": 2316,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 3,
|
||
|
|
"save_steps": 500,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": false,
|
||
|
|
"should_training_stop": false
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 9.97585917447635e+17,
|
||
|
|
"train_batch_size": 8,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|