1999 lines
69 KiB
JSON
1999 lines
69 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 3.0,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 2316,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"epoch": 0.025906735751295335,
|
||
|
|
"grad_norm": 10.65859317779541,
|
||
|
|
"kl": 44.20431137084961,
|
||
|
|
"learning_rate": 1.8999999999999998e-07,
|
||
|
|
"logits/chosen": -36499456.0,
|
||
|
|
"logits/rejected": -37668092.71794872,
|
||
|
|
"logps/chosen": -490.6620617378049,
|
||
|
|
"logps/rejected": -359.51189403044873,
|
||
|
|
"loss": 0.4506,
|
||
|
|
"loss/base_kto": 3.6045689582824707,
|
||
|
|
"metrics/advantage_var": 861.9415283203125,
|
||
|
|
"rewards/chosen": 0.9872177868354611,
|
||
|
|
"rewards/margins": 0.4244383358075665,
|
||
|
|
"rewards/rejected": 0.5627794510278946,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.05181347150259067,
|
||
|
|
"grad_norm": 10.56235408782959,
|
||
|
|
"kl": 44.744598388671875,
|
||
|
|
"learning_rate": 3.8999999999999997e-07,
|
||
|
|
"logits/chosen": -38137711.73047473,
|
||
|
|
"logits/rejected": -37998056.31897926,
|
||
|
|
"logps/chosen": -475.3998372894334,
|
||
|
|
"logps/rejected": -398.91088516746413,
|
||
|
|
"loss": 0.4484,
|
||
|
|
"loss/base_kto": 3.587329149246216,
|
||
|
|
"metrics/advantage_var": 992.3124389648438,
|
||
|
|
"rewards/chosen": 0.9791349495716883,
|
||
|
|
"rewards/margins": 0.3998072552248483,
|
||
|
|
"rewards/rejected": 0.5793276943468401,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07772020725388601,
|
||
|
|
"grad_norm": 9.909646034240723,
|
||
|
|
"kl": 27.297826766967773,
|
||
|
|
"learning_rate": 5.9e-07,
|
||
|
|
"logits/chosen": -36635869.108728945,
|
||
|
|
"logits/rejected": -37625764.54226475,
|
||
|
|
"logps/chosen": -487.3697358346095,
|
||
|
|
"logps/rejected": -386.92222388357254,
|
||
|
|
"loss": 0.4302,
|
||
|
|
"loss/base_kto": 3.441992998123169,
|
||
|
|
"metrics/advantage_var": 1028.0374755859375,
|
||
|
|
"rewards/chosen": 0.8436054971645292,
|
||
|
|
"rewards/margins": 0.5919641917247489,
|
||
|
|
"rewards/rejected": 0.2516413054397802,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10362694300518134,
|
||
|
|
"grad_norm": 11.076920509338379,
|
||
|
|
"kl": 6.907355785369873,
|
||
|
|
"learning_rate": 7.9e-07,
|
||
|
|
"logits/chosen": -38943921.03050847,
|
||
|
|
"logits/rejected": -39010942.14492754,
|
||
|
|
"logps/chosen": -488.87748940677966,
|
||
|
|
"logps/rejected": -398.9723731884058,
|
||
|
|
"loss": 0.4244,
|
||
|
|
"loss/base_kto": 3.3955726623535156,
|
||
|
|
"metrics/advantage_var": 1177.5146484375,
|
||
|
|
"rewards/chosen": 0.043001853813559324,
|
||
|
|
"rewards/margins": 0.745197527217092,
|
||
|
|
"rewards/rejected": -0.7021956734035326,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.12953367875647667,
|
||
|
|
"grad_norm": 7.774325370788574,
|
||
|
|
"kl": 15.655855178833008,
|
||
|
|
"learning_rate": 9.9e-07,
|
||
|
|
"logits/chosen": -37001158.757763974,
|
||
|
|
"logits/rejected": -38360018.91823899,
|
||
|
|
"logps/chosen": -479.5358598602484,
|
||
|
|
"logps/rejected": -358.2423840408805,
|
||
|
|
"loss": 0.428,
|
||
|
|
"loss/base_kto": 3.424107313156128,
|
||
|
|
"metrics/advantage_var": 1501.2918701171875,
|
||
|
|
"rewards/chosen": 0.3111519191576087,
|
||
|
|
"rewards/margins": 0.7818387297290513,
|
||
|
|
"rewards/rejected": -0.4706868105714426,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.15544041450777202,
|
||
|
|
"grad_norm": 8.679657936096191,
|
||
|
|
"kl": 29.89056396484375,
|
||
|
|
"learning_rate": 9.998186234298189e-07,
|
||
|
|
"logits/chosen": -40178530.96485623,
|
||
|
|
"logits/rejected": -39543200.48929664,
|
||
|
|
"logps/chosen": -461.1728234824281,
|
||
|
|
"logps/rejected": -375.8446339831804,
|
||
|
|
"loss": 0.407,
|
||
|
|
"loss/base_kto": 3.2559025287628174,
|
||
|
|
"metrics/advantage_var": 2275.13330078125,
|
||
|
|
"rewards/chosen": 1.0524772668418032,
|
||
|
|
"rewards/margins": 1.0377021831424456,
|
||
|
|
"rewards/rejected": 0.014775083699357618,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18134715025906736,
|
||
|
|
"grad_norm": 7.838543891906738,
|
||
|
|
"kl": 35.66352462768555,
|
||
|
|
"learning_rate": 9.992359552107714e-07,
|
||
|
|
"logits/chosen": -41033721.895678096,
|
||
|
|
"logits/rejected": -41711303.25123153,
|
||
|
|
"logps/chosen": -484.4836997019374,
|
||
|
|
"logps/rejected": -369.20089285714283,
|
||
|
|
"loss": 0.3773,
|
||
|
|
"loss/base_kto": 3.0183520317077637,
|
||
|
|
"metrics/advantage_var": 2924.481689453125,
|
||
|
|
"rewards/chosen": 1.3930760481673807,
|
||
|
|
"rewards/margins": 1.472008429783231,
|
||
|
|
"rewards/rejected": -0.07893238161585014,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20725388601036268,
|
||
|
|
"grad_norm": 6.404437065124512,
|
||
|
|
"kl": 35.48348617553711,
|
||
|
|
"learning_rate": 9.982519612796161e-07,
|
||
|
|
"logits/chosen": -40603175.384615384,
|
||
|
|
"logits/rejected": -40007502.048780486,
|
||
|
|
"logps/chosen": -464.8622796474359,
|
||
|
|
"logps/rejected": -349.6068502286585,
|
||
|
|
"loss": 0.3852,
|
||
|
|
"loss/base_kto": 3.0814366340637207,
|
||
|
|
"metrics/advantage_var": 3211.025634765625,
|
||
|
|
"rewards/chosen": 1.32285641401242,
|
||
|
|
"rewards/margins": 1.3913330502477268,
|
||
|
|
"rewards/rejected": -0.0684766362353069,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.23316062176165803,
|
||
|
|
"grad_norm": 8.489002227783203,
|
||
|
|
"kl": 15.075944900512695,
|
||
|
|
"learning_rate": 9.968674326492213e-07,
|
||
|
|
"logits/chosen": -39859590.171073094,
|
||
|
|
"logits/rejected": -39911282.53689168,
|
||
|
|
"logps/chosen": -446.71646578538105,
|
||
|
|
"logps/rejected": -366.1639030612245,
|
||
|
|
"loss": 0.3852,
|
||
|
|
"loss/base_kto": 3.081433057785034,
|
||
|
|
"metrics/advantage_var": 3389.28564453125,
|
||
|
|
"rewards/chosen": 0.8286895040034506,
|
||
|
|
"rewards/margins": 1.4900147161796673,
|
||
|
|
"rewards/rejected": -0.6613252121762166,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.25906735751295334,
|
||
|
|
"grad_norm": 8.60866641998291,
|
||
|
|
"kl": 37.68467712402344,
|
||
|
|
"learning_rate": 9.950834823142198e-07,
|
||
|
|
"logits/chosen": -39526004.03169572,
|
||
|
|
"logits/rejected": -41538766.693374425,
|
||
|
|
"logps/chosen": -486.00425911251983,
|
||
|
|
"logps/rejected": -376.5807251540832,
|
||
|
|
"loss": 0.3715,
|
||
|
|
"loss/base_kto": 2.971728801727295,
|
||
|
|
"metrics/advantage_var": 4009.591796875,
|
||
|
|
"rewards/chosen": 1.279915932052298,
|
||
|
|
"rewards/margins": 1.6364233612251842,
|
||
|
|
"rewards/rejected": -0.3565074291728862,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2849740932642487,
|
||
|
|
"grad_norm": 9.43969440460205,
|
||
|
|
"kl": 11.000213623046875,
|
||
|
|
"learning_rate": 9.929015443562942e-07,
|
||
|
|
"logits/chosen": -41568631.304897316,
|
||
|
|
"logits/rejected": -40817902.98608965,
|
||
|
|
"logps/chosen": -473.59414494470775,
|
||
|
|
"logps/rejected": -385.6153641808346,
|
||
|
|
"loss": 0.3744,
|
||
|
|
"loss/base_kto": 2.9952468872070312,
|
||
|
|
"metrics/advantage_var": 5182.689453125,
|
||
|
|
"rewards/chosen": 0.8071200354216035,
|
||
|
|
"rewards/margins": 1.9288033831442657,
|
||
|
|
"rewards/rejected": -1.1216833477226622,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.31088082901554404,
|
||
|
|
"grad_norm": 7.523622989654541,
|
||
|
|
"kl": 20.298025131225586,
|
||
|
|
"learning_rate": 9.90323372791347e-07,
|
||
|
|
"logits/chosen": -42630571.063565895,
|
||
|
|
"logits/rejected": -41810284.44724409,
|
||
|
|
"logps/chosen": -463.9735465116279,
|
||
|
|
"logps/rejected": -392.01377952755905,
|
||
|
|
"loss": 0.3711,
|
||
|
|
"loss/base_kto": 2.9685542583465576,
|
||
|
|
"metrics/advantage_var": 4850.92333984375,
|
||
|
|
"rewards/chosen": 1.0982537321342054,
|
||
|
|
"rewards/margins": 1.809265627741981,
|
||
|
|
"rewards/rejected": -0.7110118956077756,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.33678756476683935,
|
||
|
|
"grad_norm": 8.435532569885254,
|
||
|
|
"kl": 42.418304443359375,
|
||
|
|
"learning_rate": 9.87351040159484e-07,
|
||
|
|
"logits/chosen": -43215823.382789314,
|
||
|
|
"logits/rejected": -43751947.828382835,
|
||
|
|
"logps/chosen": -462.843471810089,
|
||
|
|
"logps/rejected": -373.2071988448845,
|
||
|
|
"loss": 0.3355,
|
||
|
|
"loss/base_kto": 2.683739423751831,
|
||
|
|
"metrics/advantage_var": 4713.56689453125,
|
||
|
|
"rewards/chosen": 1.9778451438473665,
|
||
|
|
"rewards/margins": 2.1419684272084174,
|
||
|
|
"rewards/rejected": -0.16412328336105095,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3626943005181347,
|
||
|
|
"grad_norm": 9.643255233764648,
|
||
|
|
"kl": 23.864294052124023,
|
||
|
|
"learning_rate": 9.839869358589396e-07,
|
||
|
|
"logits/chosen": -45868187.99396682,
|
||
|
|
"logits/rejected": -46083100.213938415,
|
||
|
|
"logps/chosen": -479.51649698340873,
|
||
|
|
"logps/rejected": -361.66473358995137,
|
||
|
|
"loss": 0.3686,
|
||
|
|
"loss/base_kto": 2.949044704437256,
|
||
|
|
"metrics/advantage_var": 5534.42333984375,
|
||
|
|
"rewards/chosen": 1.4372264004760558,
|
||
|
|
"rewards/margins": 2.038752002399425,
|
||
|
|
"rewards/rejected": -0.6015256019233691,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.38860103626943004,
|
||
|
|
"grad_norm": 7.352804660797119,
|
||
|
|
"kl": 19.898496627807617,
|
||
|
|
"learning_rate": 9.80233764225289e-07,
|
||
|
|
"logits/chosen": -45201262.40758294,
|
||
|
|
"logits/rejected": -44915165.972179286,
|
||
|
|
"logps/chosen": -481.7296603475513,
|
||
|
|
"logps/rejected": -392.0678129829985,
|
||
|
|
"loss": 0.3648,
|
||
|
|
"loss/base_kto": 2.918400287628174,
|
||
|
|
"metrics/advantage_var": 7212.60888671875,
|
||
|
|
"rewards/chosen": 1.2794505717441746,
|
||
|
|
"rewards/margins": 2.153510517135162,
|
||
|
|
"rewards/rejected": -0.8740599453909873,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.41450777202072536,
|
||
|
|
"grad_norm": 9.16907024383545,
|
||
|
|
"kl": 43.78324890136719,
|
||
|
|
"learning_rate": 9.760945423574868e-07,
|
||
|
|
"logits/chosen": -45530638.006079026,
|
||
|
|
"logits/rejected": -44813861.86495177,
|
||
|
|
"logps/chosen": -477.1079502279635,
|
||
|
|
"logps/rejected": -363.2409314710611,
|
||
|
|
"loss": 0.3463,
|
||
|
|
"loss/base_kto": 2.7703616619110107,
|
||
|
|
"metrics/advantage_var": 7601.89990234375,
|
||
|
|
"rewards/chosen": 1.9827135080262157,
|
||
|
|
"rewards/margins": 2.358483512642122,
|
||
|
|
"rewards/rejected": -0.3757700046159064,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.44041450777202074,
|
||
|
|
"grad_norm": 7.767402648925781,
|
||
|
|
"kl": 31.018543243408203,
|
||
|
|
"learning_rate": 9.71572597692482e-07,
|
||
|
|
"logits/chosen": -43233508.2700157,
|
||
|
|
"logits/rejected": -43803146.35147745,
|
||
|
|
"logps/chosen": -458.922193877551,
|
||
|
|
"logps/rejected": -372.84705482115083,
|
||
|
|
"loss": 0.3386,
|
||
|
|
"loss/base_kto": 2.709152936935425,
|
||
|
|
"metrics/advantage_var": 9589.0205078125,
|
||
|
|
"rewards/chosen": 1.8369244106897566,
|
||
|
|
"rewards/margins": 2.9127549444426726,
|
||
|
|
"rewards/rejected": -1.075830533752916,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.46632124352331605,
|
||
|
|
"grad_norm": 7.376603603363037,
|
||
|
|
"kl": 21.673208236694336,
|
||
|
|
"learning_rate": 9.666715653303588e-07,
|
||
|
|
"logits/chosen": -44333835.43913713,
|
||
|
|
"logits/rejected": -43826385.34389857,
|
||
|
|
"logps/chosen": -473.8385496918336,
|
||
|
|
"logps/rejected": -398.9099643423138,
|
||
|
|
"loss": 0.3491,
|
||
|
|
"loss/base_kto": 2.79294490814209,
|
||
|
|
"metrics/advantage_var": 8912.041015625,
|
||
|
|
"rewards/chosen": 1.4503943115610554,
|
||
|
|
"rewards/margins": 2.665896970411095,
|
||
|
|
"rewards/rejected": -1.2155026588500397,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.49222797927461137,
|
||
|
|
"grad_norm": 7.592734336853027,
|
||
|
|
"kl": 19.095108032226562,
|
||
|
|
"learning_rate": 9.613953851121528e-07,
|
||
|
|
"logits/chosen": -43271092.090771556,
|
||
|
|
"logits/rejected": -45212693.50565428,
|
||
|
|
"logps/chosen": -486.91579992435703,
|
||
|
|
"logps/rejected": -382.0510147415186,
|
||
|
|
"loss": 0.3425,
|
||
|
|
"loss/base_kto": 2.739999532699585,
|
||
|
|
"metrics/advantage_var": 8978.3515625,
|
||
|
|
"rewards/chosen": 1.8050278564083775,
|
||
|
|
"rewards/margins": 2.751250756404591,
|
||
|
|
"rewards/rejected": -0.9462228999962137,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5181347150259067,
|
||
|
|
"grad_norm": 6.854775428771973,
|
||
|
|
"kl": 45.80805206298828,
|
||
|
|
"learning_rate": 9.557482984526924e-07,
|
||
|
|
"logits/chosen": -43894309.42457232,
|
||
|
|
"logits/rejected": -44310335.09576138,
|
||
|
|
"logps/chosen": -443.4170392690513,
|
||
|
|
"logps/rejected": -366.22024627158555,
|
||
|
|
"loss": 0.3496,
|
||
|
|
"loss/base_kto": 2.7964887619018555,
|
||
|
|
"metrics/advantage_var": 7448.00390625,
|
||
|
|
"rewards/chosen": 1.9952160967267691,
|
||
|
|
"rewards/margins": 2.3970563785501353,
|
||
|
|
"rewards/rejected": -0.40184028182336634,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5440414507772021,
|
||
|
|
"grad_norm": 10.735581398010254,
|
||
|
|
"kl": 19.892004013061523,
|
||
|
|
"learning_rate": 9.497348449310107e-07,
|
||
|
|
"logits/chosen": -44185031.67159277,
|
||
|
|
"logits/rejected": -44101271.84500745,
|
||
|
|
"logps/chosen": -462.24266215106735,
|
||
|
|
"logps/rejected": -365.46805141579733,
|
||
|
|
"loss": 0.3534,
|
||
|
|
"loss/base_kto": 2.827474355697632,
|
||
|
|
"metrics/advantage_var": 8355.3662109375,
|
||
|
|
"rewards/chosen": 1.2435715648732553,
|
||
|
|
"rewards/margins": 2.6411173694385033,
|
||
|
|
"rewards/rejected": -1.3975458045652478,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5699481865284974,
|
||
|
|
"grad_norm": 8.053645133972168,
|
||
|
|
"kl": 14.910491943359375,
|
||
|
|
"learning_rate": 9.433598586410701e-07,
|
||
|
|
"logits/chosen": -46035449.7939394,
|
||
|
|
"logits/rejected": -45075168.61935484,
|
||
|
|
"logps/chosen": -498.8833333333333,
|
||
|
|
"logps/rejected": -387.8259072580645,
|
||
|
|
"loss": 0.3487,
|
||
|
|
"loss/base_kto": 2.789438247680664,
|
||
|
|
"metrics/advantage_var": 8716.130859375,
|
||
|
|
"rewards/chosen": 1.1289883700284091,
|
||
|
|
"rewards/margins": 2.8263150301846593,
|
||
|
|
"rewards/rejected": -1.69732666015625,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5958549222797928,
|
||
|
|
"grad_norm": 7.885801315307617,
|
||
|
|
"kl": 33.040191650390625,
|
||
|
|
"learning_rate": 9.366284643057313e-07,
|
||
|
|
"logits/chosen": -45832788.90452261,
|
||
|
|
"logits/rejected": -46052329.51098096,
|
||
|
|
"logps/chosen": -454.71492881072027,
|
||
|
|
"logps/rejected": -410.90794289897514,
|
||
|
|
"loss": 0.3342,
|
||
|
|
"loss/base_kto": 2.6736834049224854,
|
||
|
|
"metrics/advantage_var": 9005.0234375,
|
||
|
|
"rewards/chosen": 1.7976493388164783,
|
||
|
|
"rewards/margins": 2.8960478406580776,
|
||
|
|
"rewards/rejected": -1.0983985018415996,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6217616580310881,
|
||
|
|
"grad_norm": 6.836333274841309,
|
||
|
|
"kl": 28.7034854888916,
|
||
|
|
"learning_rate": 9.295460731570917e-07,
|
||
|
|
"logits/chosen": -46365180.05546995,
|
||
|
|
"logits/rejected": -45693645.28684627,
|
||
|
|
"logps/chosen": -479.06798921417567,
|
||
|
|
"logps/rejected": -382.8000198098257,
|
||
|
|
"loss": 0.3247,
|
||
|
|
"loss/base_kto": 2.5976855754852295,
|
||
|
|
"metrics/advantage_var": 10056.6845703125,
|
||
|
|
"rewards/chosen": 1.8589578167132126,
|
||
|
|
"rewards/margins": 3.031334252925574,
|
||
|
|
"rewards/rejected": -1.1723764362123614,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6476683937823834,
|
||
|
|
"grad_norm": 8.679668426513672,
|
||
|
|
"kl": 32.58065414428711,
|
||
|
|
"learning_rate": 9.221183785865056e-07,
|
||
|
|
"logits/chosen": -44614913.69536424,
|
||
|
|
"logits/rejected": -45147469.25443787,
|
||
|
|
"logps/chosen": -472.30805049668874,
|
||
|
|
"logps/rejected": -384.6820451183432,
|
||
|
|
"loss": 0.3347,
|
||
|
|
"loss/base_kto": 2.677523612976074,
|
||
|
|
"metrics/advantage_var": 9521.0166015625,
|
||
|
|
"rewards/chosen": 1.824923685844371,
|
||
|
|
"rewards/margins": 3.0118727442416526,
|
||
|
|
"rewards/rejected": -1.1869490583972817,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6735751295336787,
|
||
|
|
"grad_norm": 7.148499488830566,
|
||
|
|
"kl": 25.07105827331543,
|
||
|
|
"learning_rate": 9.143513515677817e-07,
|
||
|
|
"logits/chosen": -46456619.18072289,
|
||
|
|
"logits/rejected": -44846964.36363637,
|
||
|
|
"logps/chosen": -492.8112763554217,
|
||
|
|
"logps/rejected": -390.60830965909093,
|
||
|
|
"loss": 0.3248,
|
||
|
|
"loss/base_kto": 2.598315954208374,
|
||
|
|
"metrics/advantage_var": 11530.103515625,
|
||
|
|
"rewards/chosen": 1.8040299013436558,
|
||
|
|
"rewards/margins": 3.2287691940333882,
|
||
|
|
"rewards/rejected": -1.4247392926897322,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6994818652849741,
|
||
|
|
"grad_norm": 10.257616996765137,
|
||
|
|
"kl": 41.308380126953125,
|
||
|
|
"learning_rate": 9.062512358572373e-07,
|
||
|
|
"logits/chosen": -46776827.9300477,
|
||
|
|
"logits/rejected": -45388855.05376344,
|
||
|
|
"logps/chosen": -477.1953994435612,
|
||
|
|
"logps/rejected": -375.51804915514595,
|
||
|
|
"loss": 0.3344,
|
||
|
|
"loss/base_kto": 2.674875259399414,
|
||
|
|
"metrics/advantage_var": 9839.3037109375,
|
||
|
|
"rewards/chosen": 2.125871958527921,
|
||
|
|
"rewards/margins": 2.839637327011504,
|
||
|
|
"rewards/rejected": -0.713765368483583,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7253886010362695,
|
||
|
|
"grad_norm": 8.11175537109375,
|
||
|
|
"kl": 27.59626579284668,
|
||
|
|
"learning_rate": 8.978245429744691e-07,
|
||
|
|
"logits/chosen": -45277500.83572568,
|
||
|
|
"logits/rejected": -44860392.477794796,
|
||
|
|
"logps/chosen": -462.4535486443381,
|
||
|
|
"logps/rejected": -370.9771248085758,
|
||
|
|
"loss": 0.3345,
|
||
|
|
"loss/base_kto": 2.676085948944092,
|
||
|
|
"metrics/advantage_var": 10163.2373046875,
|
||
|
|
"rewards/chosen": 1.8515385531922848,
|
||
|
|
"rewards/margins": 2.9131068557382265,
|
||
|
|
"rewards/rejected": -1.0615683025459417,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7512953367875648,
|
||
|
|
"grad_norm": 8.053217887878418,
|
||
|
|
"kl": 17.627090454101562,
|
||
|
|
"learning_rate": 8.890780469678738e-07,
|
||
|
|
"logits/chosen": -44548563.93114241,
|
||
|
|
"logits/rejected": -46489897.13572543,
|
||
|
|
"logps/chosen": -457.03032081377154,
|
||
|
|
"logps/rejected": -408.0214996099844,
|
||
|
|
"loss": 0.3287,
|
||
|
|
"loss/base_kto": 2.6296844482421875,
|
||
|
|
"metrics/advantage_var": 12007.1591796875,
|
||
|
|
"rewards/chosen": 1.6501634863434076,
|
||
|
|
"rewards/margins": 3.347522425622854,
|
||
|
|
"rewards/rejected": -1.6973589392794461,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7772020725388601,
|
||
|
|
"grad_norm": 7.483168125152588,
|
||
|
|
"kl": 32.44775390625,
|
||
|
|
"learning_rate": 8.800187789691269e-07,
|
||
|
|
"logits/chosen": -48252049.8176,
|
||
|
|
"logits/rejected": -47718249.917557254,
|
||
|
|
"logps/chosen": -448.42165,
|
||
|
|
"logps/rejected": -407.2638358778626,
|
||
|
|
"loss": 0.3299,
|
||
|
|
"loss/base_kto": 2.6391899585723877,
|
||
|
|
"metrics/advantage_var": 10226.193359375,
|
||
|
|
"rewards/chosen": 2.0992955078125,
|
||
|
|
"rewards/margins": 3.1892319753399336,
|
||
|
|
"rewards/rejected": -1.0899364675274332,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8031088082901554,
|
||
|
|
"grad_norm": 6.4975266456604,
|
||
|
|
"kl": 20.634342193603516,
|
||
|
|
"learning_rate": 8.706540215409981e-07,
|
||
|
|
"logits/chosen": -45601083.33225284,
|
||
|
|
"logits/rejected": -46161734.66063348,
|
||
|
|
"logps/chosen": -477.0887358184765,
|
||
|
|
"logps/rejected": -379.0682032428356,
|
||
|
|
"loss": 0.3278,
|
||
|
|
"loss/base_kto": 2.62276554107666,
|
||
|
|
"metrics/advantage_var": 10482.455078125,
|
||
|
|
"rewards/chosen": 1.7784174543532212,
|
||
|
|
"rewards/margins": 3.2988043933084907,
|
||
|
|
"rewards/rejected": -1.5203869389552696,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8290155440414507,
|
||
|
|
"grad_norm": 9.11014461517334,
|
||
|
|
"kl": 18.934978485107422,
|
||
|
|
"learning_rate": 8.609913028230462e-07,
|
||
|
|
"logits/chosen": -44933326.67889908,
|
||
|
|
"logits/rejected": -45986891.24600639,
|
||
|
|
"logps/chosen": -472.6990634556575,
|
||
|
|
"logps/rejected": -383.70162739616615,
|
||
|
|
"loss": 0.3266,
|
||
|
|
"loss/base_kto": 2.6129066944122314,
|
||
|
|
"metrics/advantage_var": 10181.88671875,
|
||
|
|
"rewards/chosen": 1.4984698281011086,
|
||
|
|
"rewards/margins": 3.1533712046486326,
|
||
|
|
"rewards/rejected": -1.654901376547524,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8549222797927462,
|
||
|
|
"grad_norm": 10.547191619873047,
|
||
|
|
"kl": 19.205251693725586,
|
||
|
|
"learning_rate": 8.510383904798994e-07,
|
||
|
|
"logits/chosen": -46755092.93081761,
|
||
|
|
"logits/rejected": -44693739.32919255,
|
||
|
|
"logps/chosen": -464.08451257861634,
|
||
|
|
"logps/rejected": -399.30221273291926,
|
||
|
|
"loss": 0.3362,
|
||
|
|
"loss/base_kto": 2.6896512508392334,
|
||
|
|
"metrics/advantage_var": 9943.228515625,
|
||
|
|
"rewards/chosen": 1.3194805601108,
|
||
|
|
"rewards/margins": 2.960457175684849,
|
||
|
|
"rewards/rejected": -1.6409766155740488,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8808290155440415,
|
||
|
|
"grad_norm": 7.576268196105957,
|
||
|
|
"kl": 30.703176498413086,
|
||
|
|
"learning_rate": 8.408032854569865e-07,
|
||
|
|
"logits/chosen": -46373757.61490683,
|
||
|
|
"logits/rejected": -46271948.47798742,
|
||
|
|
"logps/chosen": -466.3391886645963,
|
||
|
|
"logps/rejected": -374.7342521619497,
|
||
|
|
"loss": 0.318,
|
||
|
|
"loss/base_kto": 2.5438382625579834,
|
||
|
|
"metrics/advantage_var": 10507.3154296875,
|
||
|
|
"rewards/chosen": 2.4138117251188858,
|
||
|
|
"rewards/margins": 3.4287190821968143,
|
||
|
|
"rewards/rejected": -1.0149073570779286,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9067357512953368,
|
||
|
|
"grad_norm": 7.023205757141113,
|
||
|
|
"kl": 22.83217430114746,
|
||
|
|
"learning_rate": 8.302942155487377e-07,
|
||
|
|
"logits/chosen": -47770873.25271318,
|
||
|
|
"logits/rejected": -47177956.988976374,
|
||
|
|
"logps/chosen": -470.42790697674417,
|
||
|
|
"logps/rejected": -380.4131151574803,
|
||
|
|
"loss": 0.304,
|
||
|
|
"loss/base_kto": 2.4317188262939453,
|
||
|
|
"metrics/advantage_var": 10339.8466796875,
|
||
|
|
"rewards/chosen": 2.1665962633236435,
|
||
|
|
"rewards/margins": 3.6137248235475607,
|
||
|
|
"rewards/rejected": -1.4471285602239172,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9326424870466321,
|
||
|
|
"grad_norm": 6.914972305297852,
|
||
|
|
"kl": 21.22562026977539,
|
||
|
|
"learning_rate": 8.195196287844278e-07,
|
||
|
|
"logits/chosen": -49408567.5980551,
|
||
|
|
"logits/rejected": -49626354.48567119,
|
||
|
|
"logps/chosen": -461.87180915721234,
|
||
|
|
"logps/rejected": -400.6054864253394,
|
||
|
|
"loss": 0.3195,
|
||
|
|
"loss/base_kto": 2.55560040473938,
|
||
|
|
"metrics/advantage_var": 12679.2861328125,
|
||
|
|
"rewards/chosen": 1.997904822351094,
|
||
|
|
"rewards/margins": 3.673362024462708,
|
||
|
|
"rewards/rejected": -1.6754572021116139,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9585492227979274,
|
||
|
|
"grad_norm": 9.571149826049805,
|
||
|
|
"kl": 8.873680114746094,
|
||
|
|
"learning_rate": 8.08488186636975e-07,
|
||
|
|
"logits/chosen": -49232709.2327044,
|
||
|
|
"logits/rejected": -48566799.900621116,
|
||
|
|
"logps/chosen": -461.3564760220126,
|
||
|
|
"logps/rejected": -389.0656541149068,
|
||
|
|
"loss": 0.3329,
|
||
|
|
"loss/base_kto": 2.6631648540496826,
|
||
|
|
"metrics/advantage_var": 12759.3857421875,
|
||
|
|
"rewards/chosen": 1.0083675744398586,
|
||
|
|
"rewards/margins": 3.495386420276427,
|
||
|
|
"rewards/rejected": -2.4870188458365683,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9844559585492227,
|
||
|
|
"grad_norm": 7.072620868682861,
|
||
|
|
"kl": 29.986997604370117,
|
||
|
|
"learning_rate": 7.972087570601576e-07,
|
||
|
|
"logits/chosen": -48027195.284210525,
|
||
|
|
"logits/rejected": -48910156.17560975,
|
||
|
|
"logps/chosen": -481.89906015037593,
|
||
|
|
"logps/rejected": -372.8478912601626,
|
||
|
|
"loss": 0.3108,
|
||
|
|
"loss/base_kto": 2.4860572814941406,
|
||
|
|
"metrics/advantage_var": 12530.525390625,
|
||
|
|
"rewards/chosen": 2.4417236328125,
|
||
|
|
"rewards/margins": 3.6977175828887194,
|
||
|
|
"rewards/rejected": -1.2559939500762196,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0103626943005182,
|
||
|
|
"grad_norm": 5.7229814529418945,
|
||
|
|
"kl": 31.123401641845703,
|
||
|
|
"learning_rate": 7.856904073598458e-07,
|
||
|
|
"logits/chosen": -49082970.35294118,
|
||
|
|
"logits/rejected": -49505685.06329114,
|
||
|
|
"logps/chosen": -450.6265963622291,
|
||
|
|
"logps/rejected": -376.8190763449367,
|
||
|
|
"loss": 0.287,
|
||
|
|
"loss/base_kto": 2.2956960201263428,
|
||
|
|
"metrics/advantage_var": 11794.609375,
|
||
|
|
"rewards/chosen": 2.3705805255901704,
|
||
|
|
"rewards/margins": 3.957680204498893,
|
||
|
|
"rewards/rejected": -1.5870996789087224,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0362694300518134,
|
||
|
|
"grad_norm": 8.308485984802246,
|
||
|
|
"kl": 21.34233283996582,
|
||
|
|
"learning_rate": 7.739423969049761e-07,
|
||
|
|
"logits/chosen": -48664785.53135314,
|
||
|
|
"logits/rejected": -49527075.703264095,
|
||
|
|
"logps/chosen": -452.4751443894389,
|
||
|
|
"logps/rejected": -387.68386034866467,
|
||
|
|
"loss": 0.2552,
|
||
|
|
"loss/base_kto": 2.0418756008148193,
|
||
|
|
"metrics/advantage_var": 15537.6943359375,
|
||
|
|
"rewards/chosen": 2.2635937177702146,
|
||
|
|
"rewards/margins": 4.9204337846517054,
|
||
|
|
"rewards/rejected": -2.656840066881491,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0621761658031088,
|
||
|
|
"grad_norm": 7.449312686920166,
|
||
|
|
"kl": 19.720563888549805,
|
||
|
|
"learning_rate": 7.619741696841322e-07,
|
||
|
|
"logits/chosen": -46948384.35387046,
|
||
|
|
"logits/rejected": -48699420.48840804,
|
||
|
|
"logps/chosen": -446.41409952606637,
|
||
|
|
"logps/rejected": -402.2554095826893,
|
||
|
|
"loss": 0.2566,
|
||
|
|
"loss/base_kto": 2.0527331829071045,
|
||
|
|
"metrics/advantage_var": 14032.3798828125,
|
||
|
|
"rewards/chosen": 2.2591982776708135,
|
||
|
|
"rewards/margins": 4.754287164480125,
|
||
|
|
"rewards/rejected": -2.495088886809312,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0880829015544042,
|
||
|
|
"grad_norm": 6.976689338684082,
|
||
|
|
"kl": 14.78875732421875,
|
||
|
|
"learning_rate": 7.497953467137135e-07,
|
||
|
|
"logits/chosen": -49828502.974358976,
|
||
|
|
"logits/rejected": -51138731.70731708,
|
||
|
|
"logps/chosen": -471.2696314102564,
|
||
|
|
"logps/rejected": -404.3307450457317,
|
||
|
|
"loss": 0.2542,
|
||
|
|
"loss/base_kto": 2.033315420150757,
|
||
|
|
"metrics/advantage_var": 14103.1162109375,
|
||
|
|
"rewards/chosen": 1.9908480521960137,
|
||
|
|
"rewards/margins": 4.918190536833018,
|
||
|
|
"rewards/rejected": -2.9273424846370046,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1139896373056994,
|
||
|
|
"grad_norm": 9.26498794555664,
|
||
|
|
"kl": 37.292884826660156,
|
||
|
|
"learning_rate": 7.37415718303793e-07,
|
||
|
|
"logits/chosen": -48743260.09508716,
|
||
|
|
"logits/rejected": -49833173.0046225,
|
||
|
|
"logps/chosen": -453.1921057844691,
|
||
|
|
"logps/rejected": -394.1962153312789,
|
||
|
|
"loss": 0.2582,
|
||
|
|
"loss/base_kto": 2.065633773803711,
|
||
|
|
"metrics/advantage_var": 13541.900390625,
|
||
|
|
"rewards/chosen": 2.3661943003417196,
|
||
|
|
"rewards/margins": 4.667860513084882,
|
||
|
|
"rewards/rejected": -2.3016662127431626,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1398963730569949,
|
||
|
|
"grad_norm": 8.440954208374023,
|
||
|
|
"kl": 24.506061553955078,
|
||
|
|
"learning_rate": 7.248452361878855e-07,
|
||
|
|
"logits/chosen": -47704310.278481014,
|
||
|
|
"logits/rejected": -48767576.493827164,
|
||
|
|
"logps/chosen": -438.6851265822785,
|
||
|
|
"logps/rejected": -387.41307388117286,
|
||
|
|
"loss": 0.2668,
|
||
|
|
"loss/base_kto": 2.134312391281128,
|
||
|
|
"metrics/advantage_var": 13320.8251953125,
|
||
|
|
"rewards/chosen": 2.290668149537678,
|
||
|
|
"rewards/margins": 4.493615318712061,
|
||
|
|
"rewards/rejected": -2.202947169174383,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.16580310880829,
|
||
|
|
"grad_norm": 6.1074981689453125,
|
||
|
|
"kl": 12.768942832946777,
|
||
|
|
"learning_rate": 7.120940055229497e-07,
|
||
|
|
"logits/chosen": -47911413.73065902,
|
||
|
|
"logits/rejected": -47319152.604811,
|
||
|
|
"logps/chosen": -444.51844555873924,
|
||
|
|
"logps/rejected": -397.1567869415808,
|
||
|
|
"loss": 0.2375,
|
||
|
|
"loss/base_kto": 1.900206446647644,
|
||
|
|
"metrics/advantage_var": 13858.2177734375,
|
||
|
|
"rewards/chosen": 2.4379781378715975,
|
||
|
|
"rewards/margins": 4.967347793624926,
|
||
|
|
"rewards/rejected": -2.529369655753329,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1917098445595855,
|
||
|
|
"grad_norm": 6.856977462768555,
|
||
|
|
"kl": 25.13776969909668,
|
||
|
|
"learning_rate": 6.991722767660556e-07,
|
||
|
|
"logits/chosen": -48549149.80542265,
|
||
|
|
"logits/rejected": -49921875.50382848,
|
||
|
|
"logps/chosen": -450.0487440191388,
|
||
|
|
"logps/rejected": -371.0899693721286,
|
||
|
|
"loss": 0.2512,
|
||
|
|
"loss/base_kto": 2.009587287902832,
|
||
|
|
"metrics/advantage_var": 13175.6533203125,
|
||
|
|
"rewards/chosen": 2.4123200290320974,
|
||
|
|
"rewards/margins": 4.605071856534969,
|
||
|
|
"rewards/rejected": -2.1927518275028715,
|
||
|
|
"step": 920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2176165803108807,
|
||
|
|
"grad_norm": 5.254368305206299,
|
||
|
|
"kl": 30.324390411376953,
|
||
|
|
"learning_rate": 6.860904374342499e-07,
|
||
|
|
"logits/chosen": -50275765.937106915,
|
||
|
|
"logits/rejected": -50992865.78881988,
|
||
|
|
"logps/chosen": -452.25776336477986,
|
||
|
|
"logps/rejected": -388.415542507764,
|
||
|
|
"loss": 0.2595,
|
||
|
|
"loss/base_kto": 2.076131582260132,
|
||
|
|
"metrics/advantage_var": 14928.7568359375,
|
||
|
|
"rewards/chosen": 2.6799888370921776,
|
||
|
|
"rewards/margins": 4.889126179083152,
|
||
|
|
"rewards/rejected": -2.2091373419909743,
|
||
|
|
"step": 940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2435233160621761,
|
||
|
|
"grad_norm": 5.639325141906738,
|
||
|
|
"kl": 24.19855308532715,
|
||
|
|
"learning_rate": 6.7285900375424e-07,
|
||
|
|
"logits/chosen": -50279631.332302935,
|
||
|
|
"logits/rejected": -49850833.8957346,
|
||
|
|
"logps/chosen": -469.4557573415765,
|
||
|
|
"logps/rejected": -370.7182316350711,
|
||
|
|
"loss": 0.2498,
|
||
|
|
"loss/base_kto": 1.9982547760009766,
|
||
|
|
"metrics/advantage_var": 13618.6552734375,
|
||
|
|
"rewards/chosen": 2.387874927550232,
|
||
|
|
"rewards/margins": 4.803065565509355,
|
||
|
|
"rewards/rejected": -2.4151906379591233,
|
||
|
|
"step": 960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2694300518134716,
|
||
|
|
"grad_norm": 8.131012916564941,
|
||
|
|
"kl": 11.774667739868164,
|
||
|
|
"learning_rate": 6.594886122086123e-07,
|
||
|
|
"logits/chosen": -50578138.77061469,
|
||
|
|
"logits/rejected": -49652056.11745514,
|
||
|
|
"logps/chosen": -481.5137275112444,
|
||
|
|
"logps/rejected": -396.49551386623165,
|
||
|
|
"loss": 0.2616,
|
||
|
|
"loss/base_kto": 2.0926153659820557,
|
||
|
|
"metrics/advantage_var": 14772.4716796875,
|
||
|
|
"rewards/chosen": 2.2389902460878934,
|
||
|
|
"rewards/margins": 4.830445802139076,
|
||
|
|
"rewards/rejected": -2.5914555560511827,
|
||
|
|
"step": 980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2953367875647668,
|
||
|
|
"grad_norm": 10.34538459777832,
|
||
|
|
"kl": 21.447887420654297,
|
||
|
|
"learning_rate": 6.459900109853766e-07,
|
||
|
|
"logits/chosen": -50675736.78668684,
|
||
|
|
"logits/rejected": -52042747.0371567,
|
||
|
|
"logps/chosen": -479.4501701966717,
|
||
|
|
"logps/rejected": -391.41013731825524,
|
||
|
|
"loss": 0.2643,
|
||
|
|
"loss/base_kto": 2.114217519760132,
|
||
|
|
"metrics/advantage_var": 13216.7001953125,
|
||
|
|
"rewards/chosen": 2.6053697640885023,
|
||
|
|
"rewards/margins": 4.449835226275497,
|
||
|
|
"rewards/rejected": -1.8444654621869951,
|
||
|
|
"step": 1000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3212435233160622,
|
||
|
|
"grad_norm": 7.639214992523193,
|
||
|
|
"kl": 25.33041763305664,
|
||
|
|
"learning_rate": 6.323740513377171e-07,
|
||
|
|
"logits/chosen": -50322896.25723473,
|
||
|
|
"logits/rejected": -51459955.93920973,
|
||
|
|
"logps/chosen": -436.558530948553,
|
||
|
|
"logps/rejected": -399.7970649696049,
|
||
|
|
"loss": 0.2517,
|
||
|
|
"loss/base_kto": 2.0138232707977295,
|
||
|
|
"metrics/advantage_var": 15641.556640625,
|
||
|
|
"rewards/chosen": 2.574196769493569,
|
||
|
|
"rewards/margins": 5.121921623751263,
|
||
|
|
"rewards/rejected": -2.5477248542576936,
|
||
|
|
"step": 1020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3471502590673574,
|
||
|
|
"grad_norm": 7.394321918487549,
|
||
|
|
"kl": 20.35123062133789,
|
||
|
|
"learning_rate": 6.186516788608865e-07,
|
||
|
|
"logits/chosen": -50750205.15103339,
|
||
|
|
"logits/rejected": -51357035.35483871,
|
||
|
|
"logps/chosen": -474.0139109697933,
|
||
|
|
"logps/rejected": -405.144777265745,
|
||
|
|
"loss": 0.2626,
|
||
|
|
"loss/base_kto": 2.1008141040802,
|
||
|
|
"metrics/advantage_var": 13648.5361328125,
|
||
|
|
"rewards/chosen": 2.109104465793919,
|
||
|
|
"rewards/margins": 4.62314536441143,
|
||
|
|
"rewards/rejected": -2.5140408986175116,
|
||
|
|
"step": 1040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3730569948186528,
|
||
|
|
"grad_norm": 8.497376441955566,
|
||
|
|
"kl": 23.546171188354492,
|
||
|
|
"learning_rate": 6.048339246932652e-07,
|
||
|
|
"logits/chosen": -48032839.29113924,
|
||
|
|
"logits/rejected": -50251178.666666664,
|
||
|
|
"logps/chosen": -456.9215783227848,
|
||
|
|
"logps/rejected": -416.03303433641975,
|
||
|
|
"loss": 0.2612,
|
||
|
|
"loss/base_kto": 2.0899407863616943,
|
||
|
|
"metrics/advantage_var": 14816.8515625,
|
||
|
|
"rewards/chosen": 2.5009827432753164,
|
||
|
|
"rewards/margins": 4.854381572605949,
|
||
|
|
"rewards/rejected": -2.353398829330633,
|
||
|
|
"step": 1060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3989637305699483,
|
||
|
|
"grad_norm": 6.526173114776611,
|
||
|
|
"kl": 19.82684898376465,
|
||
|
|
"learning_rate": 5.909318966486494e-07,
|
||
|
|
"logits/chosen": -50905499.44227886,
|
||
|
|
"logits/rejected": -50662217.918433934,
|
||
|
|
"logps/chosen": -462.2083020989505,
|
||
|
|
"logps/rejected": -396.3524418841762,
|
||
|
|
"loss": 0.2414,
|
||
|
|
"loss/base_kto": 1.930864691734314,
|
||
|
|
"metrics/advantage_var": 13378.1474609375,
|
||
|
|
"rewards/chosen": 2.5639007888633807,
|
||
|
|
"rewards/margins": 4.943357848788952,
|
||
|
|
"rewards/rejected": -2.379457059925571,
|
||
|
|
"step": 1080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4248704663212435,
|
||
|
|
"grad_norm": 6.579440593719482,
|
||
|
|
"kl": 25.88844871520996,
|
||
|
|
"learning_rate": 5.769567702869052e-07,
|
||
|
|
"logits/chosen": -52712109.75565611,
|
||
|
|
"logits/rejected": -52538728.14262561,
|
||
|
|
"logps/chosen": -444.0515177224736,
|
||
|
|
"logps/rejected": -401.4771576175041,
|
||
|
|
"loss": 0.2608,
|
||
|
|
"loss/base_kto": 2.086622714996338,
|
||
|
|
"metrics/advantage_var": 13585.439453125,
|
||
|
|
"rewards/chosen": 2.560188397023473,
|
||
|
|
"rewards/margins": 4.714666695810446,
|
||
|
|
"rewards/rejected": -2.154478298786973,
|
||
|
|
"step": 1100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.450777202072539,
|
||
|
|
"grad_norm": 9.067955017089844,
|
||
|
|
"kl": 21.861936569213867,
|
||
|
|
"learning_rate": 5.629197799301614e-07,
|
||
|
|
"logits/chosen": -52116110.97659907,
|
||
|
|
"logits/rejected": -52317502.89827856,
|
||
|
|
"logps/chosen": -483.269988299532,
|
||
|
|
"logps/rejected": -408.1690385367762,
|
||
|
|
"loss": 0.2603,
|
||
|
|
"loss/base_kto": 2.082534074783325,
|
||
|
|
"metrics/advantage_var": 14696.2470703125,
|
||
|
|
"rewards/chosen": 2.6743337360447543,
|
||
|
|
"rewards/margins": 4.77644622106774,
|
||
|
|
"rewards/rejected": -2.1021124850229853,
|
||
|
|
"step": 1120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4766839378238341,
|
||
|
|
"grad_norm": 6.779482841491699,
|
||
|
|
"kl": 25.322052001953125,
|
||
|
|
"learning_rate": 5.488322096317633e-07,
|
||
|
|
"logits/chosen": -50702644.60273973,
|
||
|
|
"logits/rejected": -50894502.83146068,
|
||
|
|
"logps/chosen": -490.74714611872145,
|
||
|
|
"logps/rejected": -384.28772070626,
|
||
|
|
"loss": 0.2773,
|
||
|
|
"loss/base_kto": 2.2181718349456787,
|
||
|
|
"metrics/advantage_var": 14075.6591796875,
|
||
|
|
"rewards/chosen": 2.5817913827649352,
|
||
|
|
"rewards/margins": 4.610652880181669,
|
||
|
|
"rewards/rejected": -2.0288614974167336,
|
||
|
|
"step": 1140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5025906735751295,
|
||
|
|
"grad_norm": 7.117109298706055,
|
||
|
|
"kl": 30.72871208190918,
|
||
|
|
"learning_rate": 5.347053841052518e-07,
|
||
|
|
"logits/chosen": -49487765.69160306,
|
||
|
|
"logits/rejected": -51846499.5328,
|
||
|
|
"logps/chosen": -436.91049618320613,
|
||
|
|
"logps/rejected": -390.759775,
|
||
|
|
"loss": 0.2602,
|
||
|
|
"loss/base_kto": 2.081653356552124,
|
||
|
|
"metrics/advantage_var": 13168.955078125,
|
||
|
|
"rewards/chosen": 2.580529691972805,
|
||
|
|
"rewards/margins": 4.646819145097805,
|
||
|
|
"rewards/rejected": -2.066289453125,
|
||
|
|
"step": 1160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.528497409326425,
|
||
|
|
"grad_norm": 7.7086100578308105,
|
||
|
|
"kl": 28.140439987182617,
|
||
|
|
"learning_rate": 5.205506596206531e-07,
|
||
|
|
"logits/chosen": -50617514.1510574,
|
||
|
|
"logits/rejected": -51723751.14563107,
|
||
|
|
"logps/chosen": -461.4189010574018,
|
||
|
|
"logps/rejected": -406.2813764158576,
|
||
|
|
"loss": 0.2693,
|
||
|
|
"loss/base_kto": 2.154078960418701,
|
||
|
|
"metrics/advantage_var": 13133.0810546875,
|
||
|
|
"rewards/chosen": 2.545702608690521,
|
||
|
|
"rewards/margins": 4.44313510262256,
|
||
|
|
"rewards/rejected": -1.8974324939320388,
|
||
|
|
"step": 1180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5544041450777202,
|
||
|
|
"grad_norm": 8.802536010742188,
|
||
|
|
"kl": 37.46778106689453,
|
||
|
|
"learning_rate": 5.063794148754032e-07,
|
||
|
|
"logits/chosen": -51498450.944,
|
||
|
|
"logits/rejected": -51648984.13435114,
|
||
|
|
"logps/chosen": -447.47215,
|
||
|
|
"logps/rejected": -403.54608778625953,
|
||
|
|
"loss": 0.2703,
|
||
|
|
"loss/base_kto": 2.1621289253234863,
|
||
|
|
"metrics/advantage_var": 13803.234375,
|
||
|
|
"rewards/chosen": 2.386833984375,
|
||
|
|
"rewards/margins": 4.364916679091126,
|
||
|
|
"rewards/rejected": -1.9780826947161259,
|
||
|
|
"step": 1200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5803108808290154,
|
||
|
|
"grad_norm": 7.323568344116211,
|
||
|
|
"kl": 26.67828369140625,
|
||
|
|
"learning_rate": 4.922030418472422e-07,
|
||
|
|
"logits/chosen": -49500446.38699187,
|
||
|
|
"logits/rejected": -52240106.827067666,
|
||
|
|
"logps/chosen": -481.9415142276423,
|
||
|
|
"logps/rejected": -424.76339285714283,
|
||
|
|
"loss": 0.2387,
|
||
|
|
"loss/base_kto": 1.9094852209091187,
|
||
|
|
"metrics/advantage_var": 14641.3720703125,
|
||
|
|
"rewards/chosen": 2.3854688690929877,
|
||
|
|
"rewards/margins": 5.121942927774849,
|
||
|
|
"rewards/rejected": -2.736474058681861,
|
||
|
|
"step": 1220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6062176165803108,
|
||
|
|
"grad_norm": 8.504008293151855,
|
||
|
|
"kl": 18.42952537536621,
|
||
|
|
"learning_rate": 4.780329366364336e-07,
|
||
|
|
"logits/chosen": -50177989.48571429,
|
||
|
|
"logits/rejected": -50049803.23902439,
|
||
|
|
"logps/chosen": -471.387969924812,
|
||
|
|
"logps/rejected": -393.54270833333334,
|
||
|
|
"loss": 0.2588,
|
||
|
|
"loss/base_kto": 2.0701587200164795,
|
||
|
|
"metrics/advantage_var": 13637.3095703125,
|
||
|
|
"rewards/chosen": 2.5208492422462405,
|
||
|
|
"rewards/margins": 4.819602735640549,
|
||
|
|
"rewards/rejected": -2.298753493394309,
|
||
|
|
"step": 1240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6321243523316062,
|
||
|
|
"grad_norm": 7.224009037017822,
|
||
|
|
"kl": 29.492969512939453,
|
||
|
|
"learning_rate": 4.638804903046684e-07,
|
||
|
|
"logits/chosen": -52442332.92598425,
|
||
|
|
"logits/rejected": -53556908.254263565,
|
||
|
|
"logps/chosen": -471.96633858267717,
|
||
|
|
"logps/rejected": -400.7843023255814,
|
||
|
|
"loss": 0.2597,
|
||
|
|
"loss/base_kto": 2.0774409770965576,
|
||
|
|
"metrics/advantage_var": 12654.884765625,
|
||
|
|
"rewards/chosen": 2.6672249861589568,
|
||
|
|
"rewards/margins": 4.531793557501011,
|
||
|
|
"rewards/rejected": -1.8645685713420543,
|
||
|
|
"step": 1260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6580310880829017,
|
||
|
|
"grad_norm": 9.018906593322754,
|
||
|
|
"kl": 24.7161922454834,
|
||
|
|
"learning_rate": 4.497570797180217e-07,
|
||
|
|
"logits/chosen": -52677000.036753446,
|
||
|
|
"logits/rejected": -54384068.3891547,
|
||
|
|
"logps/chosen": -497.3668166156202,
|
||
|
|
"logps/rejected": -371.66113437001593,
|
||
|
|
"loss": 0.2578,
|
||
|
|
"loss/base_kto": 2.062281608581543,
|
||
|
|
"metrics/advantage_var": 12963.482421875,
|
||
|
|
"rewards/chosen": 2.509745248791635,
|
||
|
|
"rewards/margins": 4.59229086843378,
|
||
|
|
"rewards/rejected": -2.0825456196421452,
|
||
|
|
"step": 1280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6839378238341969,
|
||
|
|
"grad_norm": 7.554812431335449,
|
||
|
|
"kl": 22.770795822143555,
|
||
|
|
"learning_rate": 4.3567405840131853e-07,
|
||
|
|
"logits/chosen": -52157474.628019325,
|
||
|
|
"logits/rejected": -54097755.289833084,
|
||
|
|
"logps/chosen": -455.2374194847021,
|
||
|
|
"logps/rejected": -387.06060318664646,
|
||
|
|
"loss": 0.2616,
|
||
|
|
"loss/base_kto": 2.0927064418792725,
|
||
|
|
"metrics/advantage_var": 13465.212890625,
|
||
|
|
"rewards/chosen": 2.3146233550976247,
|
||
|
|
"rewards/margins": 4.661450860669514,
|
||
|
|
"rewards/rejected": -2.346827505571889,
|
||
|
|
"step": 1300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.709844559585492,
|
||
|
|
"grad_norm": 8.2971830368042,
|
||
|
|
"kl": 27.238021850585938,
|
||
|
|
"learning_rate": 4.2164274741126506e-07,
|
||
|
|
"logits/chosen": -52593679.950155765,
|
||
|
|
"logits/rejected": -52541517.04075235,
|
||
|
|
"logps/chosen": -462.36896417445485,
|
||
|
|
"logps/rejected": -404.9313773510972,
|
||
|
|
"loss": 0.2527,
|
||
|
|
"loss/base_kto": 2.021425247192383,
|
||
|
|
"metrics/advantage_var": 14735.775390625,
|
||
|
|
"rewards/chosen": 2.5553978655568534,
|
||
|
|
"rewards/margins": 5.0356779614375355,
|
||
|
|
"rewards/rejected": -2.4802800958806817,
|
||
|
|
"step": 1320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7357512953367875,
|
||
|
|
"grad_norm": 8.200288772583008,
|
||
|
|
"kl": 31.184919357299805,
|
||
|
|
"learning_rate": 4.0767442623567856e-07,
|
||
|
|
"logits/chosen": -51745873.85318108,
|
||
|
|
"logits/rejected": -53775051.41829085,
|
||
|
|
"logps/chosen": -449.0927814029364,
|
||
|
|
"logps/rejected": -386.02633058470764,
|
||
|
|
"loss": 0.2547,
|
||
|
|
"loss/base_kto": 2.0375049114227295,
|
||
|
|
"metrics/advantage_var": 14179.8154296875,
|
||
|
|
"rewards/chosen": 2.6527774680108074,
|
||
|
|
"rewards/margins": 4.816153482640211,
|
||
|
|
"rewards/rejected": -2.163376014629404,
|
||
|
|
"step": 1340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.761658031088083,
|
||
|
|
"grad_norm": 6.618141174316406,
|
||
|
|
"kl": 16.11330223083496,
|
||
|
|
"learning_rate": 3.937803237261357e-07,
|
||
|
|
"logits/chosen": -50715854.67889908,
|
||
|
|
"logits/rejected": -52869191.974440895,
|
||
|
|
"logps/chosen": -455.9623470948012,
|
||
|
|
"logps/rejected": -380.09070487220447,
|
||
|
|
"loss": 0.2821,
|
||
|
|
"loss/base_kto": 2.2565581798553467,
|
||
|
|
"metrics/advantage_var": 12346.3212890625,
|
||
|
|
"rewards/chosen": 2.182855728569381,
|
||
|
|
"rewards/margins": 4.179773940680004,
|
||
|
|
"rewards/rejected": -1.996918212110623,
|
||
|
|
"step": 1360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7875647668393784,
|
||
|
|
"grad_norm": 7.924007892608643,
|
||
|
|
"kl": 35.323726654052734,
|
||
|
|
"learning_rate": 3.7997160907132456e-07,
|
||
|
|
"logits/chosen": -53506547.67255217,
|
||
|
|
"logits/rejected": -52517515.49467275,
|
||
|
|
"logps/chosen": -477.71960272873196,
|
||
|
|
"logps/rejected": -414.9009703196347,
|
||
|
|
"loss": 0.252,
|
||
|
|
"loss/base_kto": 2.0156280994415283,
|
||
|
|
"metrics/advantage_var": 15051.2763671875,
|
||
|
|
"rewards/chosen": 2.7909207902788924,
|
||
|
|
"rewards/margins": 5.021091889379919,
|
||
|
|
"rewards/rejected": -2.2301710991010273,
|
||
|
|
"step": 1380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8134715025906736,
|
||
|
|
"grad_norm": 7.5072784423828125,
|
||
|
|
"kl": 23.88299560546875,
|
||
|
|
"learning_rate": 3.662593828183601e-07,
|
||
|
|
"logits/chosen": -51244745.6,
|
||
|
|
"logits/rejected": -52956176.0,
|
||
|
|
"logps/chosen": -453.29130859375,
|
||
|
|
"logps/rejected": -397.8761962890625,
|
||
|
|
"loss": 0.2651,
|
||
|
|
"loss/base_kto": 2.1209917068481445,
|
||
|
|
"metrics/advantage_var": 12839.2919921875,
|
||
|
|
"rewards/chosen": 2.2394113540649414,
|
||
|
|
"rewards/margins": 4.529068565368652,
|
||
|
|
"rewards/rejected": -2.289657211303711,
|
||
|
|
"step": 1400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8393782383419688,
|
||
|
|
"grad_norm": 8.157623291015625,
|
||
|
|
"kl": 18.68647003173828,
|
||
|
|
"learning_rate": 3.5265466794927725e-07,
|
||
|
|
"logits/chosen": -53356375.12443778,
|
||
|
|
"logits/rejected": -53121465.004893966,
|
||
|
|
"logps/chosen": -451.46158170914543,
|
||
|
|
"logps/rejected": -380.848338091354,
|
||
|
|
"loss": 0.2588,
|
||
|
|
"loss/base_kto": 2.070202350616455,
|
||
|
|
"metrics/advantage_var": 13694.220703125,
|
||
|
|
"rewards/chosen": 2.3687469619682346,
|
||
|
|
"rewards/margins": 4.725921510168684,
|
||
|
|
"rewards/rejected": -2.3571745482004487,
|
||
|
|
"step": 1420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8652849740932642,
|
||
|
|
"grad_norm": 5.826056957244873,
|
||
|
|
"kl": 33.48683547973633,
|
||
|
|
"learning_rate": 3.3916840101987887e-07,
|
||
|
|
"logits/chosen": -52616473.64069952,
|
||
|
|
"logits/rejected": -53620673.08141321,
|
||
|
|
"logps/chosen": -474.6203795707472,
|
||
|
|
"logps/rejected": -400.85143049155147,
|
||
|
|
"loss": 0.2594,
|
||
|
|
"loss/base_kto": 2.0748069286346436,
|
||
|
|
"metrics/advantage_var": 12404.947265625,
|
||
|
|
"rewards/chosen": 2.778156322970489,
|
||
|
|
"rewards/margins": 4.539449045704744,
|
||
|
|
"rewards/rejected": -1.761292722734255,
|
||
|
|
"step": 1440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8911917098445596,
|
||
|
|
"grad_norm": 7.233984470367432,
|
||
|
|
"kl": 21.10243034362793,
|
||
|
|
"learning_rate": 3.258114233680583e-07,
|
||
|
|
"logits/chosen": -54232816.42433697,
|
||
|
|
"logits/rejected": -53901193.414710484,
|
||
|
|
"logps/chosen": -460.75594773790954,
|
||
|
|
"logps/rejected": -382.471904342723,
|
||
|
|
"loss": 0.257,
|
||
|
|
"loss/base_kto": 2.055638313293457,
|
||
|
|
"metrics/advantage_var": 12931.291015625,
|
||
|
|
"rewards/chosen": 2.364379311500585,
|
||
|
|
"rewards/margins": 4.625557055045186,
|
||
|
|
"rewards/rejected": -2.261177743544601,
|
||
|
|
"step": 1460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.917098445595855,
|
||
|
|
"grad_norm": 7.463532447814941,
|
||
|
|
"kl": 27.48748779296875,
|
||
|
|
"learning_rate": 3.1259447239866796e-07,
|
||
|
|
"logits/chosen": -52548510.1656051,
|
||
|
|
"logits/rejected": -54142184.44171779,
|
||
|
|
"logps/chosen": -445.79443670382165,
|
||
|
|
"logps/rejected": -411.83301380368096,
|
||
|
|
"loss": 0.2539,
|
||
|
|
"loss/base_kto": 2.031330108642578,
|
||
|
|
"metrics/advantage_var": 13952.5751953125,
|
||
|
|
"rewards/chosen": 2.413046478465864,
|
||
|
|
"rewards/margins": 4.821361563660457,
|
||
|
|
"rewards/rejected": -2.4083150851945936,
|
||
|
|
"step": 1480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9430051813471503,
|
||
|
|
"grad_norm": 6.730622291564941,
|
||
|
|
"kl": 26.386274337768555,
|
||
|
|
"learning_rate": 2.9952817295193435e-07,
|
||
|
|
"logits/chosen": -53502204.405616224,
|
||
|
|
"logits/rejected": -55983384.43818466,
|
||
|
|
"logps/chosen": -465.86388455538224,
|
||
|
|
"logps/rejected": -407.58355340375584,
|
||
|
|
"loss": 0.2547,
|
||
|
|
"loss/base_kto": 2.0376741886138916,
|
||
|
|
"metrics/advantage_var": 12988.326171875,
|
||
|
|
"rewards/chosen": 2.668120338094774,
|
||
|
|
"rewards/margins": 4.6582057070795155,
|
||
|
|
"rewards/rejected": -1.9900853689847418,
|
||
|
|
"step": 1500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9689119170984455,
|
||
|
|
"grad_norm": 6.946061134338379,
|
||
|
|
"kl": 21.508319854736328,
|
||
|
|
"learning_rate": 2.866230287623651e-07,
|
||
|
|
"logits/chosen": -51112526.58604651,
|
||
|
|
"logits/rejected": -52070383.87401575,
|
||
|
|
"logps/chosen": -486.25213178294575,
|
||
|
|
"logps/rejected": -384.9942421259843,
|
||
|
|
"loss": 0.2729,
|
||
|
|
"loss/base_kto": 2.1833534240722656,
|
||
|
|
"metrics/advantage_var": 14524.7392578125,
|
||
|
|
"rewards/chosen": 2.373459832243217,
|
||
|
|
"rewards/margins": 4.678766334457784,
|
||
|
|
"rewards/rejected": -2.305306502214567,
|
||
|
|
"step": 1520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.994818652849741,
|
||
|
|
"grad_norm": 6.844990253448486,
|
||
|
|
"kl": 30.158191680908203,
|
||
|
|
"learning_rate": 2.738894140150075e-07,
|
||
|
|
"logits/chosen": -50484298.10526316,
|
||
|
|
"logits/rejected": -51932629.333333336,
|
||
|
|
"logps/chosen": -471.68832236842104,
|
||
|
|
"logps/rejected": -386.06856863839283,
|
||
|
|
"loss": 0.2574,
|
||
|
|
"loss/base_kto": 2.0595574378967285,
|
||
|
|
"metrics/advantage_var": 12867.0029296875,
|
||
|
|
"rewards/chosen": 2.412080865157278,
|
||
|
|
"rewards/margins": 4.429532378538509,
|
||
|
|
"rewards/rejected": -2.017451513381231,
|
||
|
|
"step": 1540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0207253886010363,
|
||
|
|
"grad_norm": 9.334595680236816,
|
||
|
|
"kl": 16.50066566467285,
|
||
|
|
"learning_rate": 2.6133756500585156e-07,
|
||
|
|
"logits/chosen": -54804304.73534073,
|
||
|
|
"logits/rejected": -54237997.50231839,
|
||
|
|
"logps/chosen": -480.82334587955626,
|
||
|
|
"logps/rejected": -396.0348000386399,
|
||
|
|
"loss": 0.2379,
|
||
|
|
"loss/base_kto": 1.9035505056381226,
|
||
|
|
"metrics/advantage_var": 15041.2568359375,
|
||
|
|
"rewards/chosen": 2.3609980902089935,
|
||
|
|
"rewards/margins": 5.105607751989326,
|
||
|
|
"rewards/rejected": -2.7446096617803324,
|
||
|
|
"step": 1560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0466321243523318,
|
||
|
|
"grad_norm": 8.898676872253418,
|
||
|
|
"kl": 15.84814739227295,
|
||
|
|
"learning_rate": 2.489775719130767e-07,
|
||
|
|
"logits/chosen": -53248006.4503937,
|
||
|
|
"logits/rejected": -53730565.95348837,
|
||
|
|
"logps/chosen": -462.44576771653544,
|
||
|
|
"logps/rejected": -418.21555232558137,
|
||
|
|
"loss": 0.205,
|
||
|
|
"loss/base_kto": 1.6402209997177124,
|
||
|
|
"metrics/advantage_var": 15167.4091796875,
|
||
|
|
"rewards/chosen": 2.560914431594488,
|
||
|
|
"rewards/margins": 5.7559937678347985,
|
||
|
|
"rewards/rejected": -3.19507933624031,
|
||
|
|
"step": 1580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0725388601036268,
|
||
|
|
"grad_norm": 6.704883098602295,
|
||
|
|
"kl": 25.383604049682617,
|
||
|
|
"learning_rate": 2.3681937068576303e-07,
|
||
|
|
"logits/chosen": -53025779.587878786,
|
||
|
|
"logits/rejected": -52647830.29677419,
|
||
|
|
"logps/chosen": -457.3313446969697,
|
||
|
|
"logps/rejected": -406.0405745967742,
|
||
|
|
"loss": 0.2133,
|
||
|
|
"loss/base_kto": 1.706233263015747,
|
||
|
|
"metrics/advantage_var": 14251.7578125,
|
||
|
|
"rewards/chosen": 2.7049793590198865,
|
||
|
|
"rewards/margins": 5.467889869667568,
|
||
|
|
"rewards/rejected": -2.7629105106476817,
|
||
|
|
"step": 1600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.098445595854922,
|
||
|
|
"grad_norm": 7.268764495849609,
|
||
|
|
"kl": 25.373451232910156,
|
||
|
|
"learning_rate": 2.2487273505658e-07,
|
||
|
|
"logits/chosen": -54058547.035369776,
|
||
|
|
"logits/rejected": -55866875.331306994,
|
||
|
|
"logps/chosen": -457.23467644694534,
|
||
|
|
"logps/rejected": -430.41479863221883,
|
||
|
|
"loss": 0.2082,
|
||
|
|
"loss/base_kto": 1.6655216217041016,
|
||
|
|
"metrics/advantage_var": 14516.7158203125,
|
||
|
|
"rewards/chosen": 2.768206531978497,
|
||
|
|
"rewards/margins": 5.502744416073291,
|
||
|
|
"rewards/rejected": -2.7345378840947947,
|
||
|
|
"step": 1620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1243523316062176,
|
||
|
|
"grad_norm": 7.603555679321289,
|
||
|
|
"kl": 11.966200828552246,
|
||
|
|
"learning_rate": 2.131472686848817e-07,
|
||
|
|
"logits/chosen": -53182169.139280125,
|
||
|
|
"logits/rejected": -54440051.02028081,
|
||
|
|
"logps/chosen": -436.7267214397496,
|
||
|
|
"logps/rejected": -405.76048166926677,
|
||
|
|
"loss": 0.2122,
|
||
|
|
"loss/base_kto": 1.697871446609497,
|
||
|
|
"metrics/advantage_var": 15479.9716796875,
|
||
|
|
"rewards/chosen": 2.381063016554186,
|
||
|
|
"rewards/margins": 5.622166547414171,
|
||
|
|
"rewards/rejected": -3.2411035308599843,
|
||
|
|
"step": 1640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.150259067357513,
|
||
|
|
"grad_norm": 6.021089553833008,
|
||
|
|
"kl": 17.05512046813965,
|
||
|
|
"learning_rate": 2.016523974365188e-07,
|
||
|
|
"logits/chosen": -55499307.18072289,
|
||
|
|
"logits/rejected": -55416984.935064934,
|
||
|
|
"logps/chosen": -454.51336596385545,
|
||
|
|
"logps/rejected": -400.203226461039,
|
||
|
|
"loss": 0.2146,
|
||
|
|
"loss/base_kto": 1.7166544198989868,
|
||
|
|
"metrics/advantage_var": 13250.1689453125,
|
||
|
|
"rewards/chosen": 2.676328911838761,
|
||
|
|
"rewards/margins": 5.372626099148522,
|
||
|
|
"rewards/rejected": -2.6962971873097605,
|
||
|
|
"step": 1660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1761658031088085,
|
||
|
|
"grad_norm": 6.856086730957031,
|
||
|
|
"kl": 15.850852966308594,
|
||
|
|
"learning_rate": 1.9039736180657372e-07,
|
||
|
|
"logits/chosen": -53884351.502332814,
|
||
|
|
"logits/rejected": -55031528.288854,
|
||
|
|
"logps/chosen": -477.06454121306376,
|
||
|
|
"logps/rejected": -386.7890502354788,
|
||
|
|
"loss": 0.222,
|
||
|
|
"loss/base_kto": 1.7756065130233765,
|
||
|
|
"metrics/advantage_var": 14796.390625,
|
||
|
|
"rewards/chosen": 2.727618417695373,
|
||
|
|
"rewards/margins": 5.3221108044938426,
|
||
|
|
"rewards/rejected": -2.5944923867984695,
|
||
|
|
"step": 1680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2020725388601035,
|
||
|
|
"grad_norm": 7.76173210144043,
|
||
|
|
"kl": 22.50533103942871,
|
||
|
|
"learning_rate": 1.7939120949111498e-07,
|
||
|
|
"logits/chosen": -54061751.54716981,
|
||
|
|
"logits/rejected": -55115057.29192547,
|
||
|
|
"logps/chosen": -459.26827830188677,
|
||
|
|
"logps/rejected": -395.9180900621118,
|
||
|
|
"loss": 0.215,
|
||
|
|
"loss/base_kto": 1.7201648950576782,
|
||
|
|
"metrics/advantage_var": 13973.0341796875,
|
||
|
|
"rewards/chosen": 2.4738017148191824,
|
||
|
|
"rewards/margins": 5.320175657887408,
|
||
|
|
"rewards/rejected": -2.846373943068226,
|
||
|
|
"step": 1700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.227979274611399,
|
||
|
|
"grad_norm": 7.984103202819824,
|
||
|
|
"kl": 22.196212768554688,
|
||
|
|
"learning_rate": 1.6864278811393523e-07,
|
||
|
|
"logits/chosen": -53185831.822222225,
|
||
|
|
"logits/rejected": -52781361.624615386,
|
||
|
|
"logps/chosen": -445.7280753968254,
|
||
|
|
"logps/rejected": -405.18557692307695,
|
||
|
|
"loss": 0.2246,
|
||
|
|
"loss/base_kto": 1.7964165210723877,
|
||
|
|
"metrics/advantage_var": 14180.470703125,
|
||
|
|
"rewards/chosen": 2.587810794890873,
|
||
|
|
"rewards/margins": 5.1655412260807765,
|
||
|
|
"rewards/rejected": -2.5777304311899036,
|
||
|
|
"step": 1720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2538860103626943,
|
||
|
|
"grad_norm": 7.907989978790283,
|
||
|
|
"kl": 15.732986450195312,
|
||
|
|
"learning_rate": 1.5816073811412584e-07,
|
||
|
|
"logits/chosen": -55130164.51282051,
|
||
|
|
"logits/rejected": -55750868.29268292,
|
||
|
|
"logps/chosen": -454.47455929487177,
|
||
|
|
"logps/rejected": -396.7212271341463,
|
||
|
|
"loss": 0.2038,
|
||
|
|
"loss/base_kto": 1.6303322315216064,
|
||
|
|
"metrics/advantage_var": 17776.615234375,
|
||
|
|
"rewards/chosen": 2.779758942432893,
|
||
|
|
"rewards/margins": 5.993346261411551,
|
||
|
|
"rewards/rejected": -3.2135873189786586,
|
||
|
|
"step": 1740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2797927461139897,
|
||
|
|
"grad_norm": 5.375637054443359,
|
||
|
|
"kl": 16.633441925048828,
|
||
|
|
"learning_rate": 1.4795348580020207e-07,
|
||
|
|
"logits/chosen": -53817220.25377644,
|
||
|
|
"logits/rejected": -54321791.58576052,
|
||
|
|
"logps/chosen": -465.3703266616314,
|
||
|
|
"logps/rejected": -395.2126314724919,
|
||
|
|
"loss": 0.2098,
|
||
|
|
"loss/base_kto": 1.6782522201538086,
|
||
|
|
"metrics/advantage_var": 14872.291015625,
|
||
|
|
"rewards/chosen": 2.733623216519543,
|
||
|
|
"rewards/margins": 5.759970849066569,
|
||
|
|
"rewards/rejected": -3.0263476325470267,
|
||
|
|
"step": 1760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.305699481865285,
|
||
|
|
"grad_norm": 7.098501682281494,
|
||
|
|
"kl": 21.81011390686035,
|
||
|
|
"learning_rate": 1.3802923657636355e-07,
|
||
|
|
"logits/chosen": -53584120.7318612,
|
||
|
|
"logits/rejected": -53781951.00928792,
|
||
|
|
"logps/chosen": -464.52646884858046,
|
||
|
|
"logps/rejected": -413.0716911764706,
|
||
|
|
"loss": 0.2132,
|
||
|
|
"loss/base_kto": 1.7055721282958984,
|
||
|
|
"metrics/advantage_var": 14210.4912109375,
|
||
|
|
"rewards/chosen": 2.736393588574527,
|
||
|
|
"rewards/margins": 5.367398843266864,
|
||
|
|
"rewards/rejected": -2.6310052546923375,
|
||
|
|
"step": 1780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.33160621761658,
|
||
|
|
"grad_norm": 6.580950736999512,
|
||
|
|
"kl": 17.816463470458984,
|
||
|
|
"learning_rate": 1.28395968346336e-07,
|
||
|
|
"logits/chosen": -53349533.055214725,
|
||
|
|
"logits/rejected": -54086770.14012739,
|
||
|
|
"logps/chosen": -444.88899539877303,
|
||
|
|
"logps/rejected": -396.07538813694265,
|
||
|
|
"loss": 0.2174,
|
||
|
|
"loss/base_kto": 1.7395038604736328,
|
||
|
|
"metrics/advantage_var": 14362.5595703125,
|
||
|
|
"rewards/chosen": 2.6289107433857364,
|
||
|
|
"rewards/margins": 5.474019455940473,
|
||
|
|
"rewards/rejected": -2.845108712554737,
|
||
|
|
"step": 1800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.3575129533678756,
|
||
|
|
"grad_norm": 4.578265190124512,
|
||
|
|
"kl": 14.631937980651855,
|
||
|
|
"learning_rate": 1.1906142510009415e-07,
|
||
|
|
"logits/chosen": -54949289.25937031,
|
||
|
|
"logits/rejected": -55131558.62969005,
|
||
|
|
"logps/chosen": -502.6547976011994,
|
||
|
|
"logps/rejected": -386.00341557911906,
|
||
|
|
"loss": 0.1927,
|
||
|
|
"loss/base_kto": 1.54173743724823,
|
||
|
|
"metrics/advantage_var": 15154.0849609375,
|
||
|
|
"rewards/chosen": 2.852893194516023,
|
||
|
|
"rewards/margins": 5.928785482699139,
|
||
|
|
"rewards/rejected": -3.0758922881831157,
|
||
|
|
"step": 1820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.383419689119171,
|
||
|
|
"grad_norm": 7.377775192260742,
|
||
|
|
"kl": 21.765613555908203,
|
||
|
|
"learning_rate": 1.1003311068862547e-07,
|
||
|
|
"logits/chosen": -52769526.046656296,
|
||
|
|
"logits/rejected": -54357302.25431711,
|
||
|
|
"logps/chosen": -469.86314152410574,
|
||
|
|
"logps/rejected": -392.0960312009419,
|
||
|
|
"loss": 0.2113,
|
||
|
|
"loss/base_kto": 1.6907843351364136,
|
||
|
|
"metrics/advantage_var": 14896.6064453125,
|
||
|
|
"rewards/chosen": 2.711991329400758,
|
||
|
|
"rewards/margins": 5.5244031781546825,
|
||
|
|
"rewards/rejected": -2.8124118487539245,
|
||
|
|
"step": 1840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4093264248704664,
|
||
|
|
"grad_norm": 8.828693389892578,
|
||
|
|
"kl": 18.875425338745117,
|
||
|
|
"learning_rate": 1.0131828279173588e-07,
|
||
|
|
"logits/chosen": -53972524.9122807,
|
||
|
|
"logits/rejected": -54421036.692189895,
|
||
|
|
"logps/chosen": -430.7370414673046,
|
||
|
|
"logps/rejected": -411.7427258805513,
|
||
|
|
"loss": 0.2136,
|
||
|
|
"loss/base_kto": 1.7088441848754883,
|
||
|
|
"metrics/advantage_var": 14549.0263671875,
|
||
|
|
"rewards/chosen": 2.5342544482655502,
|
||
|
|
"rewards/margins": 5.400527449079103,
|
||
|
|
"rewards/rejected": -2.866273000813553,
|
||
|
|
"step": 1860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4352331606217614,
|
||
|
|
"grad_norm": 7.736706256866455,
|
||
|
|
"kl": 14.055915832519531,
|
||
|
|
"learning_rate": 9.292394708374596e-08,
|
||
|
|
"logits/chosen": -52026667.323076926,
|
||
|
|
"logits/rejected": -54300707.75873016,
|
||
|
|
"logps/chosen": -454.6047115384615,
|
||
|
|
"logps/rejected": -396.18685515873017,
|
||
|
|
"loss": 0.2123,
|
||
|
|
"loss/base_kto": 1.6984790563583374,
|
||
|
|
"metrics/advantage_var": 14236.0537109375,
|
||
|
|
"rewards/chosen": 2.6927809495192307,
|
||
|
|
"rewards/margins": 5.523264425461691,
|
||
|
|
"rewards/rejected": -2.83048347594246,
|
||
|
|
"step": 1880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.461139896373057,
|
||
|
|
"grad_norm": 5.858903408050537,
|
||
|
|
"kl": 15.589258193969727,
|
||
|
|
"learning_rate": 8.48568516017727e-08,
|
||
|
|
"logits/chosen": -53257451.78947368,
|
||
|
|
"logits/rejected": -55407152.76190476,
|
||
|
|
"logps/chosen": -455.32797080592104,
|
||
|
|
"logps/rejected": -401.0511532738095,
|
||
|
|
"loss": 0.2187,
|
||
|
|
"loss/base_kto": 1.749979853630066,
|
||
|
|
"metrics/advantage_var": 14435.7890625,
|
||
|
|
"rewards/chosen": 2.5456470690275492,
|
||
|
|
"rewards/margins": 5.29852553059284,
|
||
|
|
"rewards/rejected": -2.7528784615652904,
|
||
|
|
"step": 1900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4870466321243523,
|
||
|
|
"grad_norm": 6.895484924316406,
|
||
|
|
"kl": 13.590128898620605,
|
||
|
|
"learning_rate": 7.71234813211169e-08,
|
||
|
|
"logits/chosen": -53473166.57230769,
|
||
|
|
"logits/rejected": -55175941.68888889,
|
||
|
|
"logps/chosen": -457.99591346153846,
|
||
|
|
"logps/rejected": -389.46001984126985,
|
||
|
|
"loss": 0.2023,
|
||
|
|
"loss/base_kto": 1.6187559366226196,
|
||
|
|
"metrics/advantage_var": 14734.931640625,
|
||
|
|
"rewards/chosen": 2.576056941105769,
|
||
|
|
"rewards/margins": 5.7244058916886065,
|
||
|
|
"rewards/rejected": -3.1483489505828373,
|
||
|
|
"step": 1920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5129533678756477,
|
||
|
|
"grad_norm": 8.741658210754395,
|
||
|
|
"kl": 15.451461791992188,
|
||
|
|
"learning_rate": 6.973005294212353e-08,
|
||
|
|
"logits/chosen": -53555690.22992126,
|
||
|
|
"logits/rejected": -54169911.16899225,
|
||
|
|
"logps/chosen": -488.6531496062992,
|
||
|
|
"logps/rejected": -409.95174418604654,
|
||
|
|
"loss": 0.2197,
|
||
|
|
"loss/base_kto": 1.7576106786727905,
|
||
|
|
"metrics/advantage_var": 14217.2880859375,
|
||
|
|
"rewards/chosen": 2.5698707400344487,
|
||
|
|
"rewards/margins": 5.545861307500534,
|
||
|
|
"rewards/rejected": -2.975990567466085,
|
||
|
|
"step": 1940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.538860103626943,
|
||
|
|
"grad_norm": 9.848816871643066,
|
||
|
|
"kl": 16.752267837524414,
|
||
|
|
"learning_rate": 6.268250989270102e-08,
|
||
|
|
"logits/chosen": -54066898.5344,
|
||
|
|
"logits/rejected": -55513400.67175572,
|
||
|
|
"logps/chosen": -439.4038,
|
||
|
|
"logps/rejected": -405.7951812977099,
|
||
|
|
"loss": 0.212,
|
||
|
|
"loss/base_kto": 1.6957625150680542,
|
||
|
|
"metrics/advantage_var": 15750.0146484375,
|
||
|
|
"rewards/chosen": 2.6572337890625,
|
||
|
|
"rewards/margins": 5.661433753280058,
|
||
|
|
"rewards/rejected": -3.004199964217557,
|
||
|
|
"step": 1960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5647668393782386,
|
||
|
|
"grad_norm": 6.366123199462891,
|
||
|
|
"kl": 13.643994331359863,
|
||
|
|
"learning_rate": 5.598651755051975e-08,
|
||
|
|
"logits/chosen": -55293618.01846154,
|
||
|
|
"logits/rejected": -55183756.5968254,
|
||
|
|
"logps/chosen": -461.94384615384615,
|
||
|
|
"logps/rejected": -422.662251984127,
|
||
|
|
"loss": 0.1961,
|
||
|
|
"loss/base_kto": 1.5690070390701294,
|
||
|
|
"metrics/advantage_var": 15086.9375,
|
||
|
|
"rewards/chosen": 2.77778076171875,
|
||
|
|
"rewards/margins": 5.87708011687748,
|
||
|
|
"rewards/rejected": -3.0992993551587302,
|
||
|
|
"step": 1980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5906735751295336,
|
||
|
|
"grad_norm": 6.5458807945251465,
|
||
|
|
"kl": 23.304786682128906,
|
||
|
|
"learning_rate": 4.964745868872933e-08,
|
||
|
|
"logits/chosen": -54190597.03606557,
|
||
|
|
"logits/rejected": -54947399.83283582,
|
||
|
|
"logps/chosen": -454.877356557377,
|
||
|
|
"logps/rejected": -427.23903917910445,
|
||
|
|
"loss": 0.2231,
|
||
|
|
"loss/base_kto": 1.7850319147109985,
|
||
|
|
"metrics/advantage_var": 15889.9716796875,
|
||
|
|
"rewards/chosen": 2.544925477074795,
|
||
|
|
"rewards/margins": 5.412163627144757,
|
||
|
|
"rewards/rejected": -2.8672381500699626,
|
||
|
|
"step": 2000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.616580310880829,
|
||
|
|
"grad_norm": 6.88004732131958,
|
||
|
|
"kl": 19.396982192993164,
|
||
|
|
"learning_rate": 4.3670429148855493e-08,
|
||
|
|
"logits/chosen": -54500910.39745628,
|
||
|
|
"logits/rejected": -53831865.60983103,
|
||
|
|
"logps/chosen": -486.3404709856916,
|
||
|
|
"logps/rejected": -385.31322004608296,
|
||
|
|
"loss": 0.2178,
|
||
|
|
"loss/base_kto": 1.7427517175674438,
|
||
|
|
"metrics/advantage_var": 14418.283203125,
|
||
|
|
"rewards/chosen": 2.622706863573132,
|
||
|
|
"rewards/margins": 5.490551405623823,
|
||
|
|
"rewards/rejected": -2.8678445420506913,
|
||
|
|
"step": 2020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.6424870466321244,
|
||
|
|
"grad_norm": 8.664511680603027,
|
||
|
|
"kl": 23.54598045349121,
|
||
|
|
"learning_rate": 3.806023374435663e-08,
|
||
|
|
"logits/chosen": -54896742.4,
|
||
|
|
"logits/rejected": -54253043.2,
|
||
|
|
"logps/chosen": -476.05732421875,
|
||
|
|
"logps/rejected": -433.92314453125,
|
||
|
|
"loss": 0.2018,
|
||
|
|
"loss/base_kto": 1.6147011518478394,
|
||
|
|
"metrics/advantage_var": 15689.046875,
|
||
|
|
"rewards/chosen": 2.820555877685547,
|
||
|
|
"rewards/margins": 5.827705192565919,
|
||
|
|
"rewards/rejected": -3.007149314880371,
|
||
|
|
"step": 2040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.66839378238342,
|
||
|
|
"grad_norm": 7.058914661407471,
|
||
|
|
"kl": 18.342493057250977,
|
||
|
|
"learning_rate": 3.282138239813037e-08,
|
||
|
|
"logits/chosen": -54555392.0,
|
||
|
|
"logits/rejected": -55395878.4,
|
||
|
|
"logps/chosen": -477.841845703125,
|
||
|
|
"logps/rejected": -403.7883544921875,
|
||
|
|
"loss": 0.2113,
|
||
|
|
"loss/base_kto": 1.6905354261398315,
|
||
|
|
"metrics/advantage_var": 14639.556640625,
|
||
|
|
"rewards/chosen": 2.585844802856445,
|
||
|
|
"rewards/margins": 5.570068550109863,
|
||
|
|
"rewards/rejected": -2.984223747253418,
|
||
|
|
"step": 2060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.694300518134715,
|
||
|
|
"grad_norm": 7.7522735595703125,
|
||
|
|
"kl": 13.34247875213623,
|
||
|
|
"learning_rate": 2.795808651707793e-08,
|
||
|
|
"logits/chosen": -53318083.101321585,
|
||
|
|
"logits/rejected": -54890116.487479135,
|
||
|
|
"logps/chosen": -482.61926395007345,
|
||
|
|
"logps/rejected": -379.9926179048414,
|
||
|
|
"loss": 0.2016,
|
||
|
|
"loss/base_kto": 1.6129086017608643,
|
||
|
|
"metrics/advantage_var": 15127.2607421875,
|
||
|
|
"rewards/chosen": 2.6659618495319384,
|
||
|
|
"rewards/margins": 5.789327248569396,
|
||
|
|
"rewards/rejected": -3.1233653990374584,
|
||
|
|
"step": 2080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7202072538860103,
|
||
|
|
"grad_norm": 7.7964935302734375,
|
||
|
|
"kl": 14.25732421875,
|
||
|
|
"learning_rate": 2.3474255606639293e-08,
|
||
|
|
"logits/chosen": -51992039.77287066,
|
||
|
|
"logits/rejected": -54597689.06501548,
|
||
|
|
"logps/chosen": -454.4478509463722,
|
||
|
|
"logps/rejected": -378.53347523219816,
|
||
|
|
"loss": 0.2157,
|
||
|
|
"loss/base_kto": 1.7255557775497437,
|
||
|
|
"metrics/advantage_var": 13727.2841796875,
|
||
|
|
"rewards/chosen": 2.5626330450882295,
|
||
|
|
"rewards/margins": 5.474771520441364,
|
||
|
|
"rewards/rejected": -2.9121384753531347,
|
||
|
|
"step": 2100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7461139896373057,
|
||
|
|
"grad_norm": 7.031707763671875,
|
||
|
|
"kl": 17.868247985839844,
|
||
|
|
"learning_rate": 1.9373494128020195e-08,
|
||
|
|
"logits/chosen": -53826867.03924647,
|
||
|
|
"logits/rejected": -56076074.60031104,
|
||
|
|
"logps/chosen": -471.02634419152275,
|
||
|
|
"logps/rejected": -403.050933125972,
|
||
|
|
"loss": 0.2042,
|
||
|
|
"loss/base_kto": 1.6338428258895874,
|
||
|
|
"metrics/advantage_var": 14567.953125,
|
||
|
|
"rewards/chosen": 2.7185585584895016,
|
||
|
|
"rewards/margins": 5.788678176005053,
|
||
|
|
"rewards/rejected": -3.070119617515552,
|
||
|
|
"step": 2120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.772020725388601,
|
||
|
|
"grad_norm": 8.422375679016113,
|
||
|
|
"kl": 18.8111572265625,
|
||
|
|
"learning_rate": 1.5659098600638798e-08,
|
||
|
|
"logits/chosen": -55571169.28,
|
||
|
|
"logits/rejected": -56779636.21587302,
|
||
|
|
"logps/chosen": -465.2566346153846,
|
||
|
|
"logps/rejected": -405.76974206349206,
|
||
|
|
"loss": 0.2138,
|
||
|
|
"loss/base_kto": 1.7100785970687866,
|
||
|
|
"metrics/advantage_var": 14109.947265625,
|
||
|
|
"rewards/chosen": 2.5388807091346153,
|
||
|
|
"rewards/margins": 5.32893333500267,
|
||
|
|
"rewards/rejected": -2.7900526258680554,
|
||
|
|
"step": 2140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7979274611398965,
|
||
|
|
"grad_norm": 8.012675285339355,
|
||
|
|
"kl": 15.15075969696045,
|
||
|
|
"learning_rate": 1.2334054952120143e-08,
|
||
|
|
"logits/chosen": -54361836.55696203,
|
||
|
|
"logits/rejected": -53801984.0,
|
||
|
|
"logps/chosen": -461.8822191455696,
|
||
|
|
"logps/rejected": -381.108699845679,
|
||
|
|
"loss": 0.205,
|
||
|
|
"loss/base_kto": 1.6402161121368408,
|
||
|
|
"metrics/advantage_var": 14670.7783203125,
|
||
|
|
"rewards/chosen": 2.6562598506106605,
|
||
|
|
"rewards/margins": 5.770099459262281,
|
||
|
|
"rewards/rejected": -3.1138396086516202,
|
||
|
|
"step": 2160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.823834196891192,
|
||
|
|
"grad_norm": 8.247160911560059,
|
||
|
|
"kl": 25.481403350830078,
|
||
|
|
"learning_rate": 9.401036117969108e-09,
|
||
|
|
"logits/chosen": -55167100.71794872,
|
||
|
|
"logits/rejected": -54453379.12195122,
|
||
|
|
"logps/chosen": -440.9677483974359,
|
||
|
|
"logps/rejected": -428.6783060213415,
|
||
|
|
"loss": 0.2258,
|
||
|
|
"loss/base_kto": 1.8066402673721313,
|
||
|
|
"metrics/advantage_var": 15019.609375,
|
||
|
|
"rewards/chosen": 2.4441407032502003,
|
||
|
|
"rewards/margins": 5.257231702202182,
|
||
|
|
"rewards/rejected": -2.813090998951982,
|
||
|
|
"step": 2180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.849740932642487,
|
||
|
|
"grad_norm": 5.7789692878723145,
|
||
|
|
"kl": 25.09571647644043,
|
||
|
|
"learning_rate": 6.8623998928517555e-09,
|
||
|
|
"logits/chosen": -52957783.33646322,
|
||
|
|
"logits/rejected": -53720056.0124805,
|
||
|
|
"logps/chosen": -444.24002347417843,
|
||
|
|
"logps/rejected": -400.2240639625585,
|
||
|
|
"loss": 0.2127,
|
||
|
|
"loss/base_kto": 1.7019115686416626,
|
||
|
|
"metrics/advantage_var": 13801.8740234375,
|
||
|
|
"rewards/chosen": 2.5932023073772497,
|
||
|
|
"rewards/margins": 5.430687681792226,
|
||
|
|
"rewards/rejected": -2.8374853744149764,
|
||
|
|
"step": 2200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.8756476683937824,
|
||
|
|
"grad_norm": 7.976244926452637,
|
||
|
|
"kl": 13.27165412902832,
|
||
|
|
"learning_rate": 4.72018703521132e-09,
|
||
|
|
"logits/chosen": -56062677.06832298,
|
||
|
|
"logits/rejected": -55704505.1572327,
|
||
|
|
"logps/chosen": -476.4144992236025,
|
||
|
|
"logps/rejected": -395.6385613207547,
|
||
|
|
"loss": 0.2142,
|
||
|
|
"loss/base_kto": 1.713976502418518,
|
||
|
|
"metrics/advantage_var": 15611.8466796875,
|
||
|
|
"rewards/chosen": 2.5529902677358307,
|
||
|
|
"rewards/margins": 5.451443698295481,
|
||
|
|
"rewards/rejected": -2.89845343055965,
|
||
|
|
"step": 2220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.901554404145078,
|
||
|
|
"grad_norm": 5.943566799163818,
|
||
|
|
"kl": 22.686498641967773,
|
||
|
|
"learning_rate": 2.976119626744267e-09,
|
||
|
|
"logits/chosen": -54723577.559748426,
|
||
|
|
"logits/rejected": -55772865.98757764,
|
||
|
|
"logps/chosen": -451.77982507861634,
|
||
|
|
"logps/rejected": -397.55330454192546,
|
||
|
|
"loss": 0.2198,
|
||
|
|
"loss/base_kto": 1.7582666873931885,
|
||
|
|
"metrics/advantage_var": 14203.173828125,
|
||
|
|
"rewards/chosen": 2.444792741499607,
|
||
|
|
"rewards/margins": 5.329901452315795,
|
||
|
|
"rewards/rejected": -2.885108710816188,
|
||
|
|
"step": 2240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.927461139896373,
|
||
|
|
"grad_norm": 9.51264762878418,
|
||
|
|
"kl": 15.295659065246582,
|
||
|
|
"learning_rate": 1.631599688052765e-09,
|
||
|
|
"logits/chosen": -55017559.414634146,
|
||
|
|
"logits/rejected": -54868306.05128205,
|
||
|
|
"logps/chosen": -448.9042492378049,
|
||
|
|
"logps/rejected": -414.9752604166667,
|
||
|
|
"loss": 0.2162,
|
||
|
|
"loss/base_kto": 1.7298539876937866,
|
||
|
|
"metrics/advantage_var": 14221.525390625,
|
||
|
|
"rewards/chosen": 2.6078794525890814,
|
||
|
|
"rewards/margins": 5.469379923655287,
|
||
|
|
"rewards/rejected": -2.861500471066206,
|
||
|
|
"step": 2260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.9533678756476682,
|
||
|
|
"grad_norm": 8.533525466918945,
|
||
|
|
"kl": 15.003725051879883,
|
||
|
|
"learning_rate": 6.877080515894085e-10,
|
||
|
|
"logits/chosen": -54759786.36334913,
|
||
|
|
"logits/rejected": -55902100.377125196,
|
||
|
|
"logps/chosen": -458.4798578199052,
|
||
|
|
"logps/rejected": -388.92358964451313,
|
||
|
|
"loss": 0.2237,
|
||
|
|
"loss/base_kto": 1.7892799377441406,
|
||
|
|
"metrics/advantage_var": 14245.1484375,
|
||
|
|
"rewards/chosen": 2.5022184784755135,
|
||
|
|
"rewards/margins": 5.4406616461605015,
|
||
|
|
"rewards/rejected": -2.9384431676849885,
|
||
|
|
"step": 2280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.9792746113989637,
|
||
|
|
"grad_norm": 8.932574272155762,
|
||
|
|
"kl": 16.011707305908203,
|
||
|
|
"learning_rate": 1.4520349279739663e-10,
|
||
|
|
"logits/chosen": -52867014.30985916,
|
||
|
|
"logits/rejected": -54209454.527301095,
|
||
|
|
"logps/chosen": -454.9545187793427,
|
||
|
|
"logps/rejected": -377.59572445397816,
|
||
|
|
"loss": 0.207,
|
||
|
|
"loss/base_kto": 1.6560556888580322,
|
||
|
|
"metrics/advantage_var": 14029.0283203125,
|
||
|
|
"rewards/chosen": 2.5530711668011543,
|
||
|
|
"rewards/margins": 5.502070533025803,
|
||
|
|
"rewards/rejected": -2.948999366224649,
|
||
|
|
"step": 2300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 3.0,
|
||
|
|
"step": 2316,
|
||
|
|
"total_flos": 9.972016217523487e+17,
|
||
|
|
"train_loss": 0.27589865288792087,
|
||
|
|
"train_runtime": 11043.1422,
|
||
|
|
"train_samples_per_second": 13.422,
|
||
|
|
"train_steps_per_second": 0.21
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 20,
|
||
|
|
"max_steps": 2316,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 3,
|
||
|
|
"save_steps": 500,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": false,
|
||
|
|
"should_training_stop": false
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 9.972016217523487e+17,
|
||
|
|
"train_batch_size": 8,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|