2229 lines
81 KiB
JSON
2229 lines
81 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 3.0,
|
|
"eval_steps": 500,
|
|
"global_step": 2316,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.025906735751295335,
|
|
"grad_norm": 21.025110244750977,
|
|
"kl": 17.884946823120117,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"logits/chosen": -37365357.93865031,
|
|
"logits/rejected": -38282412.84076433,
|
|
"logps/chosen": -458.23054064417175,
|
|
"logps/rejected": -379.5640425955414,
|
|
"loss": 0.6021,
|
|
"loss/base_kto": 3.915940523147583,
|
|
"metrics/advantage_var": 203.65585327148438,
|
|
"regularization/lipschitz_norm": 930.5560913085938,
|
|
"regularization/w1": 0.9007782340049744,
|
|
"rewards/chosen": 0.25468828634250384,
|
|
"rewards/margins": 0.04719344128627376,
|
|
"rewards/rejected": 0.2074948450562301,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.05181347150259067,
|
|
"grad_norm": 27.312768936157227,
|
|
"kl": 8.006775856018066,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"logits/chosen": -36416431.24290221,
|
|
"logits/rejected": -37550346.303405575,
|
|
"logps/chosen": -479.43838722397476,
|
|
"logps/rejected": -363.0922020123839,
|
|
"loss": 0.613,
|
|
"loss/base_kto": 3.940383195877075,
|
|
"metrics/advantage_var": 288.2891845703125,
|
|
"regularization/lipschitz_norm": 995.1353759765625,
|
|
"regularization/w1": 0.9632909893989563,
|
|
"rewards/chosen": 0.08643815795705898,
|
|
"rewards/margins": 0.0655225330609888,
|
|
"rewards/rejected": 0.020915624896070167,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.07772020725388601,
|
|
"grad_norm": 23.3543701171875,
|
|
"kl": 16.31055450439453,
|
|
"learning_rate": 5.9e-07,
|
|
"logits/chosen": -35587574.964705884,
|
|
"logits/rejected": -37091164.16,
|
|
"logps/chosen": -462.7930606617647,
|
|
"logps/rejected": -381.9289322916667,
|
|
"loss": 0.5909,
|
|
"loss/base_kto": 3.8472793102264404,
|
|
"metrics/advantage_var": 185.0529022216797,
|
|
"regularization/lipschitz_norm": 908.7521362304688,
|
|
"regularization/w1": 0.8796720504760742,
|
|
"rewards/chosen": 0.26407690609202666,
|
|
"rewards/margins": 0.14243373796051623,
|
|
"rewards/rejected": 0.12164316813151042,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.10362694300518134,
|
|
"grad_norm": 22.52024269104004,
|
|
"kl": 13.733599662780762,
|
|
"learning_rate": 7.9e-07,
|
|
"logits/chosen": -35642301.03987241,
|
|
"logits/rejected": -36969072.122511484,
|
|
"logps/chosen": -486.08931419457736,
|
|
"logps/rejected": -384.1779287901991,
|
|
"loss": 0.5946,
|
|
"loss/base_kto": 3.894824266433716,
|
|
"metrics/advantage_var": 177.49766540527344,
|
|
"regularization/lipschitz_norm": 890.3466796875,
|
|
"regularization/w1": 0.8618556261062622,
|
|
"rewards/chosen": 0.1831431776712956,
|
|
"rewards/margins": 0.08566634696244282,
|
|
"rewards/rejected": 0.09747683070885277,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.12953367875647667,
|
|
"grad_norm": 21.335416793823242,
|
|
"kl": 16.760305404663086,
|
|
"learning_rate": 9.9e-07,
|
|
"logits/chosen": -37779421.96978852,
|
|
"logits/rejected": -37832293.074433655,
|
|
"logps/chosen": -480.33128776435046,
|
|
"logps/rejected": -369.65713491100325,
|
|
"loss": 0.5974,
|
|
"loss/base_kto": 3.840963363647461,
|
|
"metrics/advantage_var": 226.6090850830078,
|
|
"regularization/lipschitz_norm": 969.18359375,
|
|
"regularization/w1": 0.9381695985794067,
|
|
"rewards/chosen": 0.33794018149015764,
|
|
"rewards/margins": 0.14155198624244578,
|
|
"rewards/rejected": 0.19638819524771187,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.15544041450777202,
|
|
"grad_norm": 18.12994956970215,
|
|
"kl": 23.64974594116211,
|
|
"learning_rate": 9.998186234298189e-07,
|
|
"logits/chosen": -36708412.23529412,
|
|
"logits/rejected": -36441805.12302839,
|
|
"logps/chosen": -468.3295762383901,
|
|
"logps/rejected": -342.2988712539432,
|
|
"loss": 0.5903,
|
|
"loss/base_kto": 3.8368747234344482,
|
|
"metrics/advantage_var": 188.5988311767578,
|
|
"regularization/lipschitz_norm": 914.9808959960938,
|
|
"regularization/w1": 0.8857014775276184,
|
|
"rewards/chosen": 0.3696199304917279,
|
|
"rewards/margins": 0.15209650608444283,
|
|
"rewards/rejected": 0.2175234244072851,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.18134715025906736,
|
|
"grad_norm": 23.3071346282959,
|
|
"kl": 12.652411460876465,
|
|
"learning_rate": 9.992359552107714e-07,
|
|
"logits/chosen": -36364664.49445325,
|
|
"logits/rejected": -37326598.705701075,
|
|
"logps/chosen": -472.49742472266246,
|
|
"logps/rejected": -378.2548632511556,
|
|
"loss": 0.597,
|
|
"loss/base_kto": 3.830272674560547,
|
|
"metrics/advantage_var": 198.0278778076172,
|
|
"regularization/lipschitz_norm": 977.1329956054688,
|
|
"regularization/w1": 0.9458647966384888,
|
|
"rewards/chosen": 0.25103769438391443,
|
|
"rewards/margins": 0.1762750531942687,
|
|
"rewards/rejected": 0.07476264118964573,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.20725388601036268,
|
|
"grad_norm": 19.439594268798828,
|
|
"kl": 13.954642295837402,
|
|
"learning_rate": 9.982519612796161e-07,
|
|
"logits/chosen": -36075934.784810126,
|
|
"logits/rejected": -37502849.58024691,
|
|
"logps/chosen": -489.05795094936707,
|
|
"logps/rejected": -367.4696662808642,
|
|
"loss": 0.5944,
|
|
"loss/base_kto": 3.816574811935425,
|
|
"metrics/advantage_var": 201.4392852783203,
|
|
"regularization/lipschitz_norm": 970.0056762695312,
|
|
"regularization/w1": 0.9389654994010925,
|
|
"rewards/chosen": 0.25878925564922867,
|
|
"rewards/margins": 0.17822355582464822,
|
|
"rewards/rejected": 0.08056569982458044,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.23316062176165803,
|
|
"grad_norm": 20.076038360595703,
|
|
"kl": 9.410046577453613,
|
|
"learning_rate": 9.968674326492213e-07,
|
|
"logits/chosen": -36244703.594936706,
|
|
"logits/rejected": -38545262.61728395,
|
|
"logps/chosen": -520.6321696993671,
|
|
"logps/rejected": -395.3758921682099,
|
|
"loss": 0.6085,
|
|
"loss/base_kto": 3.821716070175171,
|
|
"metrics/advantage_var": 237.119140625,
|
|
"regularization/lipschitz_norm": 1080.6380615234375,
|
|
"regularization/w1": 1.0460575819015503,
|
|
"rewards/chosen": 0.0688416444802586,
|
|
"rewards/margins": 0.16230471802234725,
|
|
"rewards/rejected": -0.09346307354208863,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.25906735751295334,
|
|
"grad_norm": 23.015117645263672,
|
|
"kl": 10.640921592712402,
|
|
"learning_rate": 9.950834823142198e-07,
|
|
"logits/chosen": -36794961.52866242,
|
|
"logits/rejected": -38058430.03680982,
|
|
"logps/chosen": -518.9075935509554,
|
|
"logps/rejected": -361.63587998466255,
|
|
"loss": 0.5899,
|
|
"loss/base_kto": 3.789363145828247,
|
|
"metrics/advantage_var": 214.5233612060547,
|
|
"regularization/lipschitz_norm": 960.4099731445312,
|
|
"regularization/w1": 0.9296768307685852,
|
|
"rewards/chosen": 0.1530699517316879,
|
|
"rewards/margins": 0.2038959425138815,
|
|
"rewards/rejected": -0.05082599078219361,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.2849740932642487,
|
|
"grad_norm": 22.466655731201172,
|
|
"kl": 6.4207682609558105,
|
|
"learning_rate": 9.929015443562942e-07,
|
|
"logits/chosen": -37311027.848101266,
|
|
"logits/rejected": -37460992.0,
|
|
"logps/chosen": -471.9447685917722,
|
|
"logps/rejected": -413.5896508487654,
|
|
"loss": 0.5979,
|
|
"loss/base_kto": 3.8489441871643066,
|
|
"metrics/advantage_var": 212.0841827392578,
|
|
"regularization/lipschitz_norm": 964.771484375,
|
|
"regularization/w1": 0.9338987469673157,
|
|
"rewards/chosen": 0.037564694126950036,
|
|
"rewards/margins": 0.13484609713869591,
|
|
"rewards/rejected": -0.09728140301174587,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.31088082901554404,
|
|
"grad_norm": 17.599266052246094,
|
|
"kl": 15.75390625,
|
|
"learning_rate": 9.90323372791347e-07,
|
|
"logits/chosen": -35155466.581875995,
|
|
"logits/rejected": -34671661.615975425,
|
|
"logps/chosen": -491.6408485691574,
|
|
"logps/rejected": -387.7569604454685,
|
|
"loss": 0.5901,
|
|
"loss/base_kto": 3.7492966651916504,
|
|
"metrics/advantage_var": 238.36729431152344,
|
|
"regularization/lipschitz_norm": 1003.5248413085938,
|
|
"regularization/w1": 0.9714121222496033,
|
|
"rewards/chosen": 0.2973107851935115,
|
|
"rewards/margins": 0.25503296124871655,
|
|
"rewards/rejected": 0.042277823944794964,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.33678756476683935,
|
|
"grad_norm": 23.92975425720215,
|
|
"kl": 12.21552562713623,
|
|
"learning_rate": 9.87351040159484e-07,
|
|
"logits/chosen": -36237630.8502994,
|
|
"logits/rejected": -37203927.84313726,
|
|
"logps/chosen": -476.83004303892216,
|
|
"logps/rejected": -352.4202920751634,
|
|
"loss": 0.5994,
|
|
"loss/base_kto": 3.8031787872314453,
|
|
"metrics/advantage_var": 247.7328643798828,
|
|
"regularization/lipschitz_norm": 1025.1199951171875,
|
|
"regularization/w1": 0.9923161864280701,
|
|
"rewards/chosen": 0.2277440510824055,
|
|
"rewards/margins": 0.17849620341937383,
|
|
"rewards/rejected": 0.04924784766303168,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.3626943005181347,
|
|
"grad_norm": 19.471908569335938,
|
|
"kl": 15.156867027282715,
|
|
"learning_rate": 9.839869358589396e-07,
|
|
"logits/chosen": -36620967.56363636,
|
|
"logits/rejected": -37121526.090322584,
|
|
"logps/chosen": -485.28669507575756,
|
|
"logps/rejected": -359.53321572580643,
|
|
"loss": 0.5936,
|
|
"loss/base_kto": 3.734309434890747,
|
|
"metrics/advantage_var": 276.2356872558594,
|
|
"regularization/lipschitz_norm": 1047.706787109375,
|
|
"regularization/w1": 1.0141801834106445,
|
|
"rewards/chosen": 0.3124423865116004,
|
|
"rewards/margins": 0.24593759426739675,
|
|
"rewards/rejected": 0.06650479224420362,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.38860103626943004,
|
|
"grad_norm": 19.602449417114258,
|
|
"kl": 12.539558410644531,
|
|
"learning_rate": 9.80233764225289e-07,
|
|
"logits/chosen": -36373920.57050243,
|
|
"logits/rejected": -36801262.62443439,
|
|
"logps/chosen": -494.39723460291737,
|
|
"logps/rejected": -369.23934766214177,
|
|
"loss": 0.5855,
|
|
"loss/base_kto": 3.709517002105713,
|
|
"metrics/advantage_var": 230.6935272216797,
|
|
"regularization/lipschitz_norm": 1006.8311767578125,
|
|
"regularization/w1": 0.9746125340461731,
|
|
"rewards/chosen": 0.1949175520886092,
|
|
"rewards/margins": 0.2636557213487637,
|
|
"rewards/rejected": -0.06873816926015448,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.41450777202072536,
|
|
"grad_norm": 30.41884422302246,
|
|
"kl": 14.273750305175781,
|
|
"learning_rate": 9.760945423574868e-07,
|
|
"logits/chosen": -36266737.664,
|
|
"logits/rejected": -36792366.900763355,
|
|
"logps/chosen": -467.03355,
|
|
"logps/rejected": -394.8055104961832,
|
|
"loss": 0.6132,
|
|
"loss/base_kto": 3.8142249584198,
|
|
"metrics/advantage_var": 344.7950134277344,
|
|
"regularization/lipschitz_norm": 1127.3157958984375,
|
|
"regularization/w1": 1.0912418365478516,
|
|
"rewards/chosen": 0.196239599609375,
|
|
"rewards/margins": 0.16764388926265805,
|
|
"rewards/rejected": 0.02859571034671696,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.44041450777202074,
|
|
"grad_norm": 17.971755981445312,
|
|
"kl": 17.654523849487305,
|
|
"learning_rate": 9.71572597692482e-07,
|
|
"logits/chosen": -36827531.173047476,
|
|
"logits/rejected": -37443430.48165869,
|
|
"logps/chosen": -465.478943338438,
|
|
"logps/rejected": -361.5318480861244,
|
|
"loss": 0.5973,
|
|
"loss/base_kto": 3.7974212169647217,
|
|
"metrics/advantage_var": 233.09898376464844,
|
|
"regularization/lipschitz_norm": 1013.0390625,
|
|
"regularization/w1": 0.9806217551231384,
|
|
"rewards/chosen": 0.2925348208837816,
|
|
"rewards/margins": 0.18011168916310027,
|
|
"rewards/rejected": 0.11242313172068132,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.46632124352331605,
|
|
"grad_norm": 31.38916015625,
|
|
"kl": 8.756335258483887,
|
|
"learning_rate": 9.666715653303588e-07,
|
|
"logits/chosen": -36462891.02886248,
|
|
"logits/rejected": -37140394.04920405,
|
|
"logps/chosen": -507.6125848896435,
|
|
"logps/rejected": -369.0596960926194,
|
|
"loss": 0.5895,
|
|
"loss/base_kto": 3.728153944015503,
|
|
"metrics/advantage_var": 239.4797821044922,
|
|
"regularization/lipschitz_norm": 1020.1640625,
|
|
"regularization/w1": 0.9875189065933228,
|
|
"rewards/chosen": 0.02916706519134988,
|
|
"rewards/margins": 0.22851897076015867,
|
|
"rewards/rejected": -0.1993519055688088,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.49222797927461137,
|
|
"grad_norm": 16.97059440612793,
|
|
"kl": 3.486614227294922,
|
|
"learning_rate": 9.613953851121528e-07,
|
|
"logits/chosen": -35607642.35294118,
|
|
"logits/rejected": -35927898.443113774,
|
|
"logps/chosen": -454.1840788398693,
|
|
"logps/rejected": -358.99178985778445,
|
|
"loss": 0.5896,
|
|
"loss/base_kto": 3.7648956775665283,
|
|
"metrics/advantage_var": 245.7963409423828,
|
|
"regularization/lipschitz_norm": 983.54443359375,
|
|
"regularization/w1": 0.9520710110664368,
|
|
"rewards/chosen": -0.143492904363894,
|
|
"rewards/margins": 0.20866043970480452,
|
|
"rewards/rejected": -0.3521533440686985,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.5181347150259067,
|
|
"grad_norm": 20.872982025146484,
|
|
"kl": 0.5386886596679688,
|
|
"learning_rate": 9.557482984526924e-07,
|
|
"logits/chosen": -34554314.01916933,
|
|
"logits/rejected": -35879431.82874618,
|
|
"logps/chosen": -481.6983326677316,
|
|
"logps/rejected": -379.27814411314984,
|
|
"loss": 0.5953,
|
|
"loss/base_kto": 3.794321060180664,
|
|
"metrics/advantage_var": 223.24380493164062,
|
|
"regularization/lipschitz_norm": 1000.4750366210938,
|
|
"regularization/w1": 0.9684597253799438,
|
|
"rewards/chosen": -0.296756829697484,
|
|
"rewards/margins": 0.20141230226686996,
|
|
"rewards/rejected": -0.498169131964354,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.5440414507772021,
|
|
"grad_norm": 18.131797790527344,
|
|
"kl": 4.174106121063232,
|
|
"learning_rate": 9.497348449310107e-07,
|
|
"logits/chosen": -36220364.102874435,
|
|
"logits/rejected": -36813531.19224556,
|
|
"logps/chosen": -485.3890885022693,
|
|
"logps/rejected": -375.6084915185784,
|
|
"loss": 0.5884,
|
|
"loss/base_kto": 3.710728406906128,
|
|
"metrics/advantage_var": 244.85513305664062,
|
|
"regularization/lipschitz_norm": 1029.1322021484375,
|
|
"regularization/w1": 0.9962000250816345,
|
|
"rewards/chosen": 0.08689130128061176,
|
|
"rewards/margins": 0.3007636161281481,
|
|
"rewards/rejected": -0.21387231484753635,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.5699481865284974,
|
|
"grad_norm": 18.12434959411621,
|
|
"kl": 3.3283486366271973,
|
|
"learning_rate": 9.433598586410701e-07,
|
|
"logits/chosen": -35386618.98039216,
|
|
"logits/rejected": -36065979.01796407,
|
|
"logps/chosen": -488.015318627451,
|
|
"logps/rejected": -381.42058851047904,
|
|
"loss": 0.5881,
|
|
"loss/base_kto": 3.782311201095581,
|
|
"metrics/advantage_var": 210.64688110351562,
|
|
"regularization/lipschitz_norm": 953.1384887695312,
|
|
"regularization/w1": 0.9226380586624146,
|
|
"rewards/chosen": -0.1236166361889808,
|
|
"rewards/margins": 0.19476891631441087,
|
|
"rewards/rejected": -0.3183855525033917,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.5958549222797928,
|
|
"grad_norm": 28.6573486328125,
|
|
"kl": 7.071713447570801,
|
|
"learning_rate": 9.366284643057313e-07,
|
|
"logits/chosen": -35926649.135646686,
|
|
"logits/rejected": -37297513.4117647,
|
|
"logps/chosen": -496.1498915615142,
|
|
"logps/rejected": -359.3993566176471,
|
|
"loss": 0.5879,
|
|
"loss/base_kto": 3.6792073249816895,
|
|
"metrics/advantage_var": 269.59552001953125,
|
|
"regularization/lipschitz_norm": 1057.7291259765625,
|
|
"regularization/w1": 1.0238817930221558,
|
|
"rewards/chosen": 0.08306384161819795,
|
|
"rewards/margins": 0.3106822214768231,
|
|
"rewards/rejected": -0.22761837985862518,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.6217616580310881,
|
|
"grad_norm": 24.90556526184082,
|
|
"kl": 1.9013442993164062,
|
|
"learning_rate": 9.295460731570917e-07,
|
|
"logits/chosen": -36500341.44945568,
|
|
"logits/rejected": -36897320.99215071,
|
|
"logps/chosen": -469.07518468118195,
|
|
"logps/rejected": -380.4013196624804,
|
|
"loss": 0.5959,
|
|
"loss/base_kto": 3.786285161972046,
|
|
"metrics/advantage_var": 239.63487243652344,
|
|
"regularization/lipschitz_norm": 1013.4304809570312,
|
|
"regularization/w1": 0.981000542640686,
|
|
"rewards/chosen": -0.10877354742023473,
|
|
"rewards/margins": 0.2145892970978309,
|
|
"rewards/rejected": -0.32336284451806563,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.6476683937823834,
|
|
"grad_norm": 22.47072410583496,
|
|
"kl": 9.251765251159668,
|
|
"learning_rate": 9.221183785865056e-07,
|
|
"logits/chosen": -36450395.60383387,
|
|
"logits/rejected": -38275497.88379205,
|
|
"logps/chosen": -506.5962959265176,
|
|
"logps/rejected": -361.931312117737,
|
|
"loss": 0.5938,
|
|
"loss/base_kto": 3.7474324703216553,
|
|
"metrics/advantage_var": 252.8069610595703,
|
|
"regularization/lipschitz_norm": 1036.5286865234375,
|
|
"regularization/w1": 1.0033597946166992,
|
|
"rewards/chosen": 0.11604803895797973,
|
|
"rewards/margins": 0.21459335840142868,
|
|
"rewards/rejected": -0.09854531944344896,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.6735751295336787,
|
|
"grad_norm": 20.507471084594727,
|
|
"kl": 5.03880500793457,
|
|
"learning_rate": 9.143513515677817e-07,
|
|
"logits/chosen": -34750984.95389507,
|
|
"logits/rejected": -35568770.556067586,
|
|
"logps/chosen": -494.1115858505564,
|
|
"logps/rejected": -365.92693932411674,
|
|
"loss": 0.5961,
|
|
"loss/base_kto": 3.7468647956848145,
|
|
"metrics/advantage_var": 250.00405883789062,
|
|
"regularization/lipschitz_norm": 1055.8797607421875,
|
|
"regularization/w1": 1.0220916271209717,
|
|
"rewards/chosen": -0.052434487638488673,
|
|
"rewards/margins": 0.24706647729826725,
|
|
"rewards/rejected": -0.29950096493675593,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.6994818652849741,
|
|
"grad_norm": 17.546710968017578,
|
|
"kl": 4.73122501373291,
|
|
"learning_rate": 9.062512358572373e-07,
|
|
"logits/chosen": -37730722.687797144,
|
|
"logits/rejected": -38295618.26810478,
|
|
"logps/chosen": -504.9932151347068,
|
|
"logps/rejected": -371.80727561633284,
|
|
"loss": 0.5853,
|
|
"loss/base_kto": 3.669316053390503,
|
|
"metrics/advantage_var": 250.6295928955078,
|
|
"regularization/lipschitz_norm": 1046.5223388671875,
|
|
"regularization/w1": 1.0130335092544556,
|
|
"rewards/chosen": 0.06241330783076218,
|
|
"rewards/margins": 0.3358486447974899,
|
|
"rewards/rejected": -0.27343533696672767,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.7253886010362695,
|
|
"grad_norm": 26.337894439697266,
|
|
"kl": 6.717900276184082,
|
|
"learning_rate": 8.978245429744691e-07,
|
|
"logits/chosen": -35879189.8044164,
|
|
"logits/rejected": -37414303.306501545,
|
|
"logps/chosen": -473.075660488959,
|
|
"logps/rejected": -363.0831317724458,
|
|
"loss": 0.5945,
|
|
"loss/base_kto": 3.766246795654297,
|
|
"metrics/advantage_var": 261.7023010253906,
|
|
"regularization/lipschitz_norm": 1022.6945190429688,
|
|
"regularization/w1": 0.9899682998657227,
|
|
"rewards/chosen": 0.006618206433317263,
|
|
"rewards/margins": 0.21099697814142485,
|
|
"rewards/rejected": -0.2043787717081076,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.7512953367875648,
|
|
"grad_norm": 24.658538818359375,
|
|
"kl": 6.590710639953613,
|
|
"learning_rate": 8.890780469678738e-07,
|
|
"logits/chosen": -36786272.452119306,
|
|
"logits/rejected": -37790202.42612753,
|
|
"logps/chosen": -471.8648940345369,
|
|
"logps/rejected": -385.26095937014,
|
|
"loss": 0.5953,
|
|
"loss/base_kto": 3.7582104206085205,
|
|
"metrics/advantage_var": 271.61322021484375,
|
|
"regularization/lipschitz_norm": 1037.139892578125,
|
|
"regularization/w1": 1.0039513111114502,
|
|
"rewards/chosen": 0.08973833380538965,
|
|
"rewards/margins": 0.2203102953936782,
|
|
"rewards/rejected": -0.13057196158828854,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.7772020725388601,
|
|
"grad_norm": 18.163043975830078,
|
|
"kl": 8.109402656555176,
|
|
"learning_rate": 8.800187789691269e-07,
|
|
"logits/chosen": -36896270.44514106,
|
|
"logits/rejected": -38517629.20872274,
|
|
"logps/chosen": -502.2341300940439,
|
|
"logps/rejected": -381.8467679127726,
|
|
"loss": 0.5978,
|
|
"loss/base_kto": 3.7055413722991943,
|
|
"metrics/advantage_var": 270.1628112792969,
|
|
"regularization/lipschitz_norm": 1112.8507080078125,
|
|
"regularization/w1": 1.0772393941879272,
|
|
"rewards/chosen": 0.1265721575219803,
|
|
"rewards/margins": 0.2647886931436947,
|
|
"rewards/rejected": -0.13821653562171438,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.8031088082901554,
|
|
"grad_norm": 20.029281616210938,
|
|
"kl": 8.550772666931152,
|
|
"learning_rate": 8.706540215409981e-07,
|
|
"logits/chosen": -36633565.2345679,
|
|
"logits/rejected": -37684962.835443035,
|
|
"logps/chosen": -512.2664930555555,
|
|
"logps/rejected": -355.15822784810126,
|
|
"loss": 0.6043,
|
|
"loss/base_kto": 3.7266175746917725,
|
|
"metrics/advantage_var": 276.9561462402344,
|
|
"regularization/lipschitz_norm": 1144.205322265625,
|
|
"regularization/w1": 1.1075905561447144,
|
|
"rewards/chosen": 0.10596262378457152,
|
|
"rewards/margins": 0.2601121983540805,
|
|
"rewards/rejected": -0.154149574569509,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.8290155440414507,
|
|
"grad_norm": 23.614953994750977,
|
|
"kl": 10.841440200805664,
|
|
"learning_rate": 8.609913028230462e-07,
|
|
"logits/chosen": -36445322.37837838,
|
|
"logits/rejected": -37434868.3257329,
|
|
"logps/chosen": -477.1833708708709,
|
|
"logps/rejected": -370.58718444625407,
|
|
"loss": 0.5932,
|
|
"loss/base_kto": 3.6958107948303223,
|
|
"metrics/advantage_var": 262.4812316894531,
|
|
"regularization/lipschitz_norm": 1084.862548828125,
|
|
"regularization/w1": 1.0501469373703003,
|
|
"rewards/chosen": 0.2390412110108155,
|
|
"rewards/margins": 0.27143374390618014,
|
|
"rewards/rejected": -0.032392532895364666,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.8549222797927462,
|
|
"grad_norm": 19.80830955505371,
|
|
"kl": 12.25365924835205,
|
|
"learning_rate": 8.510383904798994e-07,
|
|
"logits/chosen": -36969143.83382789,
|
|
"logits/rejected": -37143487.78877888,
|
|
"logps/chosen": -472.02499072700294,
|
|
"logps/rejected": -382.21176258250824,
|
|
"loss": 0.5975,
|
|
"loss/base_kto": 3.7258071899414062,
|
|
"metrics/advantage_var": 252.5930633544922,
|
|
"regularization/lipschitz_norm": 1089.2669677734375,
|
|
"regularization/w1": 1.0544103384017944,
|
|
"rewards/chosen": 0.252256240618335,
|
|
"rewards/margins": 0.2503932087274155,
|
|
"rewards/rejected": 0.0018630318909194997,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.8808290155440415,
|
|
"grad_norm": 22.702198028564453,
|
|
"kl": 19.946395874023438,
|
|
"learning_rate": 8.408032854569865e-07,
|
|
"logits/chosen": -36473962.985074624,
|
|
"logits/rejected": -36656322.727868855,
|
|
"logps/chosen": -487.625,
|
|
"logps/rejected": -374.4955430327869,
|
|
"loss": 0.5919,
|
|
"loss/base_kto": 3.726224660873413,
|
|
"metrics/advantage_var": 241.41099548339844,
|
|
"regularization/lipschitz_norm": 1042.0133056640625,
|
|
"regularization/w1": 1.0086688995361328,
|
|
"rewards/chosen": 0.3612498724638526,
|
|
"rewards/margins": 0.21583413800080187,
|
|
"rewards/rejected": 0.14541573446305073,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.9067357512953368,
|
|
"grad_norm": 18.53467559814453,
|
|
"kl": 16.435373306274414,
|
|
"learning_rate": 8.302942155487377e-07,
|
|
"logits/chosen": -36852501.6677116,
|
|
"logits/rejected": -37943509.732087225,
|
|
"logps/chosen": -503.9807014106583,
|
|
"logps/rejected": -397.40917056074767,
|
|
"loss": 0.6013,
|
|
"loss/base_kto": 3.7544076442718506,
|
|
"metrics/advantage_var": 254.3803253173828,
|
|
"regularization/lipschitz_norm": 1091.273193359375,
|
|
"regularization/w1": 1.0563524961471558,
|
|
"rewards/chosen": 0.28470121282021454,
|
|
"rewards/margins": 0.23722534749476215,
|
|
"rewards/rejected": 0.04747586532545238,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.9326424870466321,
|
|
"grad_norm": 21.778186798095703,
|
|
"kl": 19.962522506713867,
|
|
"learning_rate": 8.195196287844278e-07,
|
|
"logits/chosen": -35755665.38271605,
|
|
"logits/rejected": -37403887.79746836,
|
|
"logps/chosen": -488.8063753858025,
|
|
"logps/rejected": -359.5591871044304,
|
|
"loss": 0.5913,
|
|
"loss/base_kto": 3.6668612957000732,
|
|
"metrics/advantage_var": 275.1253356933594,
|
|
"regularization/lipschitz_norm": 1098.4267578125,
|
|
"regularization/w1": 1.0632771253585815,
|
|
"rewards/chosen": 0.3463015144253955,
|
|
"rewards/margins": 0.2926325506671441,
|
|
"rewards/rejected": 0.05366896375825134,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.9585492227979274,
|
|
"grad_norm": 20.468721389770508,
|
|
"kl": 11.175115585327148,
|
|
"learning_rate": 8.08488186636975e-07,
|
|
"logits/chosen": -36549664.820512824,
|
|
"logits/rejected": -36053822.43902439,
|
|
"logps/chosen": -485.66225961538464,
|
|
"logps/rejected": -384.3454649390244,
|
|
"loss": 0.6022,
|
|
"loss/base_kto": 3.7352349758148193,
|
|
"metrics/advantage_var": 276.24713134765625,
|
|
"regularization/lipschitz_norm": 1118.5439453125,
|
|
"regularization/w1": 1.0827505588531494,
|
|
"rewards/chosen": 0.08928127777882111,
|
|
"rewards/margins": 0.2386458422557647,
|
|
"rewards/rejected": -0.1493645644769436,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.9844559585492227,
|
|
"grad_norm": 26.680635452270508,
|
|
"kl": 22.314111709594727,
|
|
"learning_rate": 7.972087570601576e-07,
|
|
"logits/chosen": -35876984.29368576,
|
|
"logits/rejected": -36651705.48247079,
|
|
"logps/chosen": -486.5537353157122,
|
|
"logps/rejected": -367.5164858096828,
|
|
"loss": 0.592,
|
|
"loss/base_kto": 3.6413025856018066,
|
|
"metrics/advantage_var": 287.0975341796875,
|
|
"regularization/lipschitz_norm": 1130.9725341796875,
|
|
"regularization/w1": 1.0947813987731934,
|
|
"rewards/chosen": 0.4582015386236922,
|
|
"rewards/margins": 0.3339961929221975,
|
|
"rewards/rejected": 0.12420534570149468,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 1.0103626943005182,
|
|
"grad_norm": 19.050148010253906,
|
|
"kl": 21.00868034362793,
|
|
"learning_rate": 7.856904073598458e-07,
|
|
"logits/chosen": -36205102.2503962,
|
|
"logits/rejected": -36348233.19938176,
|
|
"logps/chosen": -469.5121830427892,
|
|
"logps/rejected": -360.2412335780526,
|
|
"loss": 0.5923,
|
|
"loss/base_kto": 3.6048240661621094,
|
|
"metrics/advantage_var": 343.6983337402344,
|
|
"regularization/lipschitz_norm": 1171.210205078125,
|
|
"regularization/w1": 1.133731484413147,
|
|
"rewards/chosen": 0.4154094810909023,
|
|
"rewards/margins": 0.3731467095080409,
|
|
"rewards/rejected": 0.0422627715828614,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 1.0362694300518134,
|
|
"grad_norm": 20.98898696899414,
|
|
"kl": 15.312861442565918,
|
|
"learning_rate": 7.739423969049761e-07,
|
|
"logits/chosen": -36659257.33125972,
|
|
"logits/rejected": -38118092.96075353,
|
|
"logps/chosen": -483.23673211508554,
|
|
"logps/rejected": -379.24774332810046,
|
|
"loss": 0.6152,
|
|
"loss/base_kto": 3.3470497131347656,
|
|
"metrics/advantage_var": 596.2748413085938,
|
|
"regularization/lipschitz_norm": 1626.3515625,
|
|
"regularization/w1": 1.5743082761764526,
|
|
"rewards/chosen": 0.44879553811175643,
|
|
"rewards/margins": 0.7036231123581016,
|
|
"rewards/rejected": -0.25482757424634517,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 1.0621761658031088,
|
|
"grad_norm": 28.101476669311523,
|
|
"kl": 22.525440216064453,
|
|
"learning_rate": 7.619741696841322e-07,
|
|
"logits/chosen": -36435304.2962963,
|
|
"logits/rejected": -38330821.67088608,
|
|
"logps/chosen": -492.3283179012346,
|
|
"logps/rejected": -356.2132367484177,
|
|
"loss": 0.5946,
|
|
"loss/base_kto": 3.263479709625244,
|
|
"metrics/advantage_var": 544.399169921875,
|
|
"regularization/lipschitz_norm": 1542.4775390625,
|
|
"regularization/w1": 1.4931182861328125,
|
|
"rewards/chosen": 0.5767975795416185,
|
|
"rewards/margins": 0.7730232649509712,
|
|
"rewards/rejected": -0.19622568540935276,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 1.0880829015544042,
|
|
"grad_norm": 20.487337112426758,
|
|
"kl": 10.790139198303223,
|
|
"learning_rate": 7.497953467137135e-07,
|
|
"logits/chosen": -36793257.6,
|
|
"logits/rejected": -37368236.8,
|
|
"logps/chosen": -472.59248046875,
|
|
"logps/rejected": -397.316064453125,
|
|
"loss": 0.6087,
|
|
"loss/base_kto": 3.279811143875122,
|
|
"metrics/advantage_var": 677.769287109375,
|
|
"regularization/lipschitz_norm": 1641.9990234375,
|
|
"regularization/w1": 1.589455008506775,
|
|
"rewards/chosen": 0.4584324359893799,
|
|
"rewards/margins": 0.8360633373260498,
|
|
"rewards/rejected": -0.37763090133666993,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 1.1139896373056994,
|
|
"grad_norm": 21.426246643066406,
|
|
"kl": 16.129098892211914,
|
|
"learning_rate": 7.37415718303793e-07,
|
|
"logits/chosen": -35670705.10691824,
|
|
"logits/rejected": -37336127.60248447,
|
|
"logps/chosen": -498.58392295597486,
|
|
"logps/rejected": -377.6365974378882,
|
|
"loss": 0.5985,
|
|
"loss/base_kto": 3.302999973297119,
|
|
"metrics/advantage_var": 613.1333618164062,
|
|
"regularization/lipschitz_norm": 1533.8465576171875,
|
|
"regularization/w1": 1.484763503074646,
|
|
"rewards/chosen": 0.5011485837540537,
|
|
"rewards/margins": 0.7670694004954284,
|
|
"rewards/rejected": -0.2659208167413747,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 1.1398963730569949,
|
|
"grad_norm": 16.76197052001953,
|
|
"kl": 15.906763076782227,
|
|
"learning_rate": 7.248452361878855e-07,
|
|
"logits/chosen": -35275843.43032159,
|
|
"logits/rejected": -36779630.23923445,
|
|
"logps/chosen": -446.0583843797856,
|
|
"logps/rejected": -381.70255183413076,
|
|
"loss": 0.6058,
|
|
"loss/base_kto": 3.334491491317749,
|
|
"metrics/advantage_var": 602.745361328125,
|
|
"regularization/lipschitz_norm": 1562.248046875,
|
|
"regularization/w1": 1.5122560262680054,
|
|
"rewards/chosen": 0.46790195716285415,
|
|
"rewards/margins": 0.7466924447899475,
|
|
"rewards/rejected": -0.2787904876270933,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 1.16580310880829,
|
|
"grad_norm": 17.927444458007812,
|
|
"kl": 14.64819622039795,
|
|
"learning_rate": 7.120940055229497e-07,
|
|
"logits/chosen": -35588947.99372057,
|
|
"logits/rejected": -36438156.14307932,
|
|
"logps/chosen": -470.8847625588697,
|
|
"logps/rejected": -370.08266912908243,
|
|
"loss": 0.6,
|
|
"loss/base_kto": 3.3011441230773926,
|
|
"metrics/advantage_var": 569.7094116210938,
|
|
"regularization/lipschitz_norm": 1548.290771484375,
|
|
"regularization/w1": 1.4987454414367676,
|
|
"rewards/chosen": 0.5717838592768838,
|
|
"rewards/margins": 0.7820919282954005,
|
|
"rewards/rejected": -0.21030806901851673,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 1.1917098445595855,
|
|
"grad_norm": 16.97763442993164,
|
|
"kl": 15.206074714660645,
|
|
"learning_rate": 6.991722767660556e-07,
|
|
"logits/chosen": -33858970.51564828,
|
|
"logits/rejected": -35707821.609195404,
|
|
"logps/chosen": -475.7306259314456,
|
|
"logps/rejected": -367.83566810344826,
|
|
"loss": 0.6073,
|
|
"loss/base_kto": 3.3158607482910156,
|
|
"metrics/advantage_var": 592.8421630859375,
|
|
"regularization/lipschitz_norm": 1593.4368896484375,
|
|
"regularization/w1": 1.5424467325210571,
|
|
"rewards/chosen": 0.5607204749935963,
|
|
"rewards/margins": 0.7207379802572522,
|
|
"rewards/rejected": -0.16001750526365585,
|
|
"step": 920
|
|
},
|
|
{
|
|
"epoch": 1.2176165803108807,
|
|
"grad_norm": 17.47994613647461,
|
|
"kl": 19.318700790405273,
|
|
"learning_rate": 6.860904374342499e-07,
|
|
"logits/chosen": -36048236.62908012,
|
|
"logits/rejected": -36748389.38613861,
|
|
"logps/chosen": -492.2660886498516,
|
|
"logps/rejected": -364.9637737211221,
|
|
"loss": 0.6295,
|
|
"loss/base_kto": 3.355252981185913,
|
|
"metrics/advantage_var": 805.7520141601562,
|
|
"regularization/lipschitz_norm": 1736.2386474609375,
|
|
"regularization/w1": 1.6806787252426147,
|
|
"rewards/chosen": 0.615691097270841,
|
|
"rewards/margins": 0.6931039826786445,
|
|
"rewards/rejected": -0.07741288540780347,
|
|
"step": 940
|
|
},
|
|
{
|
|
"epoch": 1.2435233160621761,
|
|
"grad_norm": 19.694547653198242,
|
|
"kl": 13.314984321594238,
|
|
"learning_rate": 6.7285900375424e-07,
|
|
"logits/chosen": -35409134.06369427,
|
|
"logits/rejected": -35681192.04907975,
|
|
"logps/chosen": -495.4216261942675,
|
|
"logps/rejected": -389.0094660659509,
|
|
"loss": 0.5994,
|
|
"loss/base_kto": 3.3425984382629395,
|
|
"metrics/advantage_var": 542.2902221679688,
|
|
"regularization/lipschitz_norm": 1500.9410400390625,
|
|
"regularization/w1": 1.4529107809066772,
|
|
"rewards/chosen": 0.4016315435907643,
|
|
"rewards/margins": 0.6922148206319596,
|
|
"rewards/rejected": -0.29058327704119535,
|
|
"step": 960
|
|
},
|
|
{
|
|
"epoch": 1.2694300518134716,
|
|
"grad_norm": 16.86559295654297,
|
|
"kl": 13.960928916931152,
|
|
"learning_rate": 6.594886122086123e-07,
|
|
"logits/chosen": -35313516.838323355,
|
|
"logits/rejected": -37749077.333333336,
|
|
"logps/chosen": -488.21903068862275,
|
|
"logps/rejected": -356.86611519607845,
|
|
"loss": 0.6047,
|
|
"loss/base_kto": 3.299733877182007,
|
|
"metrics/advantage_var": 598.1213989257812,
|
|
"regularization/lipschitz_norm": 1589.00390625,
|
|
"regularization/w1": 1.5381557941436768,
|
|
"rewards/chosen": 0.46841458075060816,
|
|
"rewards/margins": 0.7895914462722073,
|
|
"rewards/rejected": -0.32117686552159924,
|
|
"step": 980
|
|
},
|
|
{
|
|
"epoch": 1.2953367875647668,
|
|
"grad_norm": 17.032405853271484,
|
|
"kl": 36.559242248535156,
|
|
"learning_rate": 6.459900109853766e-07,
|
|
"logits/chosen": -35297239.476407915,
|
|
"logits/rejected": -36544018.902086675,
|
|
"logps/chosen": -487.763603500761,
|
|
"logps/rejected": -362.2292335473515,
|
|
"loss": 0.5983,
|
|
"loss/base_kto": 3.1903305053710938,
|
|
"metrics/advantage_var": 623.8605346679688,
|
|
"regularization/lipschitz_norm": 1649.0458984375,
|
|
"regularization/w1": 1.5962764024734497,
|
|
"rewards/chosen": 0.9171306081739916,
|
|
"rewards/margins": 0.8126239823816123,
|
|
"rewards/rejected": 0.10450662579237936,
|
|
"step": 1000
|
|
},
|
|
{
|
|
"epoch": 1.3212435233160622,
|
|
"grad_norm": 21.105398178100586,
|
|
"kl": 25.526243209838867,
|
|
"learning_rate": 6.323740513377171e-07,
|
|
"logits/chosen": -35300575.447409734,
|
|
"logits/rejected": -34945192.808709174,
|
|
"logps/chosen": -472.9589874411303,
|
|
"logps/rejected": -354.9222152021773,
|
|
"loss": 0.5874,
|
|
"loss/base_kto": 3.2690811157226562,
|
|
"metrics/advantage_var": 516.9476928710938,
|
|
"regularization/lipschitz_norm": 1477.0047607421875,
|
|
"regularization/w1": 1.42974054813385,
|
|
"rewards/chosen": 0.600369583680828,
|
|
"rewards/margins": 0.7541291855645458,
|
|
"rewards/rejected": -0.15375960188371768,
|
|
"step": 1020
|
|
},
|
|
{
|
|
"epoch": 1.3471502590673574,
|
|
"grad_norm": 13.378235816955566,
|
|
"kl": 21.708932876586914,
|
|
"learning_rate": 6.186516788608865e-07,
|
|
"logits/chosen": -35344169.44761905,
|
|
"logits/rejected": -35967908.627692305,
|
|
"logps/chosen": -478.65694444444443,
|
|
"logps/rejected": -372.5554326923077,
|
|
"loss": 0.5985,
|
|
"loss/base_kto": 3.213732957839966,
|
|
"metrics/advantage_var": 622.2228393554688,
|
|
"regularization/lipschitz_norm": 1626.468994140625,
|
|
"regularization/w1": 1.5744218826293945,
|
|
"rewards/chosen": 0.6992359948536706,
|
|
"rewards/margins": 0.876377202035161,
|
|
"rewards/rejected": -0.17714120718149037,
|
|
"step": 1040
|
|
},
|
|
{
|
|
"epoch": 1.3730569948186528,
|
|
"grad_norm": 18.647127151489258,
|
|
"kl": 10.48285961151123,
|
|
"learning_rate": 6.048339246932652e-07,
|
|
"logits/chosen": -36992763.46537842,
|
|
"logits/rejected": -36654718.640364185,
|
|
"logps/chosen": -521.9863627214171,
|
|
"logps/rejected": -379.23852427921094,
|
|
"loss": 0.6143,
|
|
"loss/base_kto": 3.3263046741485596,
|
|
"metrics/advantage_var": 669.826416015625,
|
|
"regularization/lipschitz_norm": 1640.3973388671875,
|
|
"regularization/w1": 1.5879043340682983,
|
|
"rewards/chosen": 0.29602095009624096,
|
|
"rewards/margins": 0.7560132387349217,
|
|
"rewards/rejected": -0.45999228863868075,
|
|
"step": 1060
|
|
},
|
|
{
|
|
"epoch": 1.3989637305699483,
|
|
"grad_norm": 22.58229637145996,
|
|
"kl": 7.155999660491943,
|
|
"learning_rate": 5.909318966486494e-07,
|
|
"logits/chosen": -35135925.6639478,
|
|
"logits/rejected": -36256035.69415292,
|
|
"logps/chosen": -462.15630097879284,
|
|
"logps/rejected": -379.56561562968517,
|
|
"loss": 0.5991,
|
|
"loss/base_kto": 3.3373146057128906,
|
|
"metrics/advantage_var": 530.9617309570312,
|
|
"regularization/lipschitz_norm": 1503.8431396484375,
|
|
"regularization/w1": 1.4557201862335205,
|
|
"rewards/chosen": 0.2899089271831668,
|
|
"rewards/margins": 0.728012461475591,
|
|
"rewards/rejected": -0.4381035342924241,
|
|
"step": 1080
|
|
},
|
|
{
|
|
"epoch": 1.4248704663212435,
|
|
"grad_norm": 25.21034812927246,
|
|
"kl": 6.888988018035889,
|
|
"learning_rate": 5.769567702869052e-07,
|
|
"logits/chosen": -35157406.784810126,
|
|
"logits/rejected": -36752396.641975306,
|
|
"logps/chosen": -491.6643591772152,
|
|
"logps/rejected": -387.1657986111111,
|
|
"loss": 0.6154,
|
|
"loss/base_kto": 3.2402515411376953,
|
|
"metrics/advantage_var": 743.0656127929688,
|
|
"regularization/lipschitz_norm": 1738.9886474609375,
|
|
"regularization/w1": 1.6833410263061523,
|
|
"rewards/chosen": 0.3672110400622404,
|
|
"rewards/margins": 0.8371742079827651,
|
|
"rewards/rejected": -0.46996316792052467,
|
|
"step": 1100
|
|
},
|
|
{
|
|
"epoch": 1.450777202072539,
|
|
"grad_norm": 24.018444061279297,
|
|
"kl": 10.88753604888916,
|
|
"learning_rate": 5.629197799301614e-07,
|
|
"logits/chosen": -34611270.954330705,
|
|
"logits/rejected": -35785139.00155039,
|
|
"logps/chosen": -454.80063976377954,
|
|
"logps/rejected": -369.83585271317827,
|
|
"loss": 0.6133,
|
|
"loss/base_kto": 3.394625186920166,
|
|
"metrics/advantage_var": 565.0968627929688,
|
|
"regularization/lipschitz_norm": 1561.531494140625,
|
|
"regularization/w1": 1.5115625858306885,
|
|
"rewards/chosen": 0.25888282595656986,
|
|
"rewards/margins": 0.645460438072001,
|
|
"rewards/rejected": -0.3865776121154312,
|
|
"step": 1120
|
|
},
|
|
{
|
|
"epoch": 1.4766839378238341,
|
|
"grad_norm": 22.305767059326172,
|
|
"kl": 17.678312301635742,
|
|
"learning_rate": 5.488322096317633e-07,
|
|
"logits/chosen": -34485634.56394453,
|
|
"logits/rejected": -35448685.946117274,
|
|
"logps/chosen": -471.58638289676423,
|
|
"logps/rejected": -359.3486776941363,
|
|
"loss": 0.6086,
|
|
"loss/base_kto": 3.3520774841308594,
|
|
"metrics/advantage_var": 577.2743530273438,
|
|
"regularization/lipschitz_norm": 1566.6103515625,
|
|
"regularization/w1": 1.5164788961410522,
|
|
"rewards/chosen": 0.5585925744384389,
|
|
"rewards/margins": 0.691815598737202,
|
|
"rewards/rejected": -0.13322302429876312,
|
|
"step": 1140
|
|
},
|
|
{
|
|
"epoch": 1.5025906735751295,
|
|
"grad_norm": 17.83325958251953,
|
|
"kl": 20.49886131286621,
|
|
"learning_rate": 5.347053841052518e-07,
|
|
"logits/chosen": -36128266.78768233,
|
|
"logits/rejected": -36684159.0346908,
|
|
"logps/chosen": -479.5831645056726,
|
|
"logps/rejected": -368.16640742835597,
|
|
"loss": 0.5997,
|
|
"loss/base_kto": 3.2682435512542725,
|
|
"metrics/advantage_var": 565.1885986328125,
|
|
"regularization/lipschitz_norm": 1580.0950927734375,
|
|
"regularization/w1": 1.5295318365097046,
|
|
"rewards/chosen": 0.5783309070951935,
|
|
"rewards/margins": 0.7609371280597663,
|
|
"rewards/rejected": -0.18260622096457274,
|
|
"step": 1160
|
|
},
|
|
{
|
|
"epoch": 1.528497409326425,
|
|
"grad_norm": 56.203582763671875,
|
|
"kl": 14.707659721374512,
|
|
"learning_rate": 5.205506596206531e-07,
|
|
"logits/chosen": -35382577.45904173,
|
|
"logits/rejected": -36586529.97156398,
|
|
"logps/chosen": -485.0324091962906,
|
|
"logps/rejected": -366.7071484992101,
|
|
"loss": 0.6185,
|
|
"loss/base_kto": 3.33923602104187,
|
|
"metrics/advantage_var": 803.1249389648438,
|
|
"regularization/lipschitz_norm": 1662.2376708984375,
|
|
"regularization/w1": 1.609046220779419,
|
|
"rewards/chosen": 0.4047441202485027,
|
|
"rewards/margins": 0.7113397161521855,
|
|
"rewards/rejected": -0.30659559590368285,
|
|
"step": 1180
|
|
},
|
|
{
|
|
"epoch": 1.5544041450777202,
|
|
"grad_norm": 21.07695960998535,
|
|
"kl": 11.48717212677002,
|
|
"learning_rate": 5.063794148754032e-07,
|
|
"logits/chosen": -35615333.76397516,
|
|
"logits/rejected": -37745213.18238994,
|
|
"logps/chosen": -474.88538431677017,
|
|
"logps/rejected": -391.0483244889937,
|
|
"loss": 0.6156,
|
|
"loss/base_kto": 3.283986806869507,
|
|
"metrics/advantage_var": 623.0930786132812,
|
|
"regularization/lipschitz_norm": 1694.774658203125,
|
|
"regularization/w1": 1.640541911125183,
|
|
"rewards/chosen": 0.4490723224900524,
|
|
"rewards/margins": 0.7698123564854829,
|
|
"rewards/rejected": -0.32074003399543044,
|
|
"step": 1200
|
|
},
|
|
{
|
|
"epoch": 1.5803108808290154,
|
|
"grad_norm": 22.005277633666992,
|
|
"kl": 14.696810722351074,
|
|
"learning_rate": 4.922030418472422e-07,
|
|
"logits/chosen": -35796294.87947883,
|
|
"logits/rejected": -36376868.132132135,
|
|
"logps/chosen": -486.74857491856676,
|
|
"logps/rejected": -372.3071509009009,
|
|
"loss": 0.6043,
|
|
"loss/base_kto": 3.3149948120117188,
|
|
"metrics/advantage_var": 617.7034301757812,
|
|
"regularization/lipschitz_norm": 1569.4571533203125,
|
|
"regularization/w1": 1.519234538078308,
|
|
"rewards/chosen": 0.40370426737136095,
|
|
"rewards/margins": 0.7390145532431702,
|
|
"rewards/rejected": -0.33531028587180933,
|
|
"step": 1220
|
|
},
|
|
{
|
|
"epoch": 1.6062176165803108,
|
|
"grad_norm": 22.52359390258789,
|
|
"kl": 9.239676475524902,
|
|
"learning_rate": 4.780329366364336e-07,
|
|
"logits/chosen": -35149287.69372129,
|
|
"logits/rejected": -36877392.02551834,
|
|
"logps/chosen": -473.6223200612557,
|
|
"logps/rejected": -379.8645085725678,
|
|
"loss": 0.6108,
|
|
"loss/base_kto": 3.254795789718628,
|
|
"metrics/advantage_var": 659.5950927734375,
|
|
"regularization/lipschitz_norm": 1685.6546630859375,
|
|
"regularization/w1": 1.6317135095596313,
|
|
"rewards/chosen": 0.4534812097710088,
|
|
"rewards/margins": 0.8147187554841019,
|
|
"rewards/rejected": -0.3612375457130931,
|
|
"step": 1240
|
|
},
|
|
{
|
|
"epoch": 1.6321243523316062,
|
|
"grad_norm": 18.924129486083984,
|
|
"kl": 13.207402229309082,
|
|
"learning_rate": 4.638804903046684e-07,
|
|
"logits/chosen": -35524863.19242902,
|
|
"logits/rejected": -37563724.879256964,
|
|
"logps/chosen": -504.6776912460568,
|
|
"logps/rejected": -369.65992647058823,
|
|
"loss": 0.601,
|
|
"loss/base_kto": 3.313378095626831,
|
|
"metrics/advantage_var": 560.6163330078125,
|
|
"regularization/lipschitz_norm": 1544.231201171875,
|
|
"regularization/w1": 1.494815707206726,
|
|
"rewards/chosen": 0.4816870463759365,
|
|
"rewards/margins": 0.7439189735449332,
|
|
"rewards/rejected": -0.26223192716899674,
|
|
"step": 1260
|
|
},
|
|
{
|
|
"epoch": 1.6580310880829017,
|
|
"grad_norm": 23.38343048095703,
|
|
"kl": 10.662211418151855,
|
|
"learning_rate": 4.497570797180217e-07,
|
|
"logits/chosen": -36036365.39076923,
|
|
"logits/rejected": -35813623.06031746,
|
|
"logps/chosen": -472.31115384615384,
|
|
"logps/rejected": -390.9247271825397,
|
|
"loss": 0.6003,
|
|
"loss/base_kto": 3.3069992065429688,
|
|
"metrics/advantage_var": 582.510009765625,
|
|
"regularization/lipschitz_norm": 1544.6085205078125,
|
|
"regularization/w1": 1.4951810836791992,
|
|
"rewards/chosen": 0.37486323429987983,
|
|
"rewards/margins": 0.7670327628372062,
|
|
"rewards/rejected": -0.3921695285373264,
|
|
"step": 1280
|
|
},
|
|
{
|
|
"epoch": 1.6839378238341969,
|
|
"grad_norm": 16.797740936279297,
|
|
"kl": 11.943312644958496,
|
|
"learning_rate": 4.3567405840131853e-07,
|
|
"logits/chosen": -35921441.32544379,
|
|
"logits/rejected": -35954749.033112586,
|
|
"logps/chosen": -481.2622965976331,
|
|
"logps/rejected": -377.4881001655629,
|
|
"loss": 0.6092,
|
|
"loss/base_kto": 3.2473723888397217,
|
|
"metrics/advantage_var": 667.6775512695312,
|
|
"regularization/lipschitz_norm": 1679.6678466796875,
|
|
"regularization/w1": 1.6259183883666992,
|
|
"rewards/chosen": 0.5425694031122874,
|
|
"rewards/margins": 0.8230411796072041,
|
|
"rewards/rejected": -0.2804717764949167,
|
|
"step": 1300
|
|
},
|
|
{
|
|
"epoch": 1.709844559585492,
|
|
"grad_norm": 14.822357177734375,
|
|
"kl": 14.6751708984375,
|
|
"learning_rate": 4.2164274741126506e-07,
|
|
"logits/chosen": -36540168.76998369,
|
|
"logits/rejected": -36430263.07646177,
|
|
"logps/chosen": -481.2759482055465,
|
|
"logps/rejected": -383.56221889055473,
|
|
"loss": 0.5997,
|
|
"loss/base_kto": 3.2612533569335938,
|
|
"metrics/advantage_var": 564.8085327148438,
|
|
"regularization/lipschitz_norm": 1587.0164794921875,
|
|
"regularization/w1": 1.5362318754196167,
|
|
"rewards/chosen": 0.5311997181828609,
|
|
"rewards/margins": 0.7970503743977222,
|
|
"rewards/rejected": -0.2658506562148613,
|
|
"step": 1320
|
|
},
|
|
{
|
|
"epoch": 1.7357512953367875,
|
|
"grad_norm": 18.515363693237305,
|
|
"kl": 8.514366149902344,
|
|
"learning_rate": 4.0767442623567856e-07,
|
|
"logits/chosen": -34707980.962025315,
|
|
"logits/rejected": -36460354.37037037,
|
|
"logps/chosen": -485.23961629746833,
|
|
"logps/rejected": -362.2927035108025,
|
|
"loss": 0.6115,
|
|
"loss/base_kto": 3.3230254650115967,
|
|
"metrics/advantage_var": 613.64697265625,
|
|
"regularization/lipschitz_norm": 1621.0296630859375,
|
|
"regularization/w1": 1.5691566467285156,
|
|
"rewards/chosen": 0.2536700767806814,
|
|
"rewards/margins": 0.7126278430898779,
|
|
"rewards/rejected": -0.45895776630919655,
|
|
"step": 1340
|
|
},
|
|
{
|
|
"epoch": 1.761658031088083,
|
|
"grad_norm": 18.386857986450195,
|
|
"kl": 13.531366348266602,
|
|
"learning_rate": 3.937803237261357e-07,
|
|
"logits/chosen": -35775914.136645965,
|
|
"logits/rejected": -36766771.52201258,
|
|
"logps/chosen": -472.79347826086956,
|
|
"logps/rejected": -383.12224842767296,
|
|
"loss": 0.6198,
|
|
"loss/base_kto": 3.2859840393066406,
|
|
"metrics/advantage_var": 736.55517578125,
|
|
"regularization/lipschitz_norm": 1727.5960693359375,
|
|
"regularization/w1": 1.6723130941390991,
|
|
"rewards/chosen": 0.49217295202409256,
|
|
"rewards/margins": 0.8244916478601898,
|
|
"rewards/rejected": -0.3323186958360972,
|
|
"step": 1360
|
|
},
|
|
{
|
|
"epoch": 1.7875647668393784,
|
|
"grad_norm": 14.9951753616333,
|
|
"kl": 13.052634239196777,
|
|
"learning_rate": 3.7997160907132456e-07,
|
|
"logits/chosen": -35114431.90108192,
|
|
"logits/rejected": -36866433.01105845,
|
|
"logps/chosen": -500.2502897990726,
|
|
"logps/rejected": -387.1023894154818,
|
|
"loss": 0.6036,
|
|
"loss/base_kto": 3.2885186672210693,
|
|
"metrics/advantage_var": 593.1588134765625,
|
|
"regularization/lipschitz_norm": 1591.5728759765625,
|
|
"regularization/w1": 1.540642499923706,
|
|
"rewards/chosen": 0.4901510294659245,
|
|
"rewards/margins": 0.8024950909091388,
|
|
"rewards/rejected": -0.31234406144321436,
|
|
"step": 1380
|
|
},
|
|
{
|
|
"epoch": 1.8134715025906736,
|
|
"grad_norm": 14.314281463623047,
|
|
"kl": 17.484107971191406,
|
|
"learning_rate": 3.662593828183601e-07,
|
|
"logits/chosen": -34972305.46388443,
|
|
"logits/rejected": -35680801.509893455,
|
|
"logps/chosen": -491.9517455858748,
|
|
"logps/rejected": -353.6439783105023,
|
|
"loss": 0.5847,
|
|
"loss/base_kto": 3.2217767238616943,
|
|
"metrics/advantage_var": 534.62353515625,
|
|
"regularization/lipschitz_norm": 1504.163330078125,
|
|
"regularization/w1": 1.4560301303863525,
|
|
"rewards/chosen": 0.5592935969320576,
|
|
"rewards/margins": 0.807646298011776,
|
|
"rewards/rejected": -0.2483527010797184,
|
|
"step": 1400
|
|
},
|
|
{
|
|
"epoch": 1.8393782383419688,
|
|
"grad_norm": 24.737625122070312,
|
|
"kl": 11.298369407653809,
|
|
"learning_rate": 3.5265466794927725e-07,
|
|
"logits/chosen": -35272291.803486526,
|
|
"logits/rejected": -36781022.86594761,
|
|
"logps/chosen": -482.4072900158479,
|
|
"logps/rejected": -384.65092931432974,
|
|
"loss": 0.612,
|
|
"loss/base_kto": 3.3351597785949707,
|
|
"metrics/advantage_var": 562.4592895507812,
|
|
"regularization/lipschitz_norm": 1612.666748046875,
|
|
"regularization/w1": 1.5610612630844116,
|
|
"rewards/chosen": 0.4083454113943641,
|
|
"rewards/margins": 0.7207761237959555,
|
|
"rewards/rejected": -0.31243071240159137,
|
|
"step": 1420
|
|
},
|
|
{
|
|
"epoch": 1.8652849740932642,
|
|
"grad_norm": 13.026668548583984,
|
|
"kl": 8.759127616882324,
|
|
"learning_rate": 3.3916840101987887e-07,
|
|
"logits/chosen": -34090870.81339713,
|
|
"logits/rejected": -35542930.22970904,
|
|
"logps/chosen": -480.42618620414675,
|
|
"logps/rejected": -383.1708221669219,
|
|
"loss": 0.5866,
|
|
"loss/base_kto": 3.3254687786102295,
|
|
"metrics/advantage_var": 476.3669128417969,
|
|
"regularization/lipschitz_norm": 1412.2210693359375,
|
|
"regularization/w1": 1.3670300245285034,
|
|
"rewards/chosen": 0.3927106355365954,
|
|
"rewards/margins": 0.7149817284640696,
|
|
"rewards/rejected": -0.32227109292747413,
|
|
"step": 1440
|
|
},
|
|
{
|
|
"epoch": 1.8911917098445596,
|
|
"grad_norm": 19.02378273010254,
|
|
"kl": 8.384509086608887,
|
|
"learning_rate": 3.258114233680583e-07,
|
|
"logits/chosen": -35174218.81832544,
|
|
"logits/rejected": -35459534.145285934,
|
|
"logps/chosen": -456.5756319115324,
|
|
"logps/rejected": -403.34041731066463,
|
|
"loss": 0.5969,
|
|
"loss/base_kto": 3.3167827129364014,
|
|
"metrics/advantage_var": 710.2484741210938,
|
|
"regularization/lipschitz_norm": 1506.6376953125,
|
|
"regularization/w1": 1.4584254026412964,
|
|
"rewards/chosen": 0.31521430475075285,
|
|
"rewards/margins": 0.7730084942038778,
|
|
"rewards/rejected": -0.457794189453125,
|
|
"step": 1460
|
|
},
|
|
{
|
|
"epoch": 1.917098445595855,
|
|
"grad_norm": 29.536518096923828,
|
|
"kl": 7.4209303855896,
|
|
"learning_rate": 3.1259447239866796e-07,
|
|
"logits/chosen": -35098285.40609952,
|
|
"logits/rejected": -35436578.2891933,
|
|
"logps/chosen": -499.8775080256822,
|
|
"logps/rejected": -374.57667427701676,
|
|
"loss": 0.6097,
|
|
"loss/base_kto": 3.349459171295166,
|
|
"metrics/advantage_var": 566.67578125,
|
|
"regularization/lipschitz_norm": 1578.5618896484375,
|
|
"regularization/w1": 1.5280479192733765,
|
|
"rewards/chosen": 0.22507051862836075,
|
|
"rewards/margins": 0.6925968427103623,
|
|
"rewards/rejected": -0.4675263240820015,
|
|
"step": 1480
|
|
},
|
|
{
|
|
"epoch": 1.9430051813471503,
|
|
"grad_norm": 14.033746719360352,
|
|
"kl": 7.5047197341918945,
|
|
"learning_rate": 2.9952817295193435e-07,
|
|
"logits/chosen": -35247297.9029734,
|
|
"logits/rejected": -36115352.16224649,
|
|
"logps/chosen": -480.89666471048514,
|
|
"logps/rejected": -391.7812012480499,
|
|
"loss": 0.6025,
|
|
"loss/base_kto": 3.294116973876953,
|
|
"metrics/advantage_var": 564.1644897460938,
|
|
"regularization/lipschitz_norm": 1576.3626708984375,
|
|
"regularization/w1": 1.525918960571289,
|
|
"rewards/chosen": 0.35076068525956083,
|
|
"rewards/margins": 0.7848183405345218,
|
|
"rewards/rejected": -0.434057655274961,
|
|
"step": 1500
|
|
},
|
|
{
|
|
"epoch": 1.9689119170984455,
|
|
"grad_norm": 14.298561096191406,
|
|
"kl": 13.422953605651855,
|
|
"learning_rate": 2.866230287623651e-07,
|
|
"logits/chosen": -35913955.55555555,
|
|
"logits/rejected": -37833948.35443038,
|
|
"logps/chosen": -491.8377700617284,
|
|
"logps/rejected": -366.3362836234177,
|
|
"loss": 0.6036,
|
|
"loss/base_kto": 3.31005859375,
|
|
"metrics/advantage_var": 584.8604736328125,
|
|
"regularization/lipschitz_norm": 1569.2012939453125,
|
|
"regularization/w1": 1.5189868211746216,
|
|
"rewards/chosen": 0.4509073422278887,
|
|
"rewards/margins": 0.7458736656941294,
|
|
"rewards/rejected": -0.29496632346624063,
|
|
"step": 1520
|
|
},
|
|
{
|
|
"epoch": 1.994818652849741,
|
|
"grad_norm": 23.15932273864746,
|
|
"kl": 14.832297325134277,
|
|
"learning_rate": 2.738894140150075e-07,
|
|
"logits/chosen": -34271518.14128728,
|
|
"logits/rejected": -35675739.57076205,
|
|
"logps/chosen": -498.0529336734694,
|
|
"logps/rejected": -380.7037811041991,
|
|
"loss": 0.601,
|
|
"loss/base_kto": 3.275827169418335,
|
|
"metrics/advantage_var": 644.1874389648438,
|
|
"regularization/lipschitz_norm": 1582.501220703125,
|
|
"regularization/w1": 1.5318611860275269,
|
|
"rewards/chosen": 0.5628489160462863,
|
|
"rewards/margins": 0.7810239705697603,
|
|
"rewards/rejected": -0.21817505452347394,
|
|
"step": 1540
|
|
},
|
|
{
|
|
"epoch": 2.0207253886010363,
|
|
"grad_norm": 18.470226287841797,
|
|
"kl": 10.67943286895752,
|
|
"learning_rate": 2.6133756500585156e-07,
|
|
"logits/chosen": -34660532.522796355,
|
|
"logits/rejected": -36501913.6,
|
|
"logps/chosen": -489.24572568389056,
|
|
"logps/rejected": -350.358341733871,
|
|
"loss": 0.5703,
|
|
"loss/base_kto": 3.2103021144866943,
|
|
"metrics/advantage_var": 453.81402587890625,
|
|
"regularization/lipschitz_norm": 1396.7904052734375,
|
|
"regularization/w1": 1.3520931005477905,
|
|
"rewards/chosen": 0.4466327330986417,
|
|
"rewards/margins": 0.8437312497474572,
|
|
"rewards/rejected": -0.3970985166488155,
|
|
"step": 1560
|
|
},
|
|
{
|
|
"epoch": 2.0466321243523318,
|
|
"grad_norm": 16.989198684692383,
|
|
"kl": 11.994152069091797,
|
|
"learning_rate": 2.489775719130767e-07,
|
|
"logits/chosen": -33972006.54199683,
|
|
"logits/rejected": -35205495.5192604,
|
|
"logps/chosen": -465.40421949286844,
|
|
"logps/rejected": -392.57966583204933,
|
|
"loss": 0.5575,
|
|
"loss/base_kto": 3.219650983810425,
|
|
"metrics/advantage_var": 403.4252624511719,
|
|
"regularization/lipschitz_norm": 1281.2525634765625,
|
|
"regularization/w1": 1.2402524948120117,
|
|
"rewards/chosen": 0.46899085188440964,
|
|
"rewards/margins": 0.8060005110505546,
|
|
"rewards/rejected": -0.337009659166145,
|
|
"step": 1580
|
|
},
|
|
{
|
|
"epoch": 2.0725388601036268,
|
|
"grad_norm": 14.959163665771484,
|
|
"kl": 12.547826766967773,
|
|
"learning_rate": 2.3681937068576303e-07,
|
|
"logits/chosen": -35060381.596465394,
|
|
"logits/rejected": -36498508.67221298,
|
|
"logps/chosen": -488.4973766568483,
|
|
"logps/rejected": -364.11168885191347,
|
|
"loss": 0.5623,
|
|
"loss/base_kto": 3.1705167293548584,
|
|
"metrics/advantage_var": 467.0064392089844,
|
|
"regularization/lipschitz_norm": 1371.7470703125,
|
|
"regularization/w1": 1.3278511762619019,
|
|
"rewards/chosen": 0.5463290994051454,
|
|
"rewards/margins": 0.8845043350963111,
|
|
"rewards/rejected": -0.33817523569116575,
|
|
"step": 1600
|
|
},
|
|
{
|
|
"epoch": 2.098445595854922,
|
|
"grad_norm": 20.10413932800293,
|
|
"kl": 15.856521606445312,
|
|
"learning_rate": 2.2487273505658e-07,
|
|
"logits/chosen": -36529925.286135696,
|
|
"logits/rejected": -36900765.342192695,
|
|
"logps/chosen": -496.71916482300884,
|
|
"logps/rejected": -396.3875103820598,
|
|
"loss": 0.5623,
|
|
"loss/base_kto": 3.1664652824401855,
|
|
"metrics/advantage_var": 468.0096740722656,
|
|
"regularization/lipschitz_norm": 1375.593994140625,
|
|
"regularization/w1": 1.3315749168395996,
|
|
"rewards/chosen": 0.5738953446919939,
|
|
"rewards/margins": 0.8824426339605129,
|
|
"rewards/rejected": -0.308547289268519,
|
|
"step": 1620
|
|
},
|
|
{
|
|
"epoch": 2.1243523316062176,
|
|
"grad_norm": 17.898967742919922,
|
|
"kl": 16.204326629638672,
|
|
"learning_rate": 2.131472686848817e-07,
|
|
"logits/chosen": -35384816.484848484,
|
|
"logits/rejected": -36924283.87096774,
|
|
"logps/chosen": -484.1930871212121,
|
|
"logps/rejected": -385.1054183467742,
|
|
"loss": 0.5616,
|
|
"loss/base_kto": 3.196605682373047,
|
|
"metrics/advantage_var": 449.0083923339844,
|
|
"regularization/lipschitz_norm": 1338.77099609375,
|
|
"regularization/w1": 1.29593026638031,
|
|
"rewards/chosen": 0.6323146473277699,
|
|
"rewards/margins": 0.8422431968174372,
|
|
"rewards/rejected": -0.20992854948966733,
|
|
"step": 1640
|
|
},
|
|
{
|
|
"epoch": 2.150259067357513,
|
|
"grad_norm": 17.83039665222168,
|
|
"kl": 20.723678588867188,
|
|
"learning_rate": 2.016523974365188e-07,
|
|
"logits/chosen": -35518300.88495575,
|
|
"logits/rejected": -36610558.299003325,
|
|
"logps/chosen": -494.404959439528,
|
|
"logps/rejected": -388.093127076412,
|
|
"loss": 0.5603,
|
|
"loss/base_kto": 3.1352014541625977,
|
|
"metrics/advantage_var": 509.0668640136719,
|
|
"regularization/lipschitz_norm": 1391.6597900390625,
|
|
"regularization/w1": 1.3471266031265259,
|
|
"rewards/chosen": 0.7286337343289085,
|
|
"rewards/margins": 0.9123075296044225,
|
|
"rewards/rejected": -0.18367379527551392,
|
|
"step": 1660
|
|
},
|
|
{
|
|
"epoch": 2.1761658031088085,
|
|
"grad_norm": 18.897979736328125,
|
|
"kl": 22.381803512573242,
|
|
"learning_rate": 1.9039736180657372e-07,
|
|
"logits/chosen": -35291014.36918138,
|
|
"logits/rejected": -36207231.02587519,
|
|
"logps/chosen": -474.2518057784912,
|
|
"logps/rejected": -376.01084474885846,
|
|
"loss": 0.5608,
|
|
"loss/base_kto": 3.1498916149139404,
|
|
"metrics/advantage_var": 488.4566345214844,
|
|
"regularization/lipschitz_norm": 1380.9681396484375,
|
|
"regularization/w1": 1.3367770910263062,
|
|
"rewards/chosen": 0.7305796027757574,
|
|
"rewards/margins": 0.9073193462683136,
|
|
"rewards/rejected": -0.17673974349255614,
|
|
"step": 1680
|
|
},
|
|
{
|
|
"epoch": 2.2020725388601035,
|
|
"grad_norm": 16.471464157104492,
|
|
"kl": 13.330780029296875,
|
|
"learning_rate": 1.7939120949111498e-07,
|
|
"logits/chosen": -34937962.69984448,
|
|
"logits/rejected": -35513027.3155416,
|
|
"logps/chosen": -488.04417768273714,
|
|
"logps/rejected": -349.43153944270017,
|
|
"loss": 0.5527,
|
|
"loss/base_kto": 3.155369520187378,
|
|
"metrics/advantage_var": 420.8874206542969,
|
|
"regularization/lipschitz_norm": 1307.99560546875,
|
|
"regularization/w1": 1.2661398649215698,
|
|
"rewards/chosen": 0.5613595538339571,
|
|
"rewards/margins": 0.8743365047743392,
|
|
"rewards/rejected": -0.31297695094038214,
|
|
"step": 1700
|
|
},
|
|
{
|
|
"epoch": 2.227979274611399,
|
|
"grad_norm": 14.699928283691406,
|
|
"kl": 16.41225242614746,
|
|
"learning_rate": 1.6864278811393523e-07,
|
|
"logits/chosen": -34938240.81528662,
|
|
"logits/rejected": -35802017.76687117,
|
|
"logps/chosen": -442.6795382165605,
|
|
"logps/rejected": -391.77993194018404,
|
|
"loss": 0.5536,
|
|
"loss/base_kto": 3.1832523345947266,
|
|
"metrics/advantage_var": 439.9745178222656,
|
|
"regularization/lipschitz_norm": 1287.1234130859375,
|
|
"regularization/w1": 1.2459354400634766,
|
|
"rewards/chosen": 0.5204137814272741,
|
|
"rewards/margins": 0.8519921355899467,
|
|
"rewards/rejected": -0.33157835416267256,
|
|
"step": 1720
|
|
},
|
|
{
|
|
"epoch": 2.2538860103626943,
|
|
"grad_norm": 15.27790641784668,
|
|
"kl": 12.06608772277832,
|
|
"learning_rate": 1.5816073811412584e-07,
|
|
"logits/chosen": -35155774.1829653,
|
|
"logits/rejected": -35900771.071207434,
|
|
"logps/chosen": -468.29574132492115,
|
|
"logps/rejected": -375.16036184210526,
|
|
"loss": 0.5639,
|
|
"loss/base_kto": 3.1814515590667725,
|
|
"metrics/advantage_var": 439.0970764160156,
|
|
"regularization/lipschitz_norm": 1373.935546875,
|
|
"regularization/w1": 1.3299696445465088,
|
|
"rewards/chosen": 0.5102838462083498,
|
|
"rewards/margins": 0.8738979078769111,
|
|
"rewards/rejected": -0.36361406166856136,
|
|
"step": 1740
|
|
},
|
|
{
|
|
"epoch": 2.2797927461139897,
|
|
"grad_norm": 19.957759857177734,
|
|
"kl": 13.255197525024414,
|
|
"learning_rate": 1.4795348580020207e-07,
|
|
"logits/chosen": -35628739.90996785,
|
|
"logits/rejected": -37262921.14285714,
|
|
"logps/chosen": -494.9699557877814,
|
|
"logps/rejected": -357.8082731762918,
|
|
"loss": 0.5527,
|
|
"loss/base_kto": 3.2057883739471436,
|
|
"metrics/advantage_var": 396.0398254394531,
|
|
"regularization/lipschitz_norm": 1256.2628173828125,
|
|
"regularization/w1": 1.2160624265670776,
|
|
"rewards/chosen": 0.46789781380313,
|
|
"rewards/margins": 0.8238110166164612,
|
|
"rewards/rejected": -0.3559132028133311,
|
|
"step": 1760
|
|
},
|
|
{
|
|
"epoch": 2.305699481865285,
|
|
"grad_norm": 10.981877326965332,
|
|
"kl": 13.450248718261719,
|
|
"learning_rate": 1.3802923657636355e-07,
|
|
"logits/chosen": -36129261.9109063,
|
|
"logits/rejected": -37033179.777424484,
|
|
"logps/chosen": -459.95065284178185,
|
|
"logps/rejected": -370.7652027027027,
|
|
"loss": 0.5522,
|
|
"loss/base_kto": 3.215421438217163,
|
|
"metrics/advantage_var": 434.76611328125,
|
|
"regularization/lipschitz_norm": 1241.7470703125,
|
|
"regularization/w1": 1.2020111083984375,
|
|
"rewards/chosen": 0.4774749802737375,
|
|
"rewards/margins": 0.8330701671713429,
|
|
"rewards/rejected": -0.35559518689760533,
|
|
"step": 1780
|
|
},
|
|
{
|
|
"epoch": 2.33160621761658,
|
|
"grad_norm": 19.0456600189209,
|
|
"kl": 12.843103408813477,
|
|
"learning_rate": 1.28395968346336e-07,
|
|
"logits/chosen": -34794248.827586204,
|
|
"logits/rejected": -35015555.588785045,
|
|
"logps/chosen": -481.31044278996865,
|
|
"logps/rejected": -356.7027112538941,
|
|
"loss": 0.5563,
|
|
"loss/base_kto": 3.2093822956085205,
|
|
"metrics/advantage_var": 446.06109619140625,
|
|
"regularization/lipschitz_norm": 1282.4368896484375,
|
|
"regularization/w1": 1.2413989305496216,
|
|
"rewards/chosen": 0.5343358628802165,
|
|
"rewards/margins": 0.8298741264091057,
|
|
"rewards/rejected": -0.29553826352888923,
|
|
"step": 1800
|
|
},
|
|
{
|
|
"epoch": 2.3575129533678756,
|
|
"grad_norm": 20.85038948059082,
|
|
"kl": 11.654293060302734,
|
|
"learning_rate": 1.1906142510009415e-07,
|
|
"logits/chosen": -34359750.409861326,
|
|
"logits/rejected": -37603863.530903324,
|
|
"logps/chosen": -513.1727657935285,
|
|
"logps/rejected": -380.268868858954,
|
|
"loss": 0.5565,
|
|
"loss/base_kto": 3.168382406234741,
|
|
"metrics/advantage_var": 427.8387756347656,
|
|
"regularization/lipschitz_norm": 1326.203857421875,
|
|
"regularization/w1": 1.2837653160095215,
|
|
"rewards/chosen": 0.523593708620233,
|
|
"rewards/margins": 0.8711015300191782,
|
|
"rewards/rejected": -0.34750782139894515,
|
|
"step": 1820
|
|
},
|
|
{
|
|
"epoch": 2.383419689119171,
|
|
"grad_norm": 16.17123031616211,
|
|
"kl": 15.288296699523926,
|
|
"learning_rate": 1.1003311068862547e-07,
|
|
"logits/chosen": -35400721.41496599,
|
|
"logits/rejected": -36835588.439306356,
|
|
"logps/chosen": -487.6054421768707,
|
|
"logps/rejected": -391.5633128612717,
|
|
"loss": 0.5468,
|
|
"loss/base_kto": 3.1300837993621826,
|
|
"metrics/advantage_var": 433.1132507324219,
|
|
"regularization/lipschitz_norm": 1285.69287109375,
|
|
"regularization/w1": 1.2445507049560547,
|
|
"rewards/chosen": 0.5404657895873193,
|
|
"rewards/margins": 0.8912913253560377,
|
|
"rewards/rejected": -0.3508255357687184,
|
|
"step": 1840
|
|
},
|
|
{
|
|
"epoch": 2.4093264248704664,
|
|
"grad_norm": 15.69742202758789,
|
|
"kl": 12.333353996276855,
|
|
"learning_rate": 1.0131828279173588e-07,
|
|
"logits/chosen": -35549174.1696,
|
|
"logits/rejected": -36529338.03969466,
|
|
"logps/chosen": -500.6871,
|
|
"logps/rejected": -380.6728530534351,
|
|
"loss": 0.5564,
|
|
"loss/base_kto": 3.196179151535034,
|
|
"metrics/advantage_var": 434.9308776855469,
|
|
"regularization/lipschitz_norm": 1296.4019775390625,
|
|
"regularization/w1": 1.254917025566101,
|
|
"rewards/chosen": 0.44173544921875,
|
|
"rewards/margins": 0.8416032219107825,
|
|
"rewards/rejected": -0.39986777269203244,
|
|
"step": 1860
|
|
},
|
|
{
|
|
"epoch": 2.4352331606217614,
|
|
"grad_norm": 15.341116905212402,
|
|
"kl": 10.772221565246582,
|
|
"learning_rate": 9.292394708374596e-08,
|
|
"logits/chosen": -35202072.97560976,
|
|
"logits/rejected": -36885296.120300755,
|
|
"logps/chosen": -493.02459349593494,
|
|
"logps/rejected": -349.1362546992481,
|
|
"loss": 0.5593,
|
|
"loss/base_kto": 3.1654787063598633,
|
|
"metrics/advantage_var": 463.2176208496094,
|
|
"regularization/lipschitz_norm": 1351.9886474609375,
|
|
"regularization/w1": 1.3087249994277954,
|
|
"rewards/chosen": 0.4563167913173272,
|
|
"rewards/margins": 0.8843819034384504,
|
|
"rewards/rejected": -0.42806511212112314,
|
|
"step": 1880
|
|
},
|
|
{
|
|
"epoch": 2.461139896373057,
|
|
"grad_norm": 19.67717742919922,
|
|
"kl": 8.95859146118164,
|
|
"learning_rate": 8.48568516017727e-08,
|
|
"logits/chosen": -35181711.58089368,
|
|
"logits/rejected": -37352813.13470682,
|
|
"logps/chosen": -476.40523882896764,
|
|
"logps/rejected": -364.6350287242472,
|
|
"loss": 0.5583,
|
|
"loss/base_kto": 3.2194983959198,
|
|
"metrics/advantage_var": 410.23345947265625,
|
|
"regularization/lipschitz_norm": 1288.4498291015625,
|
|
"regularization/w1": 1.2472193241119385,
|
|
"rewards/chosen": 0.40437935239912365,
|
|
"rewards/margins": 0.8254889463905559,
|
|
"rewards/rejected": -0.4211095939914323,
|
|
"step": 1900
|
|
},
|
|
{
|
|
"epoch": 2.4870466321243523,
|
|
"grad_norm": 16.2672176361084,
|
|
"kl": 11.774325370788574,
|
|
"learning_rate": 7.71234813211169e-08,
|
|
"logits/chosen": -34668303.765395895,
|
|
"logits/rejected": -35742206.28762542,
|
|
"logps/chosen": -478.99780058651027,
|
|
"logps/rejected": -355.7196906354515,
|
|
"loss": 0.5535,
|
|
"loss/base_kto": 3.1635589599609375,
|
|
"metrics/advantage_var": 428.73614501953125,
|
|
"regularization/lipschitz_norm": 1306.254150390625,
|
|
"regularization/w1": 1.2644538879394531,
|
|
"rewards/chosen": 0.5405383068095904,
|
|
"rewards/margins": 0.8783245188249018,
|
|
"rewards/rejected": -0.33778621201531145,
|
|
"step": 1920
|
|
},
|
|
{
|
|
"epoch": 2.5129533678756477,
|
|
"grad_norm": 17.85692596435547,
|
|
"kl": 15.85357666015625,
|
|
"learning_rate": 6.973005294212353e-08,
|
|
"logits/chosen": -34255347.6626506,
|
|
"logits/rejected": -35674630.64935065,
|
|
"logps/chosen": -487.51430722891564,
|
|
"logps/rejected": -378.6520140016234,
|
|
"loss": 0.5591,
|
|
"loss/base_kto": 3.168553590774536,
|
|
"metrics/advantage_var": 461.27398681640625,
|
|
"regularization/lipschitz_norm": 1346.986328125,
|
|
"regularization/w1": 1.3038827180862427,
|
|
"rewards/chosen": 0.6127265102892037,
|
|
"rewards/margins": 0.8764291753457047,
|
|
"rewards/rejected": -0.26370266505650114,
|
|
"step": 1940
|
|
},
|
|
{
|
|
"epoch": 2.538860103626943,
|
|
"grad_norm": 17.892295837402344,
|
|
"kl": 14.68474292755127,
|
|
"learning_rate": 6.268250989270102e-08,
|
|
"logits/chosen": -36290994.526315786,
|
|
"logits/rejected": -36778870.85714286,
|
|
"logps/chosen": -473.0989412006579,
|
|
"logps/rejected": -372.2460239955357,
|
|
"loss": 0.5504,
|
|
"loss/base_kto": 3.1353840827941895,
|
|
"metrics/advantage_var": 479.9383850097656,
|
|
"regularization/lipschitz_norm": 1309.4774169921875,
|
|
"regularization/w1": 1.2675741910934448,
|
|
"rewards/chosen": 0.5927888970626028,
|
|
"rewards/margins": 0.9044809986774187,
|
|
"rewards/rejected": -0.31169210161481586,
|
|
"step": 1960
|
|
},
|
|
{
|
|
"epoch": 2.5647668393782386,
|
|
"grad_norm": 17.130151748657227,
|
|
"kl": 13.543744087219238,
|
|
"learning_rate": 5.598651755051975e-08,
|
|
"logits/chosen": -35352974.83771252,
|
|
"logits/rejected": -36581376.0,
|
|
"logps/chosen": -472.7634756568779,
|
|
"logps/rejected": -390.00819510268565,
|
|
"loss": 0.559,
|
|
"loss/base_kto": 3.152021884918213,
|
|
"metrics/advantage_var": 429.50201416015625,
|
|
"regularization/lipschitz_norm": 1363.3504638671875,
|
|
"regularization/w1": 1.319723129272461,
|
|
"rewards/chosen": 0.5528380498635529,
|
|
"rewards/margins": 0.8701875244405302,
|
|
"rewards/rejected": -0.3173494745769772,
|
|
"step": 1980
|
|
},
|
|
{
|
|
"epoch": 2.5906735751295336,
|
|
"grad_norm": 20.55853843688965,
|
|
"kl": 14.852238655090332,
|
|
"learning_rate": 4.964745868872933e-08,
|
|
"logits/chosen": -35541704.62695925,
|
|
"logits/rejected": -36462493.10903427,
|
|
"logps/chosen": -484.4791340125392,
|
|
"logps/rejected": -367.09642718068534,
|
|
"loss": 0.564,
|
|
"loss/base_kto": 3.1748039722442627,
|
|
"metrics/advantage_var": 455.13946533203125,
|
|
"regularization/lipschitz_norm": 1381.0677490234375,
|
|
"regularization/w1": 1.3368734121322632,
|
|
"rewards/chosen": 0.5463944677275177,
|
|
"rewards/margins": 0.8718686215254635,
|
|
"rewards/rejected": -0.32547415379794586,
|
|
"step": 2000
|
|
},
|
|
{
|
|
"epoch": 2.616580310880829,
|
|
"grad_norm": 18.252906799316406,
|
|
"kl": 13.813323974609375,
|
|
"learning_rate": 4.3670429148855493e-08,
|
|
"logits/chosen": -35506461.164556965,
|
|
"logits/rejected": -36864281.28395062,
|
|
"logps/chosen": -489.61367681962025,
|
|
"logps/rejected": -373.4073591820988,
|
|
"loss": 0.571,
|
|
"loss/base_kto": 3.1924209594726562,
|
|
"metrics/advantage_var": 476.4051818847656,
|
|
"regularization/lipschitz_norm": 1420.7030029296875,
|
|
"regularization/w1": 1.375240445137024,
|
|
"rewards/chosen": 0.5333185316641119,
|
|
"rewards/margins": 0.853571456453729,
|
|
"rewards/rejected": -0.3202529247896171,
|
|
"step": 2020
|
|
},
|
|
{
|
|
"epoch": 2.6424870466321244,
|
|
"grad_norm": 15.302053451538086,
|
|
"kl": 13.113249778747559,
|
|
"learning_rate": 3.806023374435663e-08,
|
|
"logits/chosen": -36436406.176744185,
|
|
"logits/rejected": -36537416.56692913,
|
|
"logps/chosen": -489.64287790697676,
|
|
"logps/rejected": -368.50002460629923,
|
|
"loss": 0.5556,
|
|
"loss/base_kto": 3.1588809490203857,
|
|
"metrics/advantage_var": 470.2917175292969,
|
|
"regularization/lipschitz_norm": 1328.6754150390625,
|
|
"regularization/w1": 1.2861578464508057,
|
|
"rewards/chosen": 0.5469550554142442,
|
|
"rewards/margins": 0.8917398656381614,
|
|
"rewards/rejected": -0.3447848102239173,
|
|
"step": 2040
|
|
},
|
|
{
|
|
"epoch": 2.66839378238342,
|
|
"grad_norm": 16.81578254699707,
|
|
"kl": 14.144734382629395,
|
|
"learning_rate": 3.282138239813037e-08,
|
|
"logits/chosen": -33565854.9248,
|
|
"logits/rejected": -35864763.603053436,
|
|
"logps/chosen": -451.1166,
|
|
"logps/rejected": -388.69527671755725,
|
|
"loss": 0.5461,
|
|
"loss/base_kto": 3.16017484664917,
|
|
"metrics/advantage_var": 395.6061096191406,
|
|
"regularization/lipschitz_norm": 1248.55224609375,
|
|
"regularization/w1": 1.2085986137390137,
|
|
"rewards/chosen": 0.4966119140625,
|
|
"rewards/margins": 0.8574853038525763,
|
|
"rewards/rejected": -0.3608733897900763,
|
|
"step": 2060
|
|
},
|
|
{
|
|
"epoch": 2.694300518134715,
|
|
"grad_norm": 19.078401565551758,
|
|
"kl": 13.644217491149902,
|
|
"learning_rate": 2.795808651707793e-08,
|
|
"logits/chosen": -35438682.73417722,
|
|
"logits/rejected": -36835305.87654321,
|
|
"logps/chosen": -458.0954806170886,
|
|
"logps/rejected": -364.69150270061726,
|
|
"loss": 0.5505,
|
|
"loss/base_kto": 3.154127597808838,
|
|
"metrics/advantage_var": 464.5173034667969,
|
|
"regularization/lipschitz_norm": 1291.393798828125,
|
|
"regularization/w1": 1.25006902217865,
|
|
"rewards/chosen": 0.5648512055602255,
|
|
"rewards/margins": 0.8932003778933659,
|
|
"rewards/rejected": -0.3283491723331404,
|
|
"step": 2080
|
|
},
|
|
{
|
|
"epoch": 2.7202072538860103,
|
|
"grad_norm": 17.28318977355957,
|
|
"kl": 12.8956880569458,
|
|
"learning_rate": 2.3474255606639293e-08,
|
|
"logits/chosen": -35547772.54054054,
|
|
"logits/rejected": -37020988.872964166,
|
|
"logps/chosen": -502.237987987988,
|
|
"logps/rejected": -373.9553898615635,
|
|
"loss": 0.5497,
|
|
"loss/base_kto": 3.1797616481781006,
|
|
"metrics/advantage_var": 415.0672912597656,
|
|
"regularization/lipschitz_norm": 1258.2498779296875,
|
|
"regularization/w1": 1.217985987663269,
|
|
"rewards/chosen": 0.5235647023977102,
|
|
"rewards/margins": 0.860215737172242,
|
|
"rewards/rejected": -0.33665103477453173,
|
|
"step": 2100
|
|
},
|
|
{
|
|
"epoch": 2.7461139896373057,
|
|
"grad_norm": 17.75486946105957,
|
|
"kl": 13.473390579223633,
|
|
"learning_rate": 1.9373494128020195e-08,
|
|
"logits/chosen": -34530838.26086956,
|
|
"logits/rejected": -35509151.39622641,
|
|
"logps/chosen": -475.0430900621118,
|
|
"logps/rejected": -383.69025157232704,
|
|
"loss": 0.5525,
|
|
"loss/base_kto": 3.2205536365509033,
|
|
"metrics/advantage_var": 390.6640625,
|
|
"regularization/lipschitz_norm": 1239.4990234375,
|
|
"regularization/w1": 1.1998350620269775,
|
|
"rewards/chosen": 0.5065290557671778,
|
|
"rewards/margins": 0.7972339148949662,
|
|
"rewards/rejected": -0.29070485912778843,
|
|
"step": 2120
|
|
},
|
|
{
|
|
"epoch": 2.772020725388601,
|
|
"grad_norm": 19.26053237915039,
|
|
"kl": 14.291238784790039,
|
|
"learning_rate": 1.5659098600638798e-08,
|
|
"logits/chosen": -35028423.470031545,
|
|
"logits/rejected": -34800960.19814242,
|
|
"logps/chosen": -477.7087933753943,
|
|
"logps/rejected": -374.2738003095975,
|
|
"loss": 0.5604,
|
|
"loss/base_kto": 3.202681303024292,
|
|
"metrics/advantage_var": 425.21026611328125,
|
|
"regularization/lipschitz_norm": 1323.0657958984375,
|
|
"regularization/w1": 1.280727744102478,
|
|
"rewards/chosen": 0.5387579018385252,
|
|
"rewards/margins": 0.834407238426059,
|
|
"rewards/rejected": -0.29564933658753384,
|
|
"step": 2140
|
|
},
|
|
{
|
|
"epoch": 2.7979274611398965,
|
|
"grad_norm": 20.083316802978516,
|
|
"kl": 12.89837646484375,
|
|
"learning_rate": 1.2334054952120143e-08,
|
|
"logits/chosen": -35416430.17721519,
|
|
"logits/rejected": -37350216.69135802,
|
|
"logps/chosen": -478.3454806170886,
|
|
"logps/rejected": -386.7404513888889,
|
|
"loss": 0.558,
|
|
"loss/base_kto": 3.1874847412109375,
|
|
"metrics/advantage_var": 456.527587890625,
|
|
"regularization/lipschitz_norm": 1318.8660888671875,
|
|
"regularization/w1": 1.2766623497009277,
|
|
"rewards/chosen": 0.5069576215140427,
|
|
"rewards/margins": 0.8588423456954181,
|
|
"rewards/rejected": -0.3518847241813754,
|
|
"step": 2160
|
|
},
|
|
{
|
|
"epoch": 2.823834196891192,
|
|
"grad_norm": 19.203149795532227,
|
|
"kl": 14.212717056274414,
|
|
"learning_rate": 9.401036117969108e-09,
|
|
"logits/chosen": -35609565.703349285,
|
|
"logits/rejected": -37047460.65543645,
|
|
"logps/chosen": -476.80746610845296,
|
|
"logps/rejected": -378.27455015313933,
|
|
"loss": 0.5658,
|
|
"loss/base_kto": 3.2434043884277344,
|
|
"metrics/advantage_var": 446.1128845214844,
|
|
"regularization/lipschitz_norm": 1325.3538818359375,
|
|
"regularization/w1": 1.2829424142837524,
|
|
"rewards/chosen": 0.47803357381379585,
|
|
"rewards/margins": 0.7959637866087088,
|
|
"rewards/rejected": -0.3179302127949129,
|
|
"step": 2180
|
|
},
|
|
{
|
|
"epoch": 2.849740932642487,
|
|
"grad_norm": 20.670875549316406,
|
|
"kl": 11.36286449432373,
|
|
"learning_rate": 6.8623998928517555e-09,
|
|
"logits/chosen": -36989975.59447005,
|
|
"logits/rejected": -36957013.06200318,
|
|
"logps/chosen": -482.69378840245776,
|
|
"logps/rejected": -391.89586645469,
|
|
"loss": 0.5605,
|
|
"loss/base_kto": 3.207538604736328,
|
|
"metrics/advantage_var": 426.26690673828125,
|
|
"regularization/lipschitz_norm": 1318.4716796875,
|
|
"regularization/w1": 1.2762806415557861,
|
|
"rewards/chosen": 0.48256902504260274,
|
|
"rewards/margins": 0.8347832136107665,
|
|
"rewards/rejected": -0.3522141885681638,
|
|
"step": 2200
|
|
},
|
|
{
|
|
"epoch": 2.8756476683937824,
|
|
"grad_norm": 20.096817016601562,
|
|
"kl": 12.562037467956543,
|
|
"learning_rate": 4.72018703521132e-09,
|
|
"logits/chosen": -36984000.972644374,
|
|
"logits/rejected": -36085634.881028935,
|
|
"logps/chosen": -502.9935885258359,
|
|
"logps/rejected": -382.9482013665595,
|
|
"loss": 0.5626,
|
|
"loss/base_kto": 3.1672394275665283,
|
|
"metrics/advantage_var": 472.66845703125,
|
|
"regularization/lipschitz_norm": 1377.91748046875,
|
|
"regularization/w1": 1.3338241577148438,
|
|
"rewards/chosen": 0.5462447053393332,
|
|
"rewards/margins": 0.8777947147202994,
|
|
"rewards/rejected": -0.3315500093809661,
|
|
"step": 2220
|
|
},
|
|
{
|
|
"epoch": 2.901554404145078,
|
|
"grad_norm": 14.879459381103516,
|
|
"kl": 11.950034141540527,
|
|
"learning_rate": 2.976119626744267e-09,
|
|
"logits/chosen": -35631020.14488189,
|
|
"logits/rejected": -35859317.87906977,
|
|
"logps/chosen": -462.8613188976378,
|
|
"logps/rejected": -388.47257751937985,
|
|
"loss": 0.5478,
|
|
"loss/base_kto": 3.1338982582092285,
|
|
"metrics/advantage_var": 419.29840087890625,
|
|
"regularization/lipschitz_norm": 1289.6092529296875,
|
|
"regularization/w1": 1.2483415603637695,
|
|
"rewards/chosen": 0.5252166507750984,
|
|
"rewards/margins": 0.872691782497482,
|
|
"rewards/rejected": -0.3474751317223837,
|
|
"step": 2240
|
|
},
|
|
{
|
|
"epoch": 2.927461139896373,
|
|
"grad_norm": 15.49474048614502,
|
|
"kl": 13.836501121520996,
|
|
"learning_rate": 1.631599688052765e-09,
|
|
"logits/chosen": -36151189.437398374,
|
|
"logits/rejected": -37945337.840601504,
|
|
"logps/chosen": -489.70701219512193,
|
|
"logps/rejected": -376.35352443609025,
|
|
"loss": 0.5572,
|
|
"loss/base_kto": 3.1607234477996826,
|
|
"metrics/advantage_var": 425.92974853515625,
|
|
"regularization/lipschitz_norm": 1339.749755859375,
|
|
"regularization/w1": 1.2968777418136597,
|
|
"rewards/chosen": 0.5414697166380843,
|
|
"rewards/margins": 0.8892156837286623,
|
|
"rewards/rejected": -0.34774596709057803,
|
|
"step": 2260
|
|
},
|
|
{
|
|
"epoch": 2.9533678756476682,
|
|
"grad_norm": 17.457658767700195,
|
|
"kl": 14.280056953430176,
|
|
"learning_rate": 6.877080515894085e-10,
|
|
"logits/chosen": -34176784.37441497,
|
|
"logits/rejected": -36227716.20657277,
|
|
"logps/chosen": -481.05669851794073,
|
|
"logps/rejected": -364.6233372456964,
|
|
"loss": 0.5574,
|
|
"loss/base_kto": 3.2173683643341064,
|
|
"metrics/advantage_var": 414.7599182128906,
|
|
"regularization/lipschitz_norm": 1282.919189453125,
|
|
"regularization/w1": 1.2418657541275024,
|
|
"rewards/chosen": 0.5074455273132801,
|
|
"rewards/margins": 0.7996735454874291,
|
|
"rewards/rejected": -0.29222801817414906,
|
|
"step": 2280
|
|
},
|
|
{
|
|
"epoch": 2.9792746113989637,
|
|
"grad_norm": 14.278672218322754,
|
|
"kl": 13.657382011413574,
|
|
"learning_rate": 1.4520349279739663e-10,
|
|
"logits/chosen": -34881162.75118859,
|
|
"logits/rejected": -36222960.22187982,
|
|
"logps/chosen": -480.0768621236133,
|
|
"logps/rejected": -370.05342353620955,
|
|
"loss": 0.5551,
|
|
"loss/base_kto": 3.135535955429077,
|
|
"metrics/advantage_var": 473.9613342285156,
|
|
"regularization/lipschitz_norm": 1348.775146484375,
|
|
"regularization/w1": 1.3056143522262573,
|
|
"rewards/chosen": 0.572680489952704,
|
|
"rewards/margins": 0.9162802061627386,
|
|
"rewards/rejected": -0.34359971621003466,
|
|
"step": 2300
|
|
},
|
|
{
|
|
"epoch": 3.0,
|
|
"step": 2316,
|
|
"total_flos": 9.978042558275912e+17,
|
|
"train_loss": 0.5857396982288525,
|
|
"train_runtime": 11128.919,
|
|
"train_samples_per_second": 13.318,
|
|
"train_steps_per_second": 0.208
|
|
}
|
|
],
|
|
"logging_steps": 20,
|
|
"max_steps": 2316,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 3,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": false,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 9.978042558275912e+17,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|