3874 lines
137 KiB
JSON
3874 lines
137 KiB
JSON
|
{
|
||
|
|
"best_global_step": 2500,
|
||
|
|
"best_metric": 0.45733407139778137,
|
||
|
|
"best_model_checkpoint": "models/dpo_default/checkpoint-2500",
|
||
|
|
"epoch": 2.720544217687075,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 2500,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"epoch": 0.010884353741496598,
|
||
|
|
"grad_norm": 0.7077187895774841,
|
||
|
|
"learning_rate": 2.168674698795181e-05,
|
||
|
|
"logits/chosen": -2.2123303413391113,
|
||
|
|
"logits/rejected": -2.1015005111694336,
|
||
|
|
"logps/chosen": -76.98332214355469,
|
||
|
|
"logps/rejected": -83.68041229248047,
|
||
|
|
"loss": 0.6929964542388916,
|
||
|
|
"rewards/accuracies": 0.3492187559604645,
|
||
|
|
"rewards/chosen": 0.0012271858286112547,
|
||
|
|
"rewards/margins": 0.003317171009257436,
|
||
|
|
"rewards/rejected": -0.0020899856463074684,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.021768707482993196,
|
||
|
|
"grad_norm": 0.9575150012969971,
|
||
|
|
"learning_rate": 4.578313253012048e-05,
|
||
|
|
"logits/chosen": -2.3612475395202637,
|
||
|
|
"logits/rejected": -2.317596435546875,
|
||
|
|
"logps/chosen": -79.90132904052734,
|
||
|
|
"logps/rejected": -86.09098815917969,
|
||
|
|
"loss": 0.6921308994293213,
|
||
|
|
"rewards/accuracies": 0.4671874940395355,
|
||
|
|
"rewards/chosen": 0.0005894556525163352,
|
||
|
|
"rewards/margins": 0.00607589865103364,
|
||
|
|
"rewards/rejected": -0.005486442707479,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.0326530612244898,
|
||
|
|
"grad_norm": 0.6923499703407288,
|
||
|
|
"learning_rate": 6.987951807228917e-05,
|
||
|
|
"logits/chosen": -2.2964720726013184,
|
||
|
|
"logits/rejected": -2.2370924949645996,
|
||
|
|
"logps/chosen": -80.2668228149414,
|
||
|
|
"logps/rejected": -87.2217788696289,
|
||
|
|
"loss": 0.6918315887451172,
|
||
|
|
"rewards/accuracies": 0.4765625,
|
||
|
|
"rewards/chosen": -0.019290009513497353,
|
||
|
|
"rewards/margins": 0.00765924621373415,
|
||
|
|
"rewards/rejected": -0.026949256658554077,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.04353741496598639,
|
||
|
|
"grad_norm": 0.6989305019378662,
|
||
|
|
"learning_rate": 9.397590361445784e-05,
|
||
|
|
"logits/chosen": -2.3046586513519287,
|
||
|
|
"logits/rejected": -2.2639377117156982,
|
||
|
|
"logps/chosen": -84.65824890136719,
|
||
|
|
"logps/rejected": -91.65088653564453,
|
||
|
|
"loss": 0.6887531280517578,
|
||
|
|
"rewards/accuracies": 0.48750001192092896,
|
||
|
|
"rewards/chosen": -0.08198987692594528,
|
||
|
|
"rewards/margins": 0.017511751502752304,
|
||
|
|
"rewards/rejected": -0.09950163215398788,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.05442176870748299,
|
||
|
|
"grad_norm": 0.7514657378196716,
|
||
|
|
"learning_rate": 0.00011807228915662652,
|
||
|
|
"logits/chosen": -2.576209545135498,
|
||
|
|
"logits/rejected": -2.5158941745758057,
|
||
|
|
"logps/chosen": -86.98737335205078,
|
||
|
|
"logps/rejected": -93.26823425292969,
|
||
|
|
"loss": 0.6795042991638184,
|
||
|
|
"rewards/accuracies": 0.5015624761581421,
|
||
|
|
"rewards/chosen": -0.16211052238941193,
|
||
|
|
"rewards/margins": 0.05032004788517952,
|
||
|
|
"rewards/rejected": -0.21243056654930115,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.0653061224489796,
|
||
|
|
"grad_norm": 0.8640028834342957,
|
||
|
|
"learning_rate": 0.00014216867469879518,
|
||
|
|
"logits/chosen": -2.4957079887390137,
|
||
|
|
"logits/rejected": -2.3503634929656982,
|
||
|
|
"logps/chosen": -83.06327819824219,
|
||
|
|
"logps/rejected": -88.82142639160156,
|
||
|
|
"loss": 0.6818838596343995,
|
||
|
|
"rewards/accuracies": 0.51171875,
|
||
|
|
"rewards/chosen": -0.2020430564880371,
|
||
|
|
"rewards/margins": 0.05818678066134453,
|
||
|
|
"rewards/rejected": -0.26022982597351074,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.0761904761904762,
|
||
|
|
"grad_norm": 0.8207036256790161,
|
||
|
|
"learning_rate": 0.00016626506024096388,
|
||
|
|
"logits/chosen": -2.699256420135498,
|
||
|
|
"logits/rejected": -2.621330499649048,
|
||
|
|
"logps/chosen": -81.74055480957031,
|
||
|
|
"logps/rejected": -87.18543243408203,
|
||
|
|
"loss": 0.6767777442932129,
|
||
|
|
"rewards/accuracies": 0.520312488079071,
|
||
|
|
"rewards/chosen": -0.19228772819042206,
|
||
|
|
"rewards/margins": 0.07377694547176361,
|
||
|
|
"rewards/rejected": -0.26606467366218567,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.08707482993197278,
|
||
|
|
"grad_norm": 0.8356528878211975,
|
||
|
|
"learning_rate": 0.00019036144578313252,
|
||
|
|
"logits/chosen": -2.595083236694336,
|
||
|
|
"logits/rejected": -2.484344005584717,
|
||
|
|
"logps/chosen": -82.08795166015625,
|
||
|
|
"logps/rejected": -89.19499206542969,
|
||
|
|
"loss": 0.6667680740356445,
|
||
|
|
"rewards/accuracies": 0.5289062261581421,
|
||
|
|
"rewards/chosen": -0.2063063681125641,
|
||
|
|
"rewards/margins": 0.11366782337427139,
|
||
|
|
"rewards/rejected": -0.3199741840362549,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.09795918367346938,
|
||
|
|
"grad_norm": 0.7759003639221191,
|
||
|
|
"learning_rate": 0.00019955123410620793,
|
||
|
|
"logits/chosen": -2.369516611099243,
|
||
|
|
"logits/rejected": -2.255380392074585,
|
||
|
|
"logps/chosen": -80.26573944091797,
|
||
|
|
"logps/rejected": -88.53955841064453,
|
||
|
|
"loss": 0.6667191505432128,
|
||
|
|
"rewards/accuracies": 0.546875,
|
||
|
|
"rewards/chosen": -0.1989096850156784,
|
||
|
|
"rewards/margins": 0.11919046938419342,
|
||
|
|
"rewards/rejected": -0.31810012459754944,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10884353741496598,
|
||
|
|
"grad_norm": 0.7274847626686096,
|
||
|
|
"learning_rate": 0.0001988032909498878,
|
||
|
|
"logits/chosen": -2.4584925174713135,
|
||
|
|
"logits/rejected": -2.306166172027588,
|
||
|
|
"logps/chosen": -85.4821548461914,
|
||
|
|
"logps/rejected": -92.23262786865234,
|
||
|
|
"loss": 0.6666709899902343,
|
||
|
|
"rewards/accuracies": 0.5492187738418579,
|
||
|
|
"rewards/chosen": -0.2420760691165924,
|
||
|
|
"rewards/margins": 0.1285211443901062,
|
||
|
|
"rewards/rejected": -0.370597243309021,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.11972789115646258,
|
||
|
|
"grad_norm": 0.8684931397438049,
|
||
|
|
"learning_rate": 0.0001980553477935677,
|
||
|
|
"logits/chosen": -2.1937153339385986,
|
||
|
|
"logits/rejected": -2.128317356109619,
|
||
|
|
"logps/chosen": -81.6711654663086,
|
||
|
|
"logps/rejected": -89.73558807373047,
|
||
|
|
"loss": 0.6597628593444824,
|
||
|
|
"rewards/accuracies": 0.5523437261581421,
|
||
|
|
"rewards/chosen": -0.1942557394504547,
|
||
|
|
"rewards/margins": 0.15067201852798462,
|
||
|
|
"rewards/rejected": -0.34492772817611694,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1306122448979592,
|
||
|
|
"grad_norm": 0.7956843972206116,
|
||
|
|
"learning_rate": 0.00019730740463724756,
|
||
|
|
"logits/chosen": -2.446894884109497,
|
||
|
|
"logits/rejected": -2.3522589206695557,
|
||
|
|
"logps/chosen": -84.79292297363281,
|
||
|
|
"logps/rejected": -90.49881744384766,
|
||
|
|
"loss": 0.6632381439208984,
|
||
|
|
"rewards/accuracies": 0.542187511920929,
|
||
|
|
"rewards/chosen": -0.19174879789352417,
|
||
|
|
"rewards/margins": 0.14180642366409302,
|
||
|
|
"rewards/rejected": -0.3335552215576172,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1414965986394558,
|
||
|
|
"grad_norm": 0.6242148876190186,
|
||
|
|
"learning_rate": 0.00019655946148092746,
|
||
|
|
"logits/chosen": -2.457613945007324,
|
||
|
|
"logits/rejected": -2.388310670852661,
|
||
|
|
"logps/chosen": -79.19390869140625,
|
||
|
|
"logps/rejected": -85.8818130493164,
|
||
|
|
"loss": 0.6509382724761963,
|
||
|
|
"rewards/accuracies": 0.55078125,
|
||
|
|
"rewards/chosen": -0.1272696703672409,
|
||
|
|
"rewards/margins": 0.17109361290931702,
|
||
|
|
"rewards/rejected": -0.29836326837539673,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1523809523809524,
|
||
|
|
"grad_norm": 0.6959019899368286,
|
||
|
|
"learning_rate": 0.00019581151832460733,
|
||
|
|
"logits/chosen": -2.645115375518799,
|
||
|
|
"logits/rejected": -2.5745694637298584,
|
||
|
|
"logps/chosen": -82.52925109863281,
|
||
|
|
"logps/rejected": -89.37605285644531,
|
||
|
|
"loss": 0.6571790695190429,
|
||
|
|
"rewards/accuracies": 0.5640624761581421,
|
||
|
|
"rewards/chosen": -0.11609281599521637,
|
||
|
|
"rewards/margins": 0.16595318913459778,
|
||
|
|
"rewards/rejected": -0.28204596042633057,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.16326530612244897,
|
||
|
|
"grad_norm": 0.7039981484413147,
|
||
|
|
"learning_rate": 0.00019506357516828722,
|
||
|
|
"logits/chosen": -2.6011252403259277,
|
||
|
|
"logits/rejected": -2.5297441482543945,
|
||
|
|
"logps/chosen": -84.36079406738281,
|
||
|
|
"logps/rejected": -91.32760620117188,
|
||
|
|
"loss": 0.659122371673584,
|
||
|
|
"rewards/accuracies": 0.5640624761581421,
|
||
|
|
"rewards/chosen": -0.04060233011841774,
|
||
|
|
"rewards/margins": 0.16302046179771423,
|
||
|
|
"rewards/rejected": -0.20362277328968048,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.17414965986394557,
|
||
|
|
"grad_norm": 0.6617714166641235,
|
||
|
|
"learning_rate": 0.0001943156320119671,
|
||
|
|
"logits/chosen": -2.715470790863037,
|
||
|
|
"logits/rejected": -2.5970466136932373,
|
||
|
|
"logps/chosen": -83.14956665039062,
|
||
|
|
"logps/rejected": -89.39061737060547,
|
||
|
|
"loss": 0.6440523147583008,
|
||
|
|
"rewards/accuracies": 0.5882812738418579,
|
||
|
|
"rewards/chosen": -0.012846514582633972,
|
||
|
|
"rewards/margins": 0.18811455368995667,
|
||
|
|
"rewards/rejected": -0.20096108317375183,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18503401360544217,
|
||
|
|
"grad_norm": 0.6483933925628662,
|
||
|
|
"learning_rate": 0.000193567688855647,
|
||
|
|
"logits/chosen": -2.807894468307495,
|
||
|
|
"logits/rejected": -2.7104663848876953,
|
||
|
|
"logps/chosen": -79.6620864868164,
|
||
|
|
"logps/rejected": -85.63630676269531,
|
||
|
|
"loss": 0.6529532909393311,
|
||
|
|
"rewards/accuracies": 0.5640624761581421,
|
||
|
|
"rewards/chosen": -0.05956472083926201,
|
||
|
|
"rewards/margins": 0.190535768866539,
|
||
|
|
"rewards/rejected": -0.2501004636287689,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.19591836734693877,
|
||
|
|
"grad_norm": 0.6357870697975159,
|
||
|
|
"learning_rate": 0.00019281974569932686,
|
||
|
|
"logits/chosen": -2.7263660430908203,
|
||
|
|
"logits/rejected": -2.632416248321533,
|
||
|
|
"logps/chosen": -86.24573516845703,
|
||
|
|
"logps/rejected": -91.7590560913086,
|
||
|
|
"loss": 0.6710307121276855,
|
||
|
|
"rewards/accuracies": 0.5570312738418579,
|
||
|
|
"rewards/chosen": -0.18387791514396667,
|
||
|
|
"rewards/margins": 0.14046183228492737,
|
||
|
|
"rewards/rejected": -0.32433977723121643,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20680272108843537,
|
||
|
|
"grad_norm": 0.7087427973747253,
|
||
|
|
"learning_rate": 0.00019207180254300675,
|
||
|
|
"logits/chosen": -2.575040817260742,
|
||
|
|
"logits/rejected": -2.4759504795074463,
|
||
|
|
"logps/chosen": -83.10100555419922,
|
||
|
|
"logps/rejected": -90.81092071533203,
|
||
|
|
"loss": 0.6495565414428711,
|
||
|
|
"rewards/accuracies": 0.571093738079071,
|
||
|
|
"rewards/chosen": -0.2590435743331909,
|
||
|
|
"rewards/margins": 0.18424804508686066,
|
||
|
|
"rewards/rejected": -0.44329166412353516,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.21768707482993196,
|
||
|
|
"grad_norm": 0.7276835441589355,
|
||
|
|
"learning_rate": 0.00019132385938668662,
|
||
|
|
"logits/chosen": -2.6646244525909424,
|
||
|
|
"logits/rejected": -2.642310857772827,
|
||
|
|
"logps/chosen": -84.37451934814453,
|
||
|
|
"logps/rejected": -92.43025970458984,
|
||
|
|
"loss": 0.652790880203247,
|
||
|
|
"rewards/accuracies": 0.5523437261581421,
|
||
|
|
"rewards/chosen": -0.28780898451805115,
|
||
|
|
"rewards/margins": 0.2015586793422699,
|
||
|
|
"rewards/rejected": -0.48936766386032104,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.22857142857142856,
|
||
|
|
"grad_norm": 0.704118549823761,
|
||
|
|
"learning_rate": 0.0001905759162303665,
|
||
|
|
"logits/chosen": -2.7964985370635986,
|
||
|
|
"logits/rejected": -2.7038846015930176,
|
||
|
|
"logps/chosen": -83.10550689697266,
|
||
|
|
"logps/rejected": -90.61512756347656,
|
||
|
|
"loss": 0.6555557250976562,
|
||
|
|
"rewards/accuracies": 0.5703125,
|
||
|
|
"rewards/chosen": -0.17501111328601837,
|
||
|
|
"rewards/margins": 0.1853359341621399,
|
||
|
|
"rewards/rejected": -0.36034709215164185,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.23945578231292516,
|
||
|
|
"grad_norm": 0.6894403100013733,
|
||
|
|
"learning_rate": 0.00018982797307404639,
|
||
|
|
"logits/chosen": -2.694242000579834,
|
||
|
|
"logits/rejected": -2.6799864768981934,
|
||
|
|
"logps/chosen": -78.91683959960938,
|
||
|
|
"logps/rejected": -84.78871154785156,
|
||
|
|
"loss": 0.6462616920471191,
|
||
|
|
"rewards/accuracies": 0.5804687738418579,
|
||
|
|
"rewards/chosen": 0.06024743244051933,
|
||
|
|
"rewards/margins": 0.19001427292823792,
|
||
|
|
"rewards/rejected": -0.12976683676242828,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2503401360544218,
|
||
|
|
"grad_norm": 0.6098469495773315,
|
||
|
|
"learning_rate": 0.00018908002991772625,
|
||
|
|
"logits/chosen": -2.9486870765686035,
|
||
|
|
"logits/rejected": -2.8814079761505127,
|
||
|
|
"logps/chosen": -80.17509460449219,
|
||
|
|
"logps/rejected": -86.77090454101562,
|
||
|
|
"loss": 0.631169319152832,
|
||
|
|
"rewards/accuracies": 0.5921875238418579,
|
||
|
|
"rewards/chosen": 0.039627805352211,
|
||
|
|
"rewards/margins": 0.2388683557510376,
|
||
|
|
"rewards/rejected": -0.1992405354976654,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2612244897959184,
|
||
|
|
"grad_norm": 0.6648709774017334,
|
||
|
|
"learning_rate": 0.00018833208676140615,
|
||
|
|
"logits/chosen": -3.144280195236206,
|
||
|
|
"logits/rejected": -3.156015396118164,
|
||
|
|
"logps/chosen": -82.75923156738281,
|
||
|
|
"logps/rejected": -92.11023712158203,
|
||
|
|
"loss": 0.6405491828918457,
|
||
|
|
"rewards/accuracies": 0.592968761920929,
|
||
|
|
"rewards/chosen": -0.15283265709877014,
|
||
|
|
"rewards/margins": 0.2397342175245285,
|
||
|
|
"rewards/rejected": -0.39256685972213745,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.272108843537415,
|
||
|
|
"grad_norm": 0.7529364824295044,
|
||
|
|
"learning_rate": 0.00018758414360508602,
|
||
|
|
"logits/chosen": -3.0414438247680664,
|
||
|
|
"logits/rejected": -3.0043177604675293,
|
||
|
|
"logps/chosen": -88.2968521118164,
|
||
|
|
"logps/rejected": -97.04661560058594,
|
||
|
|
"loss": 0.6290472984313965,
|
||
|
|
"rewards/accuracies": 0.5914062261581421,
|
||
|
|
"rewards/chosen": -0.38561299443244934,
|
||
|
|
"rewards/margins": 0.28672346472740173,
|
||
|
|
"rewards/rejected": -0.6723364591598511,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2829931972789116,
|
||
|
|
"grad_norm": 0.627895176410675,
|
||
|
|
"learning_rate": 0.00018683620044876591,
|
||
|
|
"logits/chosen": -3.091475009918213,
|
||
|
|
"logits/rejected": -2.9503979682922363,
|
||
|
|
"logps/chosen": -87.58129119873047,
|
||
|
|
"logps/rejected": -93.70463562011719,
|
||
|
|
"loss": 0.6199825286865235,
|
||
|
|
"rewards/accuracies": 0.5859375,
|
||
|
|
"rewards/chosen": -0.43795761466026306,
|
||
|
|
"rewards/margins": 0.28398531675338745,
|
||
|
|
"rewards/rejected": -0.7219429612159729,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2938775510204082,
|
||
|
|
"grad_norm": 0.562925398349762,
|
||
|
|
"learning_rate": 0.00018608825729244578,
|
||
|
|
"logits/chosen": -3.0558292865753174,
|
||
|
|
"logits/rejected": -2.9512274265289307,
|
||
|
|
"logps/chosen": -80.02171325683594,
|
||
|
|
"logps/rejected": -88.43952941894531,
|
||
|
|
"loss": 0.6295814514160156,
|
||
|
|
"rewards/accuracies": 0.5960937738418579,
|
||
|
|
"rewards/chosen": -0.35493478178977966,
|
||
|
|
"rewards/margins": 0.2739941477775574,
|
||
|
|
"rewards/rejected": -0.6289288997650146,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3047619047619048,
|
||
|
|
"grad_norm": 0.62668776512146,
|
||
|
|
"learning_rate": 0.00018534031413612568,
|
||
|
|
"logits/chosen": -3.073023557662964,
|
||
|
|
"logits/rejected": -2.9786252975463867,
|
||
|
|
"logps/chosen": -79.61070251464844,
|
||
|
|
"logps/rejected": -87.7259750366211,
|
||
|
|
"loss": 0.6259790420532226,
|
||
|
|
"rewards/accuracies": 0.5914062261581421,
|
||
|
|
"rewards/chosen": -0.3159467875957489,
|
||
|
|
"rewards/margins": 0.2681388556957245,
|
||
|
|
"rewards/rejected": -0.5840856432914734,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.31564625850340133,
|
||
|
|
"grad_norm": 0.6235923767089844,
|
||
|
|
"learning_rate": 0.00018459237097980555,
|
||
|
|
"logits/chosen": -3.129662036895752,
|
||
|
|
"logits/rejected": -3.073840379714966,
|
||
|
|
"logps/chosen": -81.99674987792969,
|
||
|
|
"logps/rejected": -90.57108306884766,
|
||
|
|
"loss": 0.6291594982147217,
|
||
|
|
"rewards/accuracies": 0.585156261920929,
|
||
|
|
"rewards/chosen": -0.2529909014701843,
|
||
|
|
"rewards/margins": 0.28501176834106445,
|
||
|
|
"rewards/rejected": -0.5380026698112488,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.32653061224489793,
|
||
|
|
"grad_norm": 0.6637018322944641,
|
||
|
|
"learning_rate": 0.00018384442782348544,
|
||
|
|
"logits/chosen": -3.1975247859954834,
|
||
|
|
"logits/rejected": -3.1137585639953613,
|
||
|
|
"logps/chosen": -80.12820434570312,
|
||
|
|
"logps/rejected": -88.26316833496094,
|
||
|
|
"loss": 0.6194732189178467,
|
||
|
|
"rewards/accuracies": 0.600781261920929,
|
||
|
|
"rewards/chosen": -0.0812927708029747,
|
||
|
|
"rewards/margins": 0.3114483058452606,
|
||
|
|
"rewards/rejected": -0.3927411139011383,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.33741496598639453,
|
||
|
|
"grad_norm": 0.6672607660293579,
|
||
|
|
"learning_rate": 0.0001830964846671653,
|
||
|
|
"logits/chosen": -2.985579252243042,
|
||
|
|
"logits/rejected": -2.9056222438812256,
|
||
|
|
"logps/chosen": -86.5948257446289,
|
||
|
|
"logps/rejected": -95.234619140625,
|
||
|
|
"loss": 0.6183744430541992,
|
||
|
|
"rewards/accuracies": 0.598437488079071,
|
||
|
|
"rewards/chosen": -0.3788287043571472,
|
||
|
|
"rewards/margins": 0.31600421667099,
|
||
|
|
"rewards/rejected": -0.694832980632782,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.34829931972789113,
|
||
|
|
"grad_norm": 0.6171224117279053,
|
||
|
|
"learning_rate": 0.0001823485415108452,
|
||
|
|
"logits/chosen": -3.0042669773101807,
|
||
|
|
"logits/rejected": -2.951925754547119,
|
||
|
|
"logps/chosen": -88.96502685546875,
|
||
|
|
"logps/rejected": -97.89619445800781,
|
||
|
|
"loss": 0.6247763633728027,
|
||
|
|
"rewards/accuracies": 0.5843750238418579,
|
||
|
|
"rewards/chosen": -0.5990578532218933,
|
||
|
|
"rewards/margins": 0.300613671541214,
|
||
|
|
"rewards/rejected": -0.8996715545654297,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.35918367346938773,
|
||
|
|
"grad_norm": 0.7893000245094299,
|
||
|
|
"learning_rate": 0.00018160059835452508,
|
||
|
|
"logits/chosen": -3.1310553550720215,
|
||
|
|
"logits/rejected": -3.1132781505584717,
|
||
|
|
"logps/chosen": -84.33443450927734,
|
||
|
|
"logps/rejected": -94.07998657226562,
|
||
|
|
"loss": 0.6248671531677246,
|
||
|
|
"rewards/accuracies": 0.5859375,
|
||
|
|
"rewards/chosen": -0.6389909982681274,
|
||
|
|
"rewards/margins": 0.29910796880722046,
|
||
|
|
"rewards/rejected": -0.9380990266799927,
|
||
|
|
"step": 330
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.37006802721088433,
|
||
|
|
"grad_norm": 0.5814021825790405,
|
||
|
|
"learning_rate": 0.00018085265519820494,
|
||
|
|
"logits/chosen": -3.1776278018951416,
|
||
|
|
"logits/rejected": -3.08312726020813,
|
||
|
|
"logps/chosen": -86.03544616699219,
|
||
|
|
"logps/rejected": -93.88951110839844,
|
||
|
|
"loss": 0.6233505249023438,
|
||
|
|
"rewards/accuracies": 0.5921875238418579,
|
||
|
|
"rewards/chosen": -0.6253015398979187,
|
||
|
|
"rewards/margins": 0.3192492425441742,
|
||
|
|
"rewards/rejected": -0.9445506930351257,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.38095238095238093,
|
||
|
|
"grad_norm": 0.63541179895401,
|
||
|
|
"learning_rate": 0.0001801047120418848,
|
||
|
|
"logits/chosen": -2.9307355880737305,
|
||
|
|
"logits/rejected": -2.9118292331695557,
|
||
|
|
"logps/chosen": -89.96821594238281,
|
||
|
|
"logps/rejected": -98.78160095214844,
|
||
|
|
"loss": 0.6067781925201416,
|
||
|
|
"rewards/accuracies": 0.61328125,
|
||
|
|
"rewards/chosen": -0.5151289701461792,
|
||
|
|
"rewards/margins": 0.34935957193374634,
|
||
|
|
"rewards/rejected": -0.8644886016845703,
|
||
|
|
"step": 350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.39183673469387753,
|
||
|
|
"grad_norm": 0.695551872253418,
|
||
|
|
"learning_rate": 0.0001793567688855647,
|
||
|
|
"logits/chosen": -2.7680797576904297,
|
||
|
|
"logits/rejected": -2.694873332977295,
|
||
|
|
"logps/chosen": -84.47620391845703,
|
||
|
|
"logps/rejected": -91.43312072753906,
|
||
|
|
"loss": 0.6245352268218994,
|
||
|
|
"rewards/accuracies": 0.6015625,
|
||
|
|
"rewards/chosen": -0.5855204463005066,
|
||
|
|
"rewards/margins": 0.2961046099662781,
|
||
|
|
"rewards/rejected": -0.8816250562667847,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.40272108843537413,
|
||
|
|
"grad_norm": 0.5492284297943115,
|
||
|
|
"learning_rate": 0.00017860882572924458,
|
||
|
|
"logits/chosen": -2.8768391609191895,
|
||
|
|
"logits/rejected": -2.7919318675994873,
|
||
|
|
"logps/chosen": -88.09062194824219,
|
||
|
|
"logps/rejected": -94.99168395996094,
|
||
|
|
"loss": 0.6229678630828858,
|
||
|
|
"rewards/accuracies": 0.602343738079071,
|
||
|
|
"rewards/chosen": -0.6255002021789551,
|
||
|
|
"rewards/margins": 0.2889278531074524,
|
||
|
|
"rewards/rejected": -0.9144280552864075,
|
||
|
|
"step": 370
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.41360544217687073,
|
||
|
|
"grad_norm": 0.627325713634491,
|
||
|
|
"learning_rate": 0.00017786088257292445,
|
||
|
|
"logits/chosen": -2.6934731006622314,
|
||
|
|
"logits/rejected": -2.595776319503784,
|
||
|
|
"logps/chosen": -89.07003021240234,
|
||
|
|
"logps/rejected": -97.8858871459961,
|
||
|
|
"loss": 0.6102587699890136,
|
||
|
|
"rewards/accuracies": 0.6078125238418579,
|
||
|
|
"rewards/chosen": -0.5627188682556152,
|
||
|
|
"rewards/margins": 0.3145248293876648,
|
||
|
|
"rewards/rejected": -0.87724369764328,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.42448979591836733,
|
||
|
|
"grad_norm": 0.6781789064407349,
|
||
|
|
"learning_rate": 0.00017711293941660434,
|
||
|
|
"logits/chosen": -2.902066946029663,
|
||
|
|
"logits/rejected": -2.792140245437622,
|
||
|
|
"logps/chosen": -85.96211242675781,
|
||
|
|
"logps/rejected": -94.57718658447266,
|
||
|
|
"loss": 0.6128390789031982,
|
||
|
|
"rewards/accuracies": 0.621874988079071,
|
||
|
|
"rewards/chosen": -0.5092566013336182,
|
||
|
|
"rewards/margins": 0.33838534355163574,
|
||
|
|
"rewards/rejected": -0.8476420640945435,
|
||
|
|
"step": 390
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.43537414965986393,
|
||
|
|
"grad_norm": 0.6318673491477966,
|
||
|
|
"learning_rate": 0.0001763649962602842,
|
||
|
|
"logits/chosen": -2.978280544281006,
|
||
|
|
"logits/rejected": -2.9661154747009277,
|
||
|
|
"logps/chosen": -82.64006805419922,
|
||
|
|
"logps/rejected": -93.13764953613281,
|
||
|
|
"loss": 0.6212802410125733,
|
||
|
|
"rewards/accuracies": 0.610156238079071,
|
||
|
|
"rewards/chosen": -0.354736864566803,
|
||
|
|
"rewards/margins": 0.31057295203208923,
|
||
|
|
"rewards/rejected": -0.6653097867965698,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.44625850340136053,
|
||
|
|
"grad_norm": 0.6015628576278687,
|
||
|
|
"learning_rate": 0.0001756170531039641,
|
||
|
|
"logits/chosen": -3.077155590057373,
|
||
|
|
"logits/rejected": -3.0079360008239746,
|
||
|
|
"logps/chosen": -84.97288513183594,
|
||
|
|
"logps/rejected": -93.01554870605469,
|
||
|
|
"loss": 0.6140561580657959,
|
||
|
|
"rewards/accuracies": 0.594531238079071,
|
||
|
|
"rewards/chosen": -0.3532416820526123,
|
||
|
|
"rewards/margins": 0.34475868940353394,
|
||
|
|
"rewards/rejected": -0.6980003714561462,
|
||
|
|
"step": 410
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.45714285714285713,
|
||
|
|
"grad_norm": 0.6051854491233826,
|
||
|
|
"learning_rate": 0.00017486910994764398,
|
||
|
|
"logits/chosen": -3.0866477489471436,
|
||
|
|
"logits/rejected": -2.9812045097351074,
|
||
|
|
"logps/chosen": -79.9186019897461,
|
||
|
|
"logps/rejected": -88.40470886230469,
|
||
|
|
"loss": 0.6004165649414063,
|
||
|
|
"rewards/accuracies": 0.625781238079071,
|
||
|
|
"rewards/chosen": -0.21888461709022522,
|
||
|
|
"rewards/margins": 0.36811959743499756,
|
||
|
|
"rewards/rejected": -0.5870041847229004,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.46802721088435373,
|
||
|
|
"grad_norm": 0.7300223708152771,
|
||
|
|
"learning_rate": 0.00017412116679132387,
|
||
|
|
"logits/chosen": -3.060652017593384,
|
||
|
|
"logits/rejected": -3.027360439300537,
|
||
|
|
"logps/chosen": -84.05428314208984,
|
||
|
|
"logps/rejected": -91.3875503540039,
|
||
|
|
"loss": 0.6200145244598388,
|
||
|
|
"rewards/accuracies": 0.604687511920929,
|
||
|
|
"rewards/chosen": -0.22208240628242493,
|
||
|
|
"rewards/margins": 0.3296840786933899,
|
||
|
|
"rewards/rejected": -0.5517665147781372,
|
||
|
|
"step": 430
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.47891156462585033,
|
||
|
|
"grad_norm": 0.5995689630508423,
|
||
|
|
"learning_rate": 0.00017337322363500374,
|
||
|
|
"logits/chosen": -2.9303958415985107,
|
||
|
|
"logits/rejected": -2.8347811698913574,
|
||
|
|
"logps/chosen": -88.16230773925781,
|
||
|
|
"logps/rejected": -94.56944274902344,
|
||
|
|
"loss": 0.6032320499420166,
|
||
|
|
"rewards/accuracies": 0.6304687261581421,
|
||
|
|
"rewards/chosen": -0.506262481212616,
|
||
|
|
"rewards/margins": 0.3537197411060333,
|
||
|
|
"rewards/rejected": -0.8599823117256165,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4897959183673469,
|
||
|
|
"grad_norm": 0.6093873381614685,
|
||
|
|
"learning_rate": 0.00017262528047868364,
|
||
|
|
"logits/chosen": -3.054971218109131,
|
||
|
|
"logits/rejected": -2.9332528114318848,
|
||
|
|
"logps/chosen": -85.0113525390625,
|
||
|
|
"logps/rejected": -94.11463928222656,
|
||
|
|
"loss": 0.5989380359649659,
|
||
|
|
"rewards/accuracies": 0.6117187738418579,
|
||
|
|
"rewards/chosen": -0.5802456140518188,
|
||
|
|
"rewards/margins": 0.3973398804664612,
|
||
|
|
"rewards/rejected": -0.97758549451828,
|
||
|
|
"step": 450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5006802721088436,
|
||
|
|
"grad_norm": 0.6394651532173157,
|
||
|
|
"learning_rate": 0.0001718773373223635,
|
||
|
|
"logits/chosen": -2.9823951721191406,
|
||
|
|
"logits/rejected": -2.9353485107421875,
|
||
|
|
"logps/chosen": -86.30118560791016,
|
||
|
|
"logps/rejected": -94.50347900390625,
|
||
|
|
"loss": 0.6100581169128418,
|
||
|
|
"rewards/accuracies": 0.6171875,
|
||
|
|
"rewards/chosen": -0.5156108140945435,
|
||
|
|
"rewards/margins": 0.34870487451553345,
|
||
|
|
"rewards/rejected": -0.8643158078193665,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5115646258503401,
|
||
|
|
"grad_norm": 0.5491212010383606,
|
||
|
|
"learning_rate": 0.0001711293941660434,
|
||
|
|
"logits/chosen": -2.926840305328369,
|
||
|
|
"logits/rejected": -2.8748295307159424,
|
||
|
|
"logps/chosen": -81.23180389404297,
|
||
|
|
"logps/rejected": -88.91163635253906,
|
||
|
|
"loss": 0.601882791519165,
|
||
|
|
"rewards/accuracies": 0.606249988079071,
|
||
|
|
"rewards/chosen": -0.35410138964653015,
|
||
|
|
"rewards/margins": 0.377445787191391,
|
||
|
|
"rewards/rejected": -0.7315471768379211,
|
||
|
|
"step": 470
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5224489795918368,
|
||
|
|
"grad_norm": 0.6345949172973633,
|
||
|
|
"learning_rate": 0.00017038145100972327,
|
||
|
|
"logits/chosen": -2.8460724353790283,
|
||
|
|
"logits/rejected": -2.8185269832611084,
|
||
|
|
"logps/chosen": -82.52976989746094,
|
||
|
|
"logps/rejected": -91.4373779296875,
|
||
|
|
"loss": 0.5966329097747802,
|
||
|
|
"rewards/accuracies": 0.60546875,
|
||
|
|
"rewards/chosen": -0.2420049011707306,
|
||
|
|
"rewards/margins": 0.3614209294319153,
|
||
|
|
"rewards/rejected": -0.6034258604049683,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5333333333333333,
|
||
|
|
"grad_norm": 0.6204262971878052,
|
||
|
|
"learning_rate": 0.00016963350785340316,
|
||
|
|
"logits/chosen": -3.037978410720825,
|
||
|
|
"logits/rejected": -2.9520788192749023,
|
||
|
|
"logps/chosen": -83.21837615966797,
|
||
|
|
"logps/rejected": -91.0192642211914,
|
||
|
|
"loss": 0.5877750396728516,
|
||
|
|
"rewards/accuracies": 0.6429687738418579,
|
||
|
|
"rewards/chosen": -0.35447391867637634,
|
||
|
|
"rewards/margins": 0.4061393737792969,
|
||
|
|
"rewards/rejected": -0.7606132626533508,
|
||
|
|
"step": 490
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.54421768707483,
|
||
|
|
"grad_norm": 0.6997068524360657,
|
||
|
|
"learning_rate": 0.00016888556469708303,
|
||
|
|
"logits/chosen": -3.0016021728515625,
|
||
|
|
"logits/rejected": -2.9584157466888428,
|
||
|
|
"logps/chosen": -88.25440979003906,
|
||
|
|
"logps/rejected": -98.35990142822266,
|
||
|
|
"loss": 0.5975300312042237,
|
||
|
|
"rewards/accuracies": 0.6109374761581421,
|
||
|
|
"rewards/chosen": -0.6453801393508911,
|
||
|
|
"rewards/margins": 0.40526071190834045,
|
||
|
|
"rewards/rejected": -1.0506408214569092,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.54421768707483,
|
||
|
|
"eval_logits/chosen": -3.0864946842193604,
|
||
|
|
"eval_logits/rejected": -3.039473056793213,
|
||
|
|
"eval_logps/chosen": -85.64629364013672,
|
||
|
|
"eval_logps/rejected": -96.53662872314453,
|
||
|
|
"eval_loss": 0.5770927667617798,
|
||
|
|
"eval_rewards/accuracies": 0.625,
|
||
|
|
"eval_rewards/chosen": -0.604114294052124,
|
||
|
|
"eval_rewards/margins": 0.4646129906177521,
|
||
|
|
"eval_rewards/rejected": -1.0687271356582642,
|
||
|
|
"eval_runtime": 104.8956,
|
||
|
|
"eval_samples_per_second": 22.88,
|
||
|
|
"eval_steps_per_second": 2.86,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5551020408163265,
|
||
|
|
"grad_norm": 0.5718573331832886,
|
||
|
|
"learning_rate": 0.0001681376215407629,
|
||
|
|
"logits/chosen": -2.98121976852417,
|
||
|
|
"logits/rejected": -3.0054566860198975,
|
||
|
|
"logps/chosen": -84.3095932006836,
|
||
|
|
"logps/rejected": -96.11637115478516,
|
||
|
|
"loss": 0.5776225090026855,
|
||
|
|
"rewards/accuracies": 0.628125011920929,
|
||
|
|
"rewards/chosen": -0.6291953921318054,
|
||
|
|
"rewards/margins": 0.4740595817565918,
|
||
|
|
"rewards/rejected": -1.103255033493042,
|
||
|
|
"step": 510
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5659863945578232,
|
||
|
|
"grad_norm": 0.6719241738319397,
|
||
|
|
"learning_rate": 0.0001673896783844428,
|
||
|
|
"logits/chosen": -3.1065757274627686,
|
||
|
|
"logits/rejected": -3.057704448699951,
|
||
|
|
"logps/chosen": -84.51625061035156,
|
||
|
|
"logps/rejected": -95.70087432861328,
|
||
|
|
"loss": 0.5797244071960449,
|
||
|
|
"rewards/accuracies": 0.6539062261581421,
|
||
|
|
"rewards/chosen": -0.6672269105911255,
|
||
|
|
"rewards/margins": 0.4843328595161438,
|
||
|
|
"rewards/rejected": -1.1515597105026245,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5768707482993197,
|
||
|
|
"grad_norm": 0.6521556377410889,
|
||
|
|
"learning_rate": 0.00016664173522812267,
|
||
|
|
"logits/chosen": -3.0161333084106445,
|
||
|
|
"logits/rejected": -3.000890016555786,
|
||
|
|
"logps/chosen": -85.24427795410156,
|
||
|
|
"logps/rejected": -94.50189208984375,
|
||
|
|
"loss": 0.5926599979400635,
|
||
|
|
"rewards/accuracies": 0.625,
|
||
|
|
"rewards/chosen": -0.6613038778305054,
|
||
|
|
"rewards/margins": 0.4472618103027344,
|
||
|
|
"rewards/rejected": -1.1085655689239502,
|
||
|
|
"step": 530
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5877551020408164,
|
||
|
|
"grad_norm": 0.9451216459274292,
|
||
|
|
"learning_rate": 0.00016589379207180256,
|
||
|
|
"logits/chosen": -2.8275389671325684,
|
||
|
|
"logits/rejected": -2.8195090293884277,
|
||
|
|
"logps/chosen": -87.73820495605469,
|
||
|
|
"logps/rejected": -98.05986785888672,
|
||
|
|
"loss": 0.6018657684326172,
|
||
|
|
"rewards/accuracies": 0.6265624761581421,
|
||
|
|
"rewards/chosen": -0.6853042840957642,
|
||
|
|
"rewards/margins": 0.4554131031036377,
|
||
|
|
"rewards/rejected": -1.1407173871994019,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5986394557823129,
|
||
|
|
"grad_norm": 0.610352098941803,
|
||
|
|
"learning_rate": 0.00016514584891548243,
|
||
|
|
"logits/chosen": -2.712162733078003,
|
||
|
|
"logits/rejected": -2.6361680030822754,
|
||
|
|
"logps/chosen": -86.30110168457031,
|
||
|
|
"logps/rejected": -96.72479248046875,
|
||
|
|
"loss": 0.5942647457122803,
|
||
|
|
"rewards/accuracies": 0.6312500238418579,
|
||
|
|
"rewards/chosen": -0.7625668048858643,
|
||
|
|
"rewards/margins": 0.41246652603149414,
|
||
|
|
"rewards/rejected": -1.1750333309173584,
|
||
|
|
"step": 550
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6095238095238096,
|
||
|
|
"grad_norm": 0.5952523946762085,
|
||
|
|
"learning_rate": 0.00016439790575916233,
|
||
|
|
"logits/chosen": -2.773089647293091,
|
||
|
|
"logits/rejected": -2.675023078918457,
|
||
|
|
"logps/chosen": -85.59429168701172,
|
||
|
|
"logps/rejected": -94.7846908569336,
|
||
|
|
"loss": 0.5996862411499023,
|
||
|
|
"rewards/accuracies": 0.632031261920929,
|
||
|
|
"rewards/chosen": -0.7641364336013794,
|
||
|
|
"rewards/margins": 0.39174142479896545,
|
||
|
|
"rewards/rejected": -1.1558778285980225,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6204081632653061,
|
||
|
|
"grad_norm": 0.6582514047622681,
|
||
|
|
"learning_rate": 0.0001636499626028422,
|
||
|
|
"logits/chosen": -2.78428316116333,
|
||
|
|
"logits/rejected": -2.6511282920837402,
|
||
|
|
"logps/chosen": -90.34364318847656,
|
||
|
|
"logps/rejected": -99.95307159423828,
|
||
|
|
"loss": 0.5831423759460449,
|
||
|
|
"rewards/accuracies": 0.625781238079071,
|
||
|
|
"rewards/chosen": -0.7876542210578918,
|
||
|
|
"rewards/margins": 0.46621885895729065,
|
||
|
|
"rewards/rejected": -1.2538731098175049,
|
||
|
|
"step": 570
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6312925170068027,
|
||
|
|
"grad_norm": 0.5589143633842468,
|
||
|
|
"learning_rate": 0.0001629020194465221,
|
||
|
|
"logits/chosen": -2.9068286418914795,
|
||
|
|
"logits/rejected": -2.8854146003723145,
|
||
|
|
"logps/chosen": -83.24221801757812,
|
||
|
|
"logps/rejected": -91.76869201660156,
|
||
|
|
"loss": 0.5974715709686279,
|
||
|
|
"rewards/accuracies": 0.625,
|
||
|
|
"rewards/chosen": -0.520553469657898,
|
||
|
|
"rewards/margins": 0.39965444803237915,
|
||
|
|
"rewards/rejected": -0.9202079772949219,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6421768707482993,
|
||
|
|
"grad_norm": 0.6037158370018005,
|
||
|
|
"learning_rate": 0.00016215407629020196,
|
||
|
|
"logits/chosen": -2.9200706481933594,
|
||
|
|
"logits/rejected": -2.8421108722686768,
|
||
|
|
"logps/chosen": -83.75981140136719,
|
||
|
|
"logps/rejected": -94.21287536621094,
|
||
|
|
"loss": 0.5780706405639648,
|
||
|
|
"rewards/accuracies": 0.641406238079071,
|
||
|
|
"rewards/chosen": -0.40980464220046997,
|
||
|
|
"rewards/margins": 0.4663706421852112,
|
||
|
|
"rewards/rejected": -0.8761752843856812,
|
||
|
|
"step": 590
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6530612244897959,
|
||
|
|
"grad_norm": 0.6423654556274414,
|
||
|
|
"learning_rate": 0.00016140613313388185,
|
||
|
|
"logits/chosen": -2.929079532623291,
|
||
|
|
"logits/rejected": -2.9537947177886963,
|
||
|
|
"logps/chosen": -86.27957916259766,
|
||
|
|
"logps/rejected": -96.39942932128906,
|
||
|
|
"loss": 0.5839208602905274,
|
||
|
|
"rewards/accuracies": 0.6585937738418579,
|
||
|
|
"rewards/chosen": -0.5493398904800415,
|
||
|
|
"rewards/margins": 0.4438067376613617,
|
||
|
|
"rewards/rejected": -0.9931465983390808,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6639455782312925,
|
||
|
|
"grad_norm": 0.5478850603103638,
|
||
|
|
"learning_rate": 0.00016065818997756172,
|
||
|
|
"logits/chosen": -2.835813283920288,
|
||
|
|
"logits/rejected": -2.7998528480529785,
|
||
|
|
"logps/chosen": -87.89683532714844,
|
||
|
|
"logps/rejected": -96.35087585449219,
|
||
|
|
"loss": 0.5724361896514892,
|
||
|
|
"rewards/accuracies": 0.643750011920929,
|
||
|
|
"rewards/chosen": -0.5621733665466309,
|
||
|
|
"rewards/margins": 0.4720228612422943,
|
||
|
|
"rewards/rejected": -1.034196138381958,
|
||
|
|
"step": 610
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6748299319727891,
|
||
|
|
"grad_norm": 0.671947717666626,
|
||
|
|
"learning_rate": 0.0001599102468212416,
|
||
|
|
"logits/chosen": -2.9626426696777344,
|
||
|
|
"logits/rejected": -2.8480992317199707,
|
||
|
|
"logps/chosen": -88.1216812133789,
|
||
|
|
"logps/rejected": -98.25172424316406,
|
||
|
|
"loss": 0.5745330333709717,
|
||
|
|
"rewards/accuracies": 0.653124988079071,
|
||
|
|
"rewards/chosen": -0.7491940259933472,
|
||
|
|
"rewards/margins": 0.4780084490776062,
|
||
|
|
"rewards/rejected": -1.2272025346755981,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6857142857142857,
|
||
|
|
"grad_norm": 0.6402953267097473,
|
||
|
|
"learning_rate": 0.00015916230366492146,
|
||
|
|
"logits/chosen": -2.9432570934295654,
|
||
|
|
"logits/rejected": -2.97039794921875,
|
||
|
|
"logps/chosen": -90.78302001953125,
|
||
|
|
"logps/rejected": -103.5708236694336,
|
||
|
|
"loss": 0.5667342185974121,
|
||
|
|
"rewards/accuracies": 0.66015625,
|
||
|
|
"rewards/chosen": -0.9368416666984558,
|
||
|
|
"rewards/margins": 0.5184968709945679,
|
||
|
|
"rewards/rejected": -1.455338478088379,
|
||
|
|
"step": 630
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6965986394557823,
|
||
|
|
"grad_norm": 0.6474503874778748,
|
||
|
|
"learning_rate": 0.00015841436050860136,
|
||
|
|
"logits/chosen": -2.978628635406494,
|
||
|
|
"logits/rejected": -2.9383959770202637,
|
||
|
|
"logps/chosen": -89.43360900878906,
|
||
|
|
"logps/rejected": -100.71207427978516,
|
||
|
|
"loss": 0.5545726776123047,
|
||
|
|
"rewards/accuracies": 0.6460937261581421,
|
||
|
|
"rewards/chosen": -0.9920727610588074,
|
||
|
|
"rewards/margins": 0.5569532513618469,
|
||
|
|
"rewards/rejected": -1.5490261316299438,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7074829931972789,
|
||
|
|
"grad_norm": 0.7136415243148804,
|
||
|
|
"learning_rate": 0.00015766641735228122,
|
||
|
|
"logits/chosen": -3.020536184310913,
|
||
|
|
"logits/rejected": -2.9737136363983154,
|
||
|
|
"logps/chosen": -91.73661041259766,
|
||
|
|
"logps/rejected": -102.14755249023438,
|
||
|
|
"loss": 0.5740270614624023,
|
||
|
|
"rewards/accuracies": 0.637499988079071,
|
||
|
|
"rewards/chosen": -1.027769923210144,
|
||
|
|
"rewards/margins": 0.5058225393295288,
|
||
|
|
"rewards/rejected": -1.5335925817489624,
|
||
|
|
"step": 650
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7183673469387755,
|
||
|
|
"grad_norm": 0.601219892501831,
|
||
|
|
"learning_rate": 0.0001569184741959611,
|
||
|
|
"logits/chosen": -3.1565613746643066,
|
||
|
|
"logits/rejected": -3.1246376037597656,
|
||
|
|
"logps/chosen": -90.3838119506836,
|
||
|
|
"logps/rejected": -101.56874084472656,
|
||
|
|
"loss": 0.5599509239196777,
|
||
|
|
"rewards/accuracies": 0.655468761920929,
|
||
|
|
"rewards/chosen": -1.081892490386963,
|
||
|
|
"rewards/margins": 0.559756875038147,
|
||
|
|
"rewards/rejected": -1.6416492462158203,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7292517006802721,
|
||
|
|
"grad_norm": 0.6408318877220154,
|
||
|
|
"learning_rate": 0.000156170531039641,
|
||
|
|
"logits/chosen": -3.2291476726531982,
|
||
|
|
"logits/rejected": -3.2356181144714355,
|
||
|
|
"logps/chosen": -89.50052642822266,
|
||
|
|
"logps/rejected": -100.52101135253906,
|
||
|
|
"loss": 0.5731996536254883,
|
||
|
|
"rewards/accuracies": 0.647656261920929,
|
||
|
|
"rewards/chosen": -1.1054667234420776,
|
||
|
|
"rewards/margins": 0.5072154998779297,
|
||
|
|
"rewards/rejected": -1.6126823425292969,
|
||
|
|
"step": 670
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7401360544217687,
|
||
|
|
"grad_norm": 0.6599347591400146,
|
||
|
|
"learning_rate": 0.00015542258788332086,
|
||
|
|
"logits/chosen": -3.2114415168762207,
|
||
|
|
"logits/rejected": -3.172548532485962,
|
||
|
|
"logps/chosen": -91.56523132324219,
|
||
|
|
"logps/rejected": -102.18913269042969,
|
||
|
|
"loss": 0.5616633892059326,
|
||
|
|
"rewards/accuracies": 0.6640625,
|
||
|
|
"rewards/chosen": -0.8735089302062988,
|
||
|
|
"rewards/margins": 0.5582183599472046,
|
||
|
|
"rewards/rejected": -1.4317272901535034,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7510204081632653,
|
||
|
|
"grad_norm": 0.668644905090332,
|
||
|
|
"learning_rate": 0.00015467464472700075,
|
||
|
|
"logits/chosen": -3.2348804473876953,
|
||
|
|
"logits/rejected": -3.097151041030884,
|
||
|
|
"logps/chosen": -90.90170288085938,
|
||
|
|
"logps/rejected": -102.48905944824219,
|
||
|
|
"loss": 0.549819803237915,
|
||
|
|
"rewards/accuracies": 0.6695312261581421,
|
||
|
|
"rewards/chosen": -0.6966196894645691,
|
||
|
|
"rewards/margins": 0.6086488366127014,
|
||
|
|
"rewards/rejected": -1.3052685260772705,
|
||
|
|
"step": 690
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7619047619047619,
|
||
|
|
"grad_norm": 0.776343822479248,
|
||
|
|
"learning_rate": 0.00015392670157068062,
|
||
|
|
"logits/chosen": -3.261051893234253,
|
||
|
|
"logits/rejected": -3.150120258331299,
|
||
|
|
"logps/chosen": -85.77153015136719,
|
||
|
|
"logps/rejected": -96.22288513183594,
|
||
|
|
"loss": 0.5865112781524658,
|
||
|
|
"rewards/accuracies": 0.6539062261581421,
|
||
|
|
"rewards/chosen": -0.6080220937728882,
|
||
|
|
"rewards/margins": 0.5111768841743469,
|
||
|
|
"rewards/rejected": -1.1191989183425903,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7727891156462585,
|
||
|
|
"grad_norm": 0.6534438729286194,
|
||
|
|
"learning_rate": 0.00015317875841436052,
|
||
|
|
"logits/chosen": -2.9951682090759277,
|
||
|
|
"logits/rejected": -2.9624991416931152,
|
||
|
|
"logps/chosen": -88.7505874633789,
|
||
|
|
"logps/rejected": -99.82380676269531,
|
||
|
|
"loss": 0.5636815071105957,
|
||
|
|
"rewards/accuracies": 0.6507812738418579,
|
||
|
|
"rewards/chosen": -0.7708442807197571,
|
||
|
|
"rewards/margins": 0.5377318263053894,
|
||
|
|
"rewards/rejected": -1.308576226234436,
|
||
|
|
"step": 710
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7836734693877551,
|
||
|
|
"grad_norm": 0.5064298510551453,
|
||
|
|
"learning_rate": 0.0001524308152580404,
|
||
|
|
"logits/chosen": -3.1614372730255127,
|
||
|
|
"logits/rejected": -3.1034646034240723,
|
||
|
|
"logps/chosen": -84.72537994384766,
|
||
|
|
"logps/rejected": -96.15172576904297,
|
||
|
|
"loss": 0.5600537300109864,
|
||
|
|
"rewards/accuracies": 0.637499988079071,
|
||
|
|
"rewards/chosen": -0.8120764493942261,
|
||
|
|
"rewards/margins": 0.5258986353874207,
|
||
|
|
"rewards/rejected": -1.337975263595581,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7945578231292517,
|
||
|
|
"grad_norm": 0.6030212640762329,
|
||
|
|
"learning_rate": 0.00015168287210172028,
|
||
|
|
"logits/chosen": -3.1010847091674805,
|
||
|
|
"logits/rejected": -3.0516085624694824,
|
||
|
|
"logps/chosen": -85.84700775146484,
|
||
|
|
"logps/rejected": -96.8555908203125,
|
||
|
|
"loss": 0.5588539123535157,
|
||
|
|
"rewards/accuracies": 0.6499999761581421,
|
||
|
|
"rewards/chosen": -0.7111436128616333,
|
||
|
|
"rewards/margins": 0.5404245853424072,
|
||
|
|
"rewards/rejected": -1.251568078994751,
|
||
|
|
"step": 730
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8054421768707483,
|
||
|
|
"grad_norm": 0.6080058217048645,
|
||
|
|
"learning_rate": 0.00015093492894540015,
|
||
|
|
"logits/chosen": -3.1770665645599365,
|
||
|
|
"logits/rejected": -3.198453187942505,
|
||
|
|
"logps/chosen": -91.16377258300781,
|
||
|
|
"logps/rejected": -101.21217346191406,
|
||
|
|
"loss": 0.5411731719970703,
|
||
|
|
"rewards/accuracies": 0.667187511920929,
|
||
|
|
"rewards/chosen": -0.7363663911819458,
|
||
|
|
"rewards/margins": 0.5946365594863892,
|
||
|
|
"rewards/rejected": -1.331002950668335,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8163265306122449,
|
||
|
|
"grad_norm": 0.6249299645423889,
|
||
|
|
"learning_rate": 0.00015018698578908005,
|
||
|
|
"logits/chosen": -3.2993998527526855,
|
||
|
|
"logits/rejected": -3.160282611846924,
|
||
|
|
"logps/chosen": -85.68218231201172,
|
||
|
|
"logps/rejected": -97.8273696899414,
|
||
|
|
"loss": 0.5476407051086426,
|
||
|
|
"rewards/accuracies": 0.66796875,
|
||
|
|
"rewards/chosen": -0.6863436102867126,
|
||
|
|
"rewards/margins": 0.6311149597167969,
|
||
|
|
"rewards/rejected": -1.3174583911895752,
|
||
|
|
"step": 750
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8272108843537415,
|
||
|
|
"grad_norm": 0.7027069926261902,
|
||
|
|
"learning_rate": 0.00014943904263275992,
|
||
|
|
"logits/chosen": -3.371492862701416,
|
||
|
|
"logits/rejected": -3.292865037918091,
|
||
|
|
"logps/chosen": -87.8956298828125,
|
||
|
|
"logps/rejected": -101.7688980102539,
|
||
|
|
"loss": 0.5395628929138183,
|
||
|
|
"rewards/accuracies": 0.6695312261581421,
|
||
|
|
"rewards/chosen": -0.8498029708862305,
|
||
|
|
"rewards/margins": 0.6512821912765503,
|
||
|
|
"rewards/rejected": -1.5010850429534912,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8380952380952381,
|
||
|
|
"grad_norm": 0.6294671893119812,
|
||
|
|
"learning_rate": 0.0001486910994764398,
|
||
|
|
"logits/chosen": -3.389573574066162,
|
||
|
|
"logits/rejected": -3.3465969562530518,
|
||
|
|
"logps/chosen": -89.55952453613281,
|
||
|
|
"logps/rejected": -102.7020263671875,
|
||
|
|
"loss": 0.5348502635955811,
|
||
|
|
"rewards/accuracies": 0.659375011920929,
|
||
|
|
"rewards/chosen": -1.0625698566436768,
|
||
|
|
"rewards/margins": 0.6796306371688843,
|
||
|
|
"rewards/rejected": -1.742200493812561,
|
||
|
|
"step": 770
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8489795918367347,
|
||
|
|
"grad_norm": 0.637783944606781,
|
||
|
|
"learning_rate": 0.00014794315632011968,
|
||
|
|
"logits/chosen": -3.226806163787842,
|
||
|
|
"logits/rejected": -3.1543614864349365,
|
||
|
|
"logps/chosen": -92.14598846435547,
|
||
|
|
"logps/rejected": -104.5304946899414,
|
||
|
|
"loss": 0.5414380073547364,
|
||
|
|
"rewards/accuracies": 0.667187511920929,
|
||
|
|
"rewards/chosen": -1.2139638662338257,
|
||
|
|
"rewards/margins": 0.6445013284683228,
|
||
|
|
"rewards/rejected": -1.8584649562835693,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8598639455782313,
|
||
|
|
"grad_norm": 0.7089376449584961,
|
||
|
|
"learning_rate": 0.00014719521316379955,
|
||
|
|
"logits/chosen": -3.0788469314575195,
|
||
|
|
"logits/rejected": -3.0586276054382324,
|
||
|
|
"logps/chosen": -92.21556854248047,
|
||
|
|
"logps/rejected": -102.19468688964844,
|
||
|
|
"loss": 0.5625462532043457,
|
||
|
|
"rewards/accuracies": 0.6351562738418579,
|
||
|
|
"rewards/chosen": -1.1437532901763916,
|
||
|
|
"rewards/margins": 0.5667166709899902,
|
||
|
|
"rewards/rejected": -1.7104698419570923,
|
||
|
|
"step": 790
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8707482993197279,
|
||
|
|
"grad_norm": 0.6719876527786255,
|
||
|
|
"learning_rate": 0.00014644727000747944,
|
||
|
|
"logits/chosen": -3.21879506111145,
|
||
|
|
"logits/rejected": -3.1802637577056885,
|
||
|
|
"logps/chosen": -87.95653533935547,
|
||
|
|
"logps/rejected": -99.78177642822266,
|
||
|
|
"loss": 0.5563519954681396,
|
||
|
|
"rewards/accuracies": 0.6585937738418579,
|
||
|
|
"rewards/chosen": -1.027986764907837,
|
||
|
|
"rewards/margins": 0.5857411623001099,
|
||
|
|
"rewards/rejected": -1.6137279272079468,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8816326530612245,
|
||
|
|
"grad_norm": 0.7437763810157776,
|
||
|
|
"learning_rate": 0.0001456993268511593,
|
||
|
|
"logits/chosen": -3.17462158203125,
|
||
|
|
"logits/rejected": -3.1289212703704834,
|
||
|
|
"logps/chosen": -90.2762680053711,
|
||
|
|
"logps/rejected": -101.78657531738281,
|
||
|
|
"loss": 0.5628280162811279,
|
||
|
|
"rewards/accuracies": 0.659375011920929,
|
||
|
|
"rewards/chosen": -1.0404772758483887,
|
||
|
|
"rewards/margins": 0.5995458364486694,
|
||
|
|
"rewards/rejected": -1.6400229930877686,
|
||
|
|
"step": 810
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8925170068027211,
|
||
|
|
"grad_norm": 0.7401530146598816,
|
||
|
|
"learning_rate": 0.0001449513836948392,
|
||
|
|
"logits/chosen": -3.1435704231262207,
|
||
|
|
"logits/rejected": -3.0372138023376465,
|
||
|
|
"logps/chosen": -92.34797668457031,
|
||
|
|
"logps/rejected": -103.2850341796875,
|
||
|
|
"loss": 0.5592099189758301,
|
||
|
|
"rewards/accuracies": 0.65625,
|
||
|
|
"rewards/chosen": -1.03983473777771,
|
||
|
|
"rewards/margins": 0.5651776194572449,
|
||
|
|
"rewards/rejected": -1.60501229763031,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9034013605442177,
|
||
|
|
"grad_norm": 0.6176671385765076,
|
||
|
|
"learning_rate": 0.00014420344053851908,
|
||
|
|
"logits/chosen": -3.2093448638916016,
|
||
|
|
"logits/rejected": -3.1419150829315186,
|
||
|
|
"logps/chosen": -92.94093322753906,
|
||
|
|
"logps/rejected": -102.44952392578125,
|
||
|
|
"loss": 0.5435313224792481,
|
||
|
|
"rewards/accuracies": 0.675000011920929,
|
||
|
|
"rewards/chosen": -0.9639943838119507,
|
||
|
|
"rewards/margins": 0.598076343536377,
|
||
|
|
"rewards/rejected": -1.562070608139038,
|
||
|
|
"step": 830
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9142857142857143,
|
||
|
|
"grad_norm": 0.6202688217163086,
|
||
|
|
"learning_rate": 0.00014345549738219897,
|
||
|
|
"logits/chosen": -3.3524563312530518,
|
||
|
|
"logits/rejected": -3.2346653938293457,
|
||
|
|
"logps/chosen": -91.24100494384766,
|
||
|
|
"logps/rejected": -101.37155151367188,
|
||
|
|
"loss": 0.5487663269042968,
|
||
|
|
"rewards/accuracies": 0.678906261920929,
|
||
|
|
"rewards/chosen": -0.9898284673690796,
|
||
|
|
"rewards/margins": 0.5904334783554077,
|
||
|
|
"rewards/rejected": -1.5802619457244873,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9251700680272109,
|
||
|
|
"grad_norm": 0.6689825057983398,
|
||
|
|
"learning_rate": 0.00014270755422587884,
|
||
|
|
"logits/chosen": -3.454129695892334,
|
||
|
|
"logits/rejected": -3.3540592193603516,
|
||
|
|
"logps/chosen": -89.66830444335938,
|
||
|
|
"logps/rejected": -102.6904525756836,
|
||
|
|
"loss": 0.5206645011901856,
|
||
|
|
"rewards/accuracies": 0.68359375,
|
||
|
|
"rewards/chosen": -0.9211218953132629,
|
||
|
|
"rewards/margins": 0.658340573310852,
|
||
|
|
"rewards/rejected": -1.5794624090194702,
|
||
|
|
"step": 850
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9360544217687075,
|
||
|
|
"grad_norm": 0.738223671913147,
|
||
|
|
"learning_rate": 0.00014195961106955874,
|
||
|
|
"logits/chosen": -3.5349831581115723,
|
||
|
|
"logits/rejected": -3.452519655227661,
|
||
|
|
"logps/chosen": -92.4320297241211,
|
||
|
|
"logps/rejected": -103.63383483886719,
|
||
|
|
"loss": 0.5230117797851562,
|
||
|
|
"rewards/accuracies": 0.6976562738418579,
|
||
|
|
"rewards/chosen": -0.9207289814949036,
|
||
|
|
"rewards/margins": 0.6930528879165649,
|
||
|
|
"rewards/rejected": -1.6137816905975342,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9469387755102041,
|
||
|
|
"grad_norm": 0.702201247215271,
|
||
|
|
"learning_rate": 0.0001412116679132386,
|
||
|
|
"logits/chosen": -3.439744234085083,
|
||
|
|
"logits/rejected": -3.3527960777282715,
|
||
|
|
"logps/chosen": -91.38029479980469,
|
||
|
|
"logps/rejected": -104.20161437988281,
|
||
|
|
"loss": 0.5320132732391357,
|
||
|
|
"rewards/accuracies": 0.6953125,
|
||
|
|
"rewards/chosen": -1.1446105241775513,
|
||
|
|
"rewards/margins": 0.6743196845054626,
|
||
|
|
"rewards/rejected": -1.8189302682876587,
|
||
|
|
"step": 870
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9578231292517007,
|
||
|
|
"grad_norm": 0.6528463363647461,
|
||
|
|
"learning_rate": 0.0001404637247569185,
|
||
|
|
"logits/chosen": -3.513301134109497,
|
||
|
|
"logits/rejected": -3.5274531841278076,
|
||
|
|
"logps/chosen": -90.98865509033203,
|
||
|
|
"logps/rejected": -103.4592056274414,
|
||
|
|
"loss": 0.5318188667297363,
|
||
|
|
"rewards/accuracies": 0.6773437261581421,
|
||
|
|
"rewards/chosen": -1.1643035411834717,
|
||
|
|
"rewards/margins": 0.6567374467849731,
|
||
|
|
"rewards/rejected": -1.8210411071777344,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9687074829931973,
|
||
|
|
"grad_norm": 0.6700842976570129,
|
||
|
|
"learning_rate": 0.00013971578160059837,
|
||
|
|
"logits/chosen": -3.6123263835906982,
|
||
|
|
"logits/rejected": -3.54679536819458,
|
||
|
|
"logps/chosen": -89.24981689453125,
|
||
|
|
"logps/rejected": -101.41873931884766,
|
||
|
|
"loss": 0.5319746017456055,
|
||
|
|
"rewards/accuracies": 0.678906261920929,
|
||
|
|
"rewards/chosen": -1.0091780424118042,
|
||
|
|
"rewards/margins": 0.707198977470398,
|
||
|
|
"rewards/rejected": -1.7163772583007812,
|
||
|
|
"step": 890
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9795918367346939,
|
||
|
|
"grad_norm": 0.6881840229034424,
|
||
|
|
"learning_rate": 0.00013896783844427824,
|
||
|
|
"logits/chosen": -3.528007984161377,
|
||
|
|
"logits/rejected": -3.411106824874878,
|
||
|
|
"logps/chosen": -88.0205078125,
|
||
|
|
"logps/rejected": -99.57227325439453,
|
||
|
|
"loss": 0.5377126693725586,
|
||
|
|
"rewards/accuracies": 0.680468738079071,
|
||
|
|
"rewards/chosen": -0.9363842010498047,
|
||
|
|
"rewards/margins": 0.6687086820602417,
|
||
|
|
"rewards/rejected": -1.605093002319336,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.9904761904761905,
|
||
|
|
"grad_norm": 0.6982113718986511,
|
||
|
|
"learning_rate": 0.0001382198952879581,
|
||
|
|
"logits/chosen": -3.544215440750122,
|
||
|
|
"logits/rejected": -3.499542236328125,
|
||
|
|
"logps/chosen": -87.49764251708984,
|
||
|
|
"logps/rejected": -98.82611083984375,
|
||
|
|
"loss": 0.5362007141113281,
|
||
|
|
"rewards/accuracies": 0.65234375,
|
||
|
|
"rewards/chosen": -0.8832570314407349,
|
||
|
|
"rewards/margins": 0.6539750695228577,
|
||
|
|
"rewards/rejected": -1.5372319221496582,
|
||
|
|
"step": 910
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0010884353741496,
|
||
|
|
"grad_norm": 0.49051275849342346,
|
||
|
|
"learning_rate": 0.000137471952131638,
|
||
|
|
"logits/chosen": -3.4829673767089844,
|
||
|
|
"logits/rejected": -3.455536365509033,
|
||
|
|
"logps/chosen": -86.75341796875,
|
||
|
|
"logps/rejected": -98.81475067138672,
|
||
|
|
"loss": 0.5034448623657226,
|
||
|
|
"rewards/accuracies": 0.7075320482254028,
|
||
|
|
"rewards/chosen": -0.9121702909469604,
|
||
|
|
"rewards/margins": 0.770575225353241,
|
||
|
|
"rewards/rejected": -1.6827455759048462,
|
||
|
|
"step": 920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0119727891156463,
|
||
|
|
"grad_norm": 0.5276588201522827,
|
||
|
|
"learning_rate": 0.00013672400897531787,
|
||
|
|
"logits/chosen": -3.5929553508758545,
|
||
|
|
"logits/rejected": -3.5475852489471436,
|
||
|
|
"logps/chosen": -92.64207458496094,
|
||
|
|
"logps/rejected": -107.84423828125,
|
||
|
|
"loss": 0.4122049808502197,
|
||
|
|
"rewards/accuracies": 0.7789062261581421,
|
||
|
|
"rewards/chosen": -1.0530273914337158,
|
||
|
|
"rewards/margins": 1.0623085498809814,
|
||
|
|
"rewards/rejected": -2.1153359413146973,
|
||
|
|
"step": 930
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.022857142857143,
|
||
|
|
"grad_norm": 0.5503054261207581,
|
||
|
|
"learning_rate": 0.00013597606581899777,
|
||
|
|
"logits/chosen": -3.6768276691436768,
|
||
|
|
"logits/rejected": -3.5878639221191406,
|
||
|
|
"logps/chosen": -94.07461547851562,
|
||
|
|
"logps/rejected": -112.07344055175781,
|
||
|
|
"loss": 0.40670342445373536,
|
||
|
|
"rewards/accuracies": 0.784375011920929,
|
||
|
|
"rewards/chosen": -1.2350085973739624,
|
||
|
|
"rewards/margins": 1.135968804359436,
|
||
|
|
"rewards/rejected": -2.3709776401519775,
|
||
|
|
"step": 940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0337414965986396,
|
||
|
|
"grad_norm": 0.5421497225761414,
|
||
|
|
"learning_rate": 0.00013522812266267764,
|
||
|
|
"logits/chosen": -3.7791919708251953,
|
||
|
|
"logits/rejected": -3.726738452911377,
|
||
|
|
"logps/chosen": -90.40796661376953,
|
||
|
|
"logps/rejected": -106.10577392578125,
|
||
|
|
"loss": 0.4037026882171631,
|
||
|
|
"rewards/accuracies": 0.784375011920929,
|
||
|
|
"rewards/chosen": -1.1298649311065674,
|
||
|
|
"rewards/margins": 1.157552719116211,
|
||
|
|
"rewards/rejected": -2.2874176502227783,
|
||
|
|
"step": 950
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.044625850340136,
|
||
|
|
"grad_norm": 0.5323552489280701,
|
||
|
|
"learning_rate": 0.0001344801795063575,
|
||
|
|
"logits/chosen": -3.955470323562622,
|
||
|
|
"logits/rejected": -3.9261558055877686,
|
||
|
|
"logps/chosen": -92.28099060058594,
|
||
|
|
"logps/rejected": -108.50651550292969,
|
||
|
|
"loss": 0.4047850608825684,
|
||
|
|
"rewards/accuracies": 0.793749988079071,
|
||
|
|
"rewards/chosen": -0.9677878618240356,
|
||
|
|
"rewards/margins": 1.1882990598678589,
|
||
|
|
"rewards/rejected": -2.1560869216918945,
|
||
|
|
"step": 960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0555102040816327,
|
||
|
|
"grad_norm": 0.4538813829421997,
|
||
|
|
"learning_rate": 0.0001337322363500374,
|
||
|
|
"logits/chosen": -3.7241110801696777,
|
||
|
|
"logits/rejected": -3.7324581146240234,
|
||
|
|
"logps/chosen": -90.46250915527344,
|
||
|
|
"logps/rejected": -109.62055969238281,
|
||
|
|
"loss": 0.39860632419586184,
|
||
|
|
"rewards/accuracies": 0.780468761920929,
|
||
|
|
"rewards/chosen": -0.9383746981620789,
|
||
|
|
"rewards/margins": 1.2237045764923096,
|
||
|
|
"rewards/rejected": -2.162079334259033,
|
||
|
|
"step": 970
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0663945578231293,
|
||
|
|
"grad_norm": 0.5154798030853271,
|
||
|
|
"learning_rate": 0.00013298429319371727,
|
||
|
|
"logits/chosen": -3.8078360557556152,
|
||
|
|
"logits/rejected": -3.7505202293395996,
|
||
|
|
"logps/chosen": -88.5647201538086,
|
||
|
|
"logps/rejected": -105.87337493896484,
|
||
|
|
"loss": 0.4086859703063965,
|
||
|
|
"rewards/accuracies": 0.7718750238418579,
|
||
|
|
"rewards/chosen": -1.0960875749588013,
|
||
|
|
"rewards/margins": 1.1370435953140259,
|
||
|
|
"rewards/rejected": -2.2331314086914062,
|
||
|
|
"step": 980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0772789115646257,
|
||
|
|
"grad_norm": 0.5626422166824341,
|
||
|
|
"learning_rate": 0.00013223635003739717,
|
||
|
|
"logits/chosen": -3.832609176635742,
|
||
|
|
"logits/rejected": -3.7331223487854004,
|
||
|
|
"logps/chosen": -93.8613052368164,
|
||
|
|
"logps/rejected": -112.10726165771484,
|
||
|
|
"loss": 0.38766965866088865,
|
||
|
|
"rewards/accuracies": 0.7945312261581421,
|
||
|
|
"rewards/chosen": -1.118674397468567,
|
||
|
|
"rewards/margins": 1.2444055080413818,
|
||
|
|
"rewards/rejected": -2.36307954788208,
|
||
|
|
"step": 990
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0881632653061224,
|
||
|
|
"grad_norm": 0.4980772137641907,
|
||
|
|
"learning_rate": 0.00013148840688107703,
|
||
|
|
"logits/chosen": -3.9342827796936035,
|
||
|
|
"logits/rejected": -3.891047239303589,
|
||
|
|
"logps/chosen": -90.54890441894531,
|
||
|
|
"logps/rejected": -110.37210845947266,
|
||
|
|
"loss": 0.38568527698516847,
|
||
|
|
"rewards/accuracies": 0.784375011920929,
|
||
|
|
"rewards/chosen": -1.2535765171051025,
|
||
|
|
"rewards/margins": 1.2768959999084473,
|
||
|
|
"rewards/rejected": -2.5304722785949707,
|
||
|
|
"step": 1000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.0881632653061224,
|
||
|
|
"eval_logits/chosen": -4.035802364349365,
|
||
|
|
"eval_logits/rejected": -3.9840340614318848,
|
||
|
|
"eval_logps/chosen": -94.60836791992188,
|
||
|
|
"eval_logps/rejected": -109.67314910888672,
|
||
|
|
"eval_loss": 0.5218217968940735,
|
||
|
|
"eval_rewards/accuracies": 0.6712499856948853,
|
||
|
|
"eval_rewards/chosen": -1.5003221035003662,
|
||
|
|
"eval_rewards/margins": 0.8820583820343018,
|
||
|
|
"eval_rewards/rejected": -2.382380723953247,
|
||
|
|
"eval_runtime": 104.4812,
|
||
|
|
"eval_samples_per_second": 22.971,
|
||
|
|
"eval_steps_per_second": 2.871,
|
||
|
|
"step": 1000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.099047619047619,
|
||
|
|
"grad_norm": 0.5595287084579468,
|
||
|
|
"learning_rate": 0.00013074046372475693,
|
||
|
|
"logits/chosen": -3.9862780570983887,
|
||
|
|
"logits/rejected": -3.910135269165039,
|
||
|
|
"logps/chosen": -99.29423522949219,
|
||
|
|
"logps/rejected": -118.37043762207031,
|
||
|
|
"loss": 0.382628059387207,
|
||
|
|
"rewards/accuracies": 0.7914062738418579,
|
||
|
|
"rewards/chosen": -1.4844170808792114,
|
||
|
|
"rewards/margins": 1.2958667278289795,
|
||
|
|
"rewards/rejected": -2.7802836894989014,
|
||
|
|
"step": 1010
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1099319727891157,
|
||
|
|
"grad_norm": 0.5150955319404602,
|
||
|
|
"learning_rate": 0.0001299925205684368,
|
||
|
|
"logits/chosen": -3.9361672401428223,
|
||
|
|
"logits/rejected": -3.924912929534912,
|
||
|
|
"logps/chosen": -95.98111724853516,
|
||
|
|
"logps/rejected": -114.6279525756836,
|
||
|
|
"loss": 0.37689783573150637,
|
||
|
|
"rewards/accuracies": 0.785937488079071,
|
||
|
|
"rewards/chosen": -1.5440380573272705,
|
||
|
|
"rewards/margins": 1.3521003723144531,
|
||
|
|
"rewards/rejected": -2.8961384296417236,
|
||
|
|
"step": 1020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1208163265306124,
|
||
|
|
"grad_norm": 0.5913591384887695,
|
||
|
|
"learning_rate": 0.0001292445774121167,
|
||
|
|
"logits/chosen": -3.943892002105713,
|
||
|
|
"logits/rejected": -3.976665496826172,
|
||
|
|
"logps/chosen": -95.18277740478516,
|
||
|
|
"logps/rejected": -113.27921295166016,
|
||
|
|
"loss": 0.39126296043395997,
|
||
|
|
"rewards/accuracies": 0.776562511920929,
|
||
|
|
"rewards/chosen": -1.530815839767456,
|
||
|
|
"rewards/margins": 1.3069045543670654,
|
||
|
|
"rewards/rejected": -2.8377201557159424,
|
||
|
|
"step": 1030
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1317006802721088,
|
||
|
|
"grad_norm": 0.6089534163475037,
|
||
|
|
"learning_rate": 0.00012849663425579656,
|
||
|
|
"logits/chosen": -3.9231574535369873,
|
||
|
|
"logits/rejected": -3.8793838024139404,
|
||
|
|
"logps/chosen": -91.93929290771484,
|
||
|
|
"logps/rejected": -109.3221664428711,
|
||
|
|
"loss": 0.4064298629760742,
|
||
|
|
"rewards/accuracies": 0.7562500238418579,
|
||
|
|
"rewards/chosen": -1.3621362447738647,
|
||
|
|
"rewards/margins": 1.2190048694610596,
|
||
|
|
"rewards/rejected": -2.5811409950256348,
|
||
|
|
"step": 1040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1425850340136054,
|
||
|
|
"grad_norm": 0.5580401420593262,
|
||
|
|
"learning_rate": 0.00012774869109947646,
|
||
|
|
"logits/chosen": -4.0801682472229,
|
||
|
|
"logits/rejected": -4.014742851257324,
|
||
|
|
"logps/chosen": -93.5271987915039,
|
||
|
|
"logps/rejected": -110.57737731933594,
|
||
|
|
"loss": 0.39088118076324463,
|
||
|
|
"rewards/accuracies": 0.778124988079071,
|
||
|
|
"rewards/chosen": -1.1289399862289429,
|
||
|
|
"rewards/margins": 1.294062852859497,
|
||
|
|
"rewards/rejected": -2.4230027198791504,
|
||
|
|
"step": 1050
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.153469387755102,
|
||
|
|
"grad_norm": 0.6687185764312744,
|
||
|
|
"learning_rate": 0.00012700074794315633,
|
||
|
|
"logits/chosen": -3.948697566986084,
|
||
|
|
"logits/rejected": -3.8599441051483154,
|
||
|
|
"logps/chosen": -92.7010269165039,
|
||
|
|
"logps/rejected": -109.04603576660156,
|
||
|
|
"loss": 0.4010897159576416,
|
||
|
|
"rewards/accuracies": 0.76953125,
|
||
|
|
"rewards/chosen": -1.2695072889328003,
|
||
|
|
"rewards/margins": 1.2557579278945923,
|
||
|
|
"rewards/rejected": -2.5252652168273926,
|
||
|
|
"step": 1060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1643537414965985,
|
||
|
|
"grad_norm": 0.5772605538368225,
|
||
|
|
"learning_rate": 0.00012625280478683622,
|
||
|
|
"logits/chosen": -4.051478385925293,
|
||
|
|
"logits/rejected": -3.9212899208068848,
|
||
|
|
"logps/chosen": -93.09245300292969,
|
||
|
|
"logps/rejected": -112.09034729003906,
|
||
|
|
"loss": 0.3885282754898071,
|
||
|
|
"rewards/accuracies": 0.776562511920929,
|
||
|
|
"rewards/chosen": -1.3699064254760742,
|
||
|
|
"rewards/margins": 1.302438497543335,
|
||
|
|
"rewards/rejected": -2.67234468460083,
|
||
|
|
"step": 1070
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1752380952380952,
|
||
|
|
"grad_norm": 0.55010986328125,
|
||
|
|
"learning_rate": 0.0001255048616305161,
|
||
|
|
"logits/chosen": -3.901043653488159,
|
||
|
|
"logits/rejected": -3.7977395057678223,
|
||
|
|
"logps/chosen": -93.72676086425781,
|
||
|
|
"logps/rejected": -112.05916595458984,
|
||
|
|
"loss": 0.39031736850738524,
|
||
|
|
"rewards/accuracies": 0.7796875238418579,
|
||
|
|
"rewards/chosen": -1.4006497859954834,
|
||
|
|
"rewards/margins": 1.2986645698547363,
|
||
|
|
"rewards/rejected": -2.6993141174316406,
|
||
|
|
"step": 1080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1861224489795918,
|
||
|
|
"grad_norm": 0.6019951105117798,
|
||
|
|
"learning_rate": 0.00012475691847419596,
|
||
|
|
"logits/chosen": -3.874293804168701,
|
||
|
|
"logits/rejected": -3.8425419330596924,
|
||
|
|
"logps/chosen": -97.39969635009766,
|
||
|
|
"logps/rejected": -116.4852523803711,
|
||
|
|
"loss": 0.38108861446380615,
|
||
|
|
"rewards/accuracies": 0.785937488079071,
|
||
|
|
"rewards/chosen": -1.5423808097839355,
|
||
|
|
"rewards/margins": 1.3624523878097534,
|
||
|
|
"rewards/rejected": -2.9048333168029785,
|
||
|
|
"step": 1090
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.1970068027210885,
|
||
|
|
"grad_norm": 0.5465772151947021,
|
||
|
|
"learning_rate": 0.00012400897531787586,
|
||
|
|
"logits/chosen": -3.8221359252929688,
|
||
|
|
"logits/rejected": -3.7892353534698486,
|
||
|
|
"logps/chosen": -97.63330841064453,
|
||
|
|
"logps/rejected": -116.29536437988281,
|
||
|
|
"loss": 0.3859058141708374,
|
||
|
|
"rewards/accuracies": 0.7867187261581421,
|
||
|
|
"rewards/chosen": -1.561934232711792,
|
||
|
|
"rewards/margins": 1.3092113733291626,
|
||
|
|
"rewards/rejected": -2.871145248413086,
|
||
|
|
"step": 1100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2078911564625852,
|
||
|
|
"grad_norm": 0.6201313138008118,
|
||
|
|
"learning_rate": 0.00012326103216155572,
|
||
|
|
"logits/chosen": -3.8689208030700684,
|
||
|
|
"logits/rejected": -3.814204454421997,
|
||
|
|
"logps/chosen": -94.79856872558594,
|
||
|
|
"logps/rejected": -114.0688705444336,
|
||
|
|
"loss": 0.38736424446105955,
|
||
|
|
"rewards/accuracies": 0.780468761920929,
|
||
|
|
"rewards/chosen": -1.537530541419983,
|
||
|
|
"rewards/margins": 1.3583099842071533,
|
||
|
|
"rewards/rejected": -2.895840644836426,
|
||
|
|
"step": 1110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2187755102040816,
|
||
|
|
"grad_norm": 0.6293015480041504,
|
||
|
|
"learning_rate": 0.00012251308900523562,
|
||
|
|
"logits/chosen": -3.8243889808654785,
|
||
|
|
"logits/rejected": -3.680662155151367,
|
||
|
|
"logps/chosen": -98.95996856689453,
|
||
|
|
"logps/rejected": -118.1916732788086,
|
||
|
|
"loss": 0.3828889846801758,
|
||
|
|
"rewards/accuracies": 0.782031238079071,
|
||
|
|
"rewards/chosen": -1.7024767398834229,
|
||
|
|
"rewards/margins": 1.3618117570877075,
|
||
|
|
"rewards/rejected": -3.06428861618042,
|
||
|
|
"step": 1120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2296598639455782,
|
||
|
|
"grad_norm": 0.6542093753814697,
|
||
|
|
"learning_rate": 0.0001217651458489155,
|
||
|
|
"logits/chosen": -3.806445360183716,
|
||
|
|
"logits/rejected": -3.8042194843292236,
|
||
|
|
"logps/chosen": -95.04216003417969,
|
||
|
|
"logps/rejected": -113.69169616699219,
|
||
|
|
"loss": 0.38822317123413086,
|
||
|
|
"rewards/accuracies": 0.7710937261581421,
|
||
|
|
"rewards/chosen": -1.6071590185165405,
|
||
|
|
"rewards/margins": 1.3052054643630981,
|
||
|
|
"rewards/rejected": -2.9123644828796387,
|
||
|
|
"step": 1130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.240544217687075,
|
||
|
|
"grad_norm": 0.6824989914894104,
|
||
|
|
"learning_rate": 0.00012101720269259537,
|
||
|
|
"logits/chosen": -3.831465482711792,
|
||
|
|
"logits/rejected": -3.731153964996338,
|
||
|
|
"logps/chosen": -97.02737426757812,
|
||
|
|
"logps/rejected": -115.9981918334961,
|
||
|
|
"loss": 0.39344158172607424,
|
||
|
|
"rewards/accuracies": 0.7796875238418579,
|
||
|
|
"rewards/chosen": -1.642228364944458,
|
||
|
|
"rewards/margins": 1.3141920566558838,
|
||
|
|
"rewards/rejected": -2.956420421600342,
|
||
|
|
"step": 1140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2514285714285713,
|
||
|
|
"grad_norm": 0.5602055788040161,
|
||
|
|
"learning_rate": 0.00012026925953627525,
|
||
|
|
"logits/chosen": -3.8642430305480957,
|
||
|
|
"logits/rejected": -3.8064823150634766,
|
||
|
|
"logps/chosen": -96.62242126464844,
|
||
|
|
"logps/rejected": -116.4171142578125,
|
||
|
|
"loss": 0.38083562850952146,
|
||
|
|
"rewards/accuracies": 0.789843738079071,
|
||
|
|
"rewards/chosen": -1.6083332300186157,
|
||
|
|
"rewards/margins": 1.389948844909668,
|
||
|
|
"rewards/rejected": -2.998281717300415,
|
||
|
|
"step": 1150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.262312925170068,
|
||
|
|
"grad_norm": 0.5861300826072693,
|
||
|
|
"learning_rate": 0.00011952131637995514,
|
||
|
|
"logits/chosen": -3.833646297454834,
|
||
|
|
"logits/rejected": -3.7338168621063232,
|
||
|
|
"logps/chosen": -95.55216979980469,
|
||
|
|
"logps/rejected": -114.51301574707031,
|
||
|
|
"loss": 0.3848297595977783,
|
||
|
|
"rewards/accuracies": 0.7749999761581421,
|
||
|
|
"rewards/chosen": -1.6490459442138672,
|
||
|
|
"rewards/margins": 1.3717120885849,
|
||
|
|
"rewards/rejected": -3.0207581520080566,
|
||
|
|
"step": 1160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2731972789115646,
|
||
|
|
"grad_norm": 0.5589214563369751,
|
||
|
|
"learning_rate": 0.00011877337322363502,
|
||
|
|
"logits/chosen": -3.80780291557312,
|
||
|
|
"logits/rejected": -3.658127546310425,
|
||
|
|
"logps/chosen": -98.92552185058594,
|
||
|
|
"logps/rejected": -120.0743408203125,
|
||
|
|
"loss": 0.3743234872817993,
|
||
|
|
"rewards/accuracies": 0.788281261920929,
|
||
|
|
"rewards/chosen": -1.8888273239135742,
|
||
|
|
"rewards/margins": 1.4256237745285034,
|
||
|
|
"rewards/rejected": -3.314451217651367,
|
||
|
|
"step": 1170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.2840816326530613,
|
||
|
|
"grad_norm": 0.773446798324585,
|
||
|
|
"learning_rate": 0.00011802543006731489,
|
||
|
|
"logits/chosen": -3.791332244873047,
|
||
|
|
"logits/rejected": -3.7509467601776123,
|
||
|
|
"logps/chosen": -100.43743133544922,
|
||
|
|
"logps/rejected": -119.4636001586914,
|
||
|
|
"loss": 0.3916522979736328,
|
||
|
|
"rewards/accuracies": 0.772656261920929,
|
||
|
|
"rewards/chosen": -2.132657766342163,
|
||
|
|
"rewards/margins": 1.3497222661972046,
|
||
|
|
"rewards/rejected": -3.482379913330078,
|
||
|
|
"step": 1180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.294965986394558,
|
||
|
|
"grad_norm": 0.6507579684257507,
|
||
|
|
"learning_rate": 0.00011727748691099475,
|
||
|
|
"logits/chosen": -3.631664752960205,
|
||
|
|
"logits/rejected": -3.559692859649658,
|
||
|
|
"logps/chosen": -102.70137786865234,
|
||
|
|
"logps/rejected": -119.88555908203125,
|
||
|
|
"loss": 0.38732936382293703,
|
||
|
|
"rewards/accuracies": 0.780468761920929,
|
||
|
|
"rewards/chosen": -1.9864223003387451,
|
||
|
|
"rewards/margins": 1.3899860382080078,
|
||
|
|
"rewards/rejected": -3.376408338546753,
|
||
|
|
"step": 1190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3058503401360544,
|
||
|
|
"grad_norm": 0.5761600136756897,
|
||
|
|
"learning_rate": 0.00011652954375467464,
|
||
|
|
"logits/chosen": -3.7357120513916016,
|
||
|
|
"logits/rejected": -3.7192249298095703,
|
||
|
|
"logps/chosen": -96.95130157470703,
|
||
|
|
"logps/rejected": -118.63916015625,
|
||
|
|
"loss": 0.386475658416748,
|
||
|
|
"rewards/accuracies": 0.7789062261581421,
|
||
|
|
"rewards/chosen": -1.7698255777359009,
|
||
|
|
"rewards/margins": 1.3770931959152222,
|
||
|
|
"rewards/rejected": -3.146918773651123,
|
||
|
|
"step": 1200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.316734693877551,
|
||
|
|
"grad_norm": 0.751353919506073,
|
||
|
|
"learning_rate": 0.00011578160059835452,
|
||
|
|
"logits/chosen": -3.761479139328003,
|
||
|
|
"logits/rejected": -3.714494228363037,
|
||
|
|
"logps/chosen": -99.30330657958984,
|
||
|
|
"logps/rejected": -118.73576354980469,
|
||
|
|
"loss": 0.3787353515625,
|
||
|
|
"rewards/accuracies": 0.789843738079071,
|
||
|
|
"rewards/chosen": -1.8722871541976929,
|
||
|
|
"rewards/margins": 1.4351270198822021,
|
||
|
|
"rewards/rejected": -3.3074138164520264,
|
||
|
|
"step": 1210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3276190476190477,
|
||
|
|
"grad_norm": 0.6535031795501709,
|
||
|
|
"learning_rate": 0.0001150336574420344,
|
||
|
|
"logits/chosen": -3.6952037811279297,
|
||
|
|
"logits/rejected": -3.615670680999756,
|
||
|
|
"logps/chosen": -100.49695587158203,
|
||
|
|
"logps/rejected": -121.04229736328125,
|
||
|
|
"loss": 0.3696281433105469,
|
||
|
|
"rewards/accuracies": 0.7945312261581421,
|
||
|
|
"rewards/chosen": -1.8464534282684326,
|
||
|
|
"rewards/margins": 1.4216703176498413,
|
||
|
|
"rewards/rejected": -3.2681241035461426,
|
||
|
|
"step": 1220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3385034013605441,
|
||
|
|
"grad_norm": 0.660273551940918,
|
||
|
|
"learning_rate": 0.00011428571428571428,
|
||
|
|
"logits/chosen": -3.7871832847595215,
|
||
|
|
"logits/rejected": -3.731839418411255,
|
||
|
|
"logps/chosen": -98.5318374633789,
|
||
|
|
"logps/rejected": -115.12672424316406,
|
||
|
|
"loss": 0.3887288808822632,
|
||
|
|
"rewards/accuracies": 0.7828124761581421,
|
||
|
|
"rewards/chosen": -1.8460729122161865,
|
||
|
|
"rewards/margins": 1.3180320262908936,
|
||
|
|
"rewards/rejected": -3.164105176925659,
|
||
|
|
"step": 1230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3493877551020408,
|
||
|
|
"grad_norm": 0.7036870121955872,
|
||
|
|
"learning_rate": 0.00011353777112939417,
|
||
|
|
"logits/chosen": -3.7227275371551514,
|
||
|
|
"logits/rejected": -3.703014373779297,
|
||
|
|
"logps/chosen": -96.31185913085938,
|
||
|
|
"logps/rejected": -115.09098815917969,
|
||
|
|
"loss": 0.3810297966003418,
|
||
|
|
"rewards/accuracies": 0.7875000238418579,
|
||
|
|
"rewards/chosen": -1.6640751361846924,
|
||
|
|
"rewards/margins": 1.3640429973602295,
|
||
|
|
"rewards/rejected": -3.028118133544922,
|
||
|
|
"step": 1240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3602721088435374,
|
||
|
|
"grad_norm": 0.6380351185798645,
|
||
|
|
"learning_rate": 0.00011278982797307405,
|
||
|
|
"logits/chosen": -3.743776798248291,
|
||
|
|
"logits/rejected": -3.590733051300049,
|
||
|
|
"logps/chosen": -95.80562591552734,
|
||
|
|
"logps/rejected": -114.26639556884766,
|
||
|
|
"loss": 0.38879597187042236,
|
||
|
|
"rewards/accuracies": 0.776562511920929,
|
||
|
|
"rewards/chosen": -1.5717710256576538,
|
||
|
|
"rewards/margins": 1.3705636262893677,
|
||
|
|
"rewards/rejected": -2.9423346519470215,
|
||
|
|
"step": 1250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3711564625850339,
|
||
|
|
"grad_norm": 0.695436418056488,
|
||
|
|
"learning_rate": 0.00011204188481675393,
|
||
|
|
"logits/chosen": -3.7618496417999268,
|
||
|
|
"logits/rejected": -3.742903470993042,
|
||
|
|
"logps/chosen": -94.9074478149414,
|
||
|
|
"logps/rejected": -116.57875061035156,
|
||
|
|
"loss": 0.3822426080703735,
|
||
|
|
"rewards/accuracies": 0.776562511920929,
|
||
|
|
"rewards/chosen": -1.6461149454116821,
|
||
|
|
"rewards/margins": 1.4086042642593384,
|
||
|
|
"rewards/rejected": -3.0547192096710205,
|
||
|
|
"step": 1260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3820408163265305,
|
||
|
|
"grad_norm": 0.5786954760551453,
|
||
|
|
"learning_rate": 0.00011129394166043381,
|
||
|
|
"logits/chosen": -3.881277561187744,
|
||
|
|
"logits/rejected": -3.810837984085083,
|
||
|
|
"logps/chosen": -98.8532485961914,
|
||
|
|
"logps/rejected": -118.10380554199219,
|
||
|
|
"loss": 0.3603384256362915,
|
||
|
|
"rewards/accuracies": 0.8046875,
|
||
|
|
"rewards/chosen": -1.8184385299682617,
|
||
|
|
"rewards/margins": 1.4062011241912842,
|
||
|
|
"rewards/rejected": -3.224639415740967,
|
||
|
|
"step": 1270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.3929251700680272,
|
||
|
|
"grad_norm": 0.6679598093032837,
|
||
|
|
"learning_rate": 0.0001105459985041137,
|
||
|
|
"logits/chosen": -3.8195717334747314,
|
||
|
|
"logits/rejected": -3.790808916091919,
|
||
|
|
"logps/chosen": -100.193603515625,
|
||
|
|
"logps/rejected": -120.28971862792969,
|
||
|
|
"loss": 0.366161584854126,
|
||
|
|
"rewards/accuracies": 0.803906261920929,
|
||
|
|
"rewards/chosen": -1.7882047891616821,
|
||
|
|
"rewards/margins": 1.4713773727416992,
|
||
|
|
"rewards/rejected": -3.259582042694092,
|
||
|
|
"step": 1280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4038095238095238,
|
||
|
|
"grad_norm": 0.6280708909034729,
|
||
|
|
"learning_rate": 0.00010979805534779358,
|
||
|
|
"logits/chosen": -3.7300808429718018,
|
||
|
|
"logits/rejected": -3.7090256214141846,
|
||
|
|
"logps/chosen": -95.59440612792969,
|
||
|
|
"logps/rejected": -115.55784606933594,
|
||
|
|
"loss": 0.38781962394714353,
|
||
|
|
"rewards/accuracies": 0.770312488079071,
|
||
|
|
"rewards/chosen": -1.5936332941055298,
|
||
|
|
"rewards/margins": 1.418022871017456,
|
||
|
|
"rewards/rejected": -3.0116562843322754,
|
||
|
|
"step": 1290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4146938775510205,
|
||
|
|
"grad_norm": 0.6393949389457703,
|
||
|
|
"learning_rate": 0.00010905011219147346,
|
||
|
|
"logits/chosen": -3.6777634620666504,
|
||
|
|
"logits/rejected": -3.60345721244812,
|
||
|
|
"logps/chosen": -97.09107971191406,
|
||
|
|
"logps/rejected": -115.95845031738281,
|
||
|
|
"loss": 0.37133514881134033,
|
||
|
|
"rewards/accuracies": 0.776562511920929,
|
||
|
|
"rewards/chosen": -1.5525085926055908,
|
||
|
|
"rewards/margins": 1.4391489028930664,
|
||
|
|
"rewards/rejected": -2.9916577339172363,
|
||
|
|
"step": 1300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.425578231292517,
|
||
|
|
"grad_norm": 0.5887162685394287,
|
||
|
|
"learning_rate": 0.00010830216903515333,
|
||
|
|
"logits/chosen": -3.858353853225708,
|
||
|
|
"logits/rejected": -3.729213237762451,
|
||
|
|
"logps/chosen": -97.08758544921875,
|
||
|
|
"logps/rejected": -115.01194763183594,
|
||
|
|
"loss": 0.3837103366851807,
|
||
|
|
"rewards/accuracies": 0.77734375,
|
||
|
|
"rewards/chosen": -1.7705045938491821,
|
||
|
|
"rewards/margins": 1.3879071474075317,
|
||
|
|
"rewards/rejected": -3.158411741256714,
|
||
|
|
"step": 1310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4364625850340136,
|
||
|
|
"grad_norm": 0.6474682092666626,
|
||
|
|
"learning_rate": 0.00010755422587883321,
|
||
|
|
"logits/chosen": -3.9399871826171875,
|
||
|
|
"logits/rejected": -3.7817294597625732,
|
||
|
|
"logps/chosen": -97.05113220214844,
|
||
|
|
"logps/rejected": -119.55680847167969,
|
||
|
|
"loss": 0.36365351676940916,
|
||
|
|
"rewards/accuracies": 0.788281261920929,
|
||
|
|
"rewards/chosen": -1.7540229558944702,
|
||
|
|
"rewards/margins": 1.5094449520111084,
|
||
|
|
"rewards/rejected": -3.263467788696289,
|
||
|
|
"step": 1320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4473469387755102,
|
||
|
|
"grad_norm": 0.6645624041557312,
|
||
|
|
"learning_rate": 0.00010680628272251309,
|
||
|
|
"logits/chosen": -3.977292537689209,
|
||
|
|
"logits/rejected": -3.8931617736816406,
|
||
|
|
"logps/chosen": -96.40562438964844,
|
||
|
|
"logps/rejected": -117.29931640625,
|
||
|
|
"loss": 0.37579851150512694,
|
||
|
|
"rewards/accuracies": 0.7718750238418579,
|
||
|
|
"rewards/chosen": -1.6829277276992798,
|
||
|
|
"rewards/margins": 1.5337533950805664,
|
||
|
|
"rewards/rejected": -3.2166812419891357,
|
||
|
|
"step": 1330
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4582312925170067,
|
||
|
|
"grad_norm": 0.795894980430603,
|
||
|
|
"learning_rate": 0.00010605833956619297,
|
||
|
|
"logits/chosen": -4.0191144943237305,
|
||
|
|
"logits/rejected": -3.899752140045166,
|
||
|
|
"logps/chosen": -92.3825454711914,
|
||
|
|
"logps/rejected": -112.3360595703125,
|
||
|
|
"loss": 0.3910695552825928,
|
||
|
|
"rewards/accuracies": 0.7671874761581421,
|
||
|
|
"rewards/chosen": -1.5036238431930542,
|
||
|
|
"rewards/margins": 1.455990195274353,
|
||
|
|
"rewards/rejected": -2.9596142768859863,
|
||
|
|
"step": 1340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4691156462585033,
|
||
|
|
"grad_norm": 0.6805703639984131,
|
||
|
|
"learning_rate": 0.00010531039640987286,
|
||
|
|
"logits/chosen": -3.859619140625,
|
||
|
|
"logits/rejected": -3.820143938064575,
|
||
|
|
"logps/chosen": -101.65718841552734,
|
||
|
|
"logps/rejected": -121.03543853759766,
|
||
|
|
"loss": 0.38420472145080564,
|
||
|
|
"rewards/accuracies": 0.7757812738418579,
|
||
|
|
"rewards/chosen": -1.6729732751846313,
|
||
|
|
"rewards/margins": 1.4384580850601196,
|
||
|
|
"rewards/rejected": -3.111431121826172,
|
||
|
|
"step": 1350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.48,
|
||
|
|
"grad_norm": 0.6531292200088501,
|
||
|
|
"learning_rate": 0.00010456245325355274,
|
||
|
|
"logits/chosen": -3.826378583908081,
|
||
|
|
"logits/rejected": -3.772704601287842,
|
||
|
|
"logps/chosen": -100.1290283203125,
|
||
|
|
"logps/rejected": -119.37181091308594,
|
||
|
|
"loss": 0.3683862447738647,
|
||
|
|
"rewards/accuracies": 0.78515625,
|
||
|
|
"rewards/chosen": -1.9099397659301758,
|
||
|
|
"rewards/margins": 1.4697914123535156,
|
||
|
|
"rewards/rejected": -3.3797316551208496,
|
||
|
|
"step": 1360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.4908843537414966,
|
||
|
|
"grad_norm": 0.6555562019348145,
|
||
|
|
"learning_rate": 0.00010381451009723262,
|
||
|
|
"logits/chosen": -3.857220411300659,
|
||
|
|
"logits/rejected": -3.754575252532959,
|
||
|
|
"logps/chosen": -101.08953857421875,
|
||
|
|
"logps/rejected": -120.2709732055664,
|
||
|
|
"loss": 0.3801293849945068,
|
||
|
|
"rewards/accuracies": 0.764843761920929,
|
||
|
|
"rewards/chosen": -1.9699108600616455,
|
||
|
|
"rewards/margins": 1.38763427734375,
|
||
|
|
"rewards/rejected": -3.3575451374053955,
|
||
|
|
"step": 1370
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5017687074829933,
|
||
|
|
"grad_norm": 0.5829383134841919,
|
||
|
|
"learning_rate": 0.0001030665669409125,
|
||
|
|
"logits/chosen": -3.9333407878875732,
|
||
|
|
"logits/rejected": -3.8222217559814453,
|
||
|
|
"logps/chosen": -99.89224243164062,
|
||
|
|
"logps/rejected": -118.7002182006836,
|
||
|
|
"loss": 0.3699865102767944,
|
||
|
|
"rewards/accuracies": 0.8031250238418579,
|
||
|
|
"rewards/chosen": -1.854455590248108,
|
||
|
|
"rewards/margins": 1.431877851486206,
|
||
|
|
"rewards/rejected": -3.2863335609436035,
|
||
|
|
"step": 1380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5126530612244897,
|
||
|
|
"grad_norm": 0.7183417677879333,
|
||
|
|
"learning_rate": 0.00010231862378459238,
|
||
|
|
"logits/chosen": -4.135873317718506,
|
||
|
|
"logits/rejected": -3.9824626445770264,
|
||
|
|
"logps/chosen": -97.94303131103516,
|
||
|
|
"logps/rejected": -117.84385681152344,
|
||
|
|
"loss": 0.373444128036499,
|
||
|
|
"rewards/accuracies": 0.7828124761581421,
|
||
|
|
"rewards/chosen": -1.8929665088653564,
|
||
|
|
"rewards/margins": 1.4810130596160889,
|
||
|
|
"rewards/rejected": -3.373979091644287,
|
||
|
|
"step": 1390
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5235374149659864,
|
||
|
|
"grad_norm": 0.62285315990448,
|
||
|
|
"learning_rate": 0.00010157068062827227,
|
||
|
|
"logits/chosen": -3.943370819091797,
|
||
|
|
"logits/rejected": -3.885420322418213,
|
||
|
|
"logps/chosen": -101.32877349853516,
|
||
|
|
"logps/rejected": -122.38797760009766,
|
||
|
|
"loss": 0.36237530708312987,
|
||
|
|
"rewards/accuracies": 0.7875000238418579,
|
||
|
|
"rewards/chosen": -1.6976028680801392,
|
||
|
|
"rewards/margins": 1.5130655765533447,
|
||
|
|
"rewards/rejected": -3.2106685638427734,
|
||
|
|
"step": 1400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.534421768707483,
|
||
|
|
"grad_norm": 0.7117959856987,
|
||
|
|
"learning_rate": 0.00010082273747195215,
|
||
|
|
"logits/chosen": -4.146048545837402,
|
||
|
|
"logits/rejected": -4.0635809898376465,
|
||
|
|
"logps/chosen": -96.91676330566406,
|
||
|
|
"logps/rejected": -115.88706970214844,
|
||
|
|
"loss": 0.3611806631088257,
|
||
|
|
"rewards/accuracies": 0.796093761920929,
|
||
|
|
"rewards/chosen": -1.7968066930770874,
|
||
|
|
"rewards/margins": 1.4590308666229248,
|
||
|
|
"rewards/rejected": -3.2558376789093018,
|
||
|
|
"step": 1410
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5453061224489795,
|
||
|
|
"grad_norm": 0.4945203363895416,
|
||
|
|
"learning_rate": 0.00010007479431563203,
|
||
|
|
"logits/chosen": -4.0039167404174805,
|
||
|
|
"logits/rejected": -3.9553489685058594,
|
||
|
|
"logps/chosen": -100.09791564941406,
|
||
|
|
"logps/rejected": -120.74732971191406,
|
||
|
|
"loss": 0.37725100517272947,
|
||
|
|
"rewards/accuracies": 0.780468761920929,
|
||
|
|
"rewards/chosen": -1.9454562664031982,
|
||
|
|
"rewards/margins": 1.4613468647003174,
|
||
|
|
"rewards/rejected": -3.406803607940674,
|
||
|
|
"step": 1420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5561904761904763,
|
||
|
|
"grad_norm": 0.5278945565223694,
|
||
|
|
"learning_rate": 9.93268511593119e-05,
|
||
|
|
"logits/chosen": -3.986499309539795,
|
||
|
|
"logits/rejected": -3.932950496673584,
|
||
|
|
"logps/chosen": -102.49642181396484,
|
||
|
|
"logps/rejected": -122.0504150390625,
|
||
|
|
"loss": 0.37192845344543457,
|
||
|
|
"rewards/accuracies": 0.7796875238418579,
|
||
|
|
"rewards/chosen": -1.9946882724761963,
|
||
|
|
"rewards/margins": 1.5072121620178223,
|
||
|
|
"rewards/rejected": -3.5019004344940186,
|
||
|
|
"step": 1430
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5670748299319728,
|
||
|
|
"grad_norm": 0.5255491733551025,
|
||
|
|
"learning_rate": 9.857890800299178e-05,
|
||
|
|
"logits/chosen": -3.868673801422119,
|
||
|
|
"logits/rejected": -3.8117432594299316,
|
||
|
|
"logps/chosen": -100.22129821777344,
|
||
|
|
"logps/rejected": -119.8470687866211,
|
||
|
|
"loss": 0.374340295791626,
|
||
|
|
"rewards/accuracies": 0.7890625,
|
||
|
|
"rewards/chosen": -1.920660376548767,
|
||
|
|
"rewards/margins": 1.4606765508651733,
|
||
|
|
"rewards/rejected": -3.3813369274139404,
|
||
|
|
"step": 1440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5779591836734694,
|
||
|
|
"grad_norm": 0.6175586581230164,
|
||
|
|
"learning_rate": 9.783096484667166e-05,
|
||
|
|
"logits/chosen": -3.869438886642456,
|
||
|
|
"logits/rejected": -3.7423954010009766,
|
||
|
|
"logps/chosen": -101.46055603027344,
|
||
|
|
"logps/rejected": -121.7767562866211,
|
||
|
|
"loss": 0.3565861940383911,
|
||
|
|
"rewards/accuracies": 0.800000011920929,
|
||
|
|
"rewards/chosen": -1.973406195640564,
|
||
|
|
"rewards/margins": 1.5132112503051758,
|
||
|
|
"rewards/rejected": -3.48661732673645,
|
||
|
|
"step": 1450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.588843537414966,
|
||
|
|
"grad_norm": 0.6701238751411438,
|
||
|
|
"learning_rate": 9.708302169035153e-05,
|
||
|
|
"logits/chosen": -3.9192538261413574,
|
||
|
|
"logits/rejected": -3.899573564529419,
|
||
|
|
"logps/chosen": -97.5078353881836,
|
||
|
|
"logps/rejected": -118.8314437866211,
|
||
|
|
"loss": 0.3684267997741699,
|
||
|
|
"rewards/accuracies": 0.778124988079071,
|
||
|
|
"rewards/chosen": -2.0173895359039307,
|
||
|
|
"rewards/margins": 1.507660150527954,
|
||
|
|
"rewards/rejected": -3.525049924850464,
|
||
|
|
"step": 1460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.5997278911564625,
|
||
|
|
"grad_norm": 0.8422195315361023,
|
||
|
|
"learning_rate": 9.633507853403142e-05,
|
||
|
|
"logits/chosen": -3.8154900074005127,
|
||
|
|
"logits/rejected": -3.763354539871216,
|
||
|
|
"logps/chosen": -103.52520751953125,
|
||
|
|
"logps/rejected": -125.2503890991211,
|
||
|
|
"loss": 0.36898369789123536,
|
||
|
|
"rewards/accuracies": 0.7835937738418579,
|
||
|
|
"rewards/chosen": -1.952511191368103,
|
||
|
|
"rewards/margins": 1.5541565418243408,
|
||
|
|
"rewards/rejected": -3.5066676139831543,
|
||
|
|
"step": 1470
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6106122448979592,
|
||
|
|
"grad_norm": 0.6260126233100891,
|
||
|
|
"learning_rate": 9.55871353777113e-05,
|
||
|
|
"logits/chosen": -3.938748836517334,
|
||
|
|
"logits/rejected": -3.8893561363220215,
|
||
|
|
"logps/chosen": -98.54754638671875,
|
||
|
|
"logps/rejected": -118.55876159667969,
|
||
|
|
"loss": 0.378691291809082,
|
||
|
|
"rewards/accuracies": 0.788281261920929,
|
||
|
|
"rewards/chosen": -1.7820552587509155,
|
||
|
|
"rewards/margins": 1.4857076406478882,
|
||
|
|
"rewards/rejected": -3.2677626609802246,
|
||
|
|
"step": 1480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6214965986394558,
|
||
|
|
"grad_norm": 0.6023070216178894,
|
||
|
|
"learning_rate": 9.483919222139118e-05,
|
||
|
|
"logits/chosen": -4.026276111602783,
|
||
|
|
"logits/rejected": -3.953202724456787,
|
||
|
|
"logps/chosen": -96.39505004882812,
|
||
|
|
"logps/rejected": -115.8733901977539,
|
||
|
|
"loss": 0.36678736209869384,
|
||
|
|
"rewards/accuracies": 0.78125,
|
||
|
|
"rewards/chosen": -1.5815566778182983,
|
||
|
|
"rewards/margins": 1.527267336845398,
|
||
|
|
"rewards/rejected": -3.1088242530822754,
|
||
|
|
"step": 1490
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6323809523809523,
|
||
|
|
"grad_norm": 0.5291011929512024,
|
||
|
|
"learning_rate": 9.409124906507106e-05,
|
||
|
|
"logits/chosen": -3.9902701377868652,
|
||
|
|
"logits/rejected": -3.968461513519287,
|
||
|
|
"logps/chosen": -96.85718536376953,
|
||
|
|
"logps/rejected": -118.74107360839844,
|
||
|
|
"loss": 0.3673092365264893,
|
||
|
|
"rewards/accuracies": 0.789843738079071,
|
||
|
|
"rewards/chosen": -1.714505910873413,
|
||
|
|
"rewards/margins": 1.5418702363967896,
|
||
|
|
"rewards/rejected": -3.256376266479492,
|
||
|
|
"step": 1500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6323809523809523,
|
||
|
|
"eval_logits/chosen": -4.040825366973877,
|
||
|
|
"eval_logits/rejected": -3.988185167312622,
|
||
|
|
"eval_logps/chosen": -98.07389068603516,
|
||
|
|
"eval_logps/rejected": -115.78838348388672,
|
||
|
|
"eval_loss": 0.47849762439727783,
|
||
|
|
"eval_rewards/accuracies": 0.7079166769981384,
|
||
|
|
"eval_rewards/chosen": -1.8468737602233887,
|
||
|
|
"eval_rewards/margins": 1.1470292806625366,
|
||
|
|
"eval_rewards/rejected": -2.993903160095215,
|
||
|
|
"eval_runtime": 106.0205,
|
||
|
|
"eval_samples_per_second": 22.637,
|
||
|
|
"eval_steps_per_second": 2.83,
|
||
|
|
"step": 1500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6432653061224491,
|
||
|
|
"grad_norm": 0.6328744292259216,
|
||
|
|
"learning_rate": 9.334330590875094e-05,
|
||
|
|
"logits/chosen": -4.012321949005127,
|
||
|
|
"logits/rejected": -3.944819927215576,
|
||
|
|
"logps/chosen": -97.4483642578125,
|
||
|
|
"logps/rejected": -118.50297546386719,
|
||
|
|
"loss": 0.3724888801574707,
|
||
|
|
"rewards/accuracies": 0.7749999761581421,
|
||
|
|
"rewards/chosen": -1.79526686668396,
|
||
|
|
"rewards/margins": 1.4999581575393677,
|
||
|
|
"rewards/rejected": -3.2952256202697754,
|
||
|
|
"step": 1510
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6541496598639456,
|
||
|
|
"grad_norm": 0.8266810178756714,
|
||
|
|
"learning_rate": 9.259536275243081e-05,
|
||
|
|
"logits/chosen": -4.003256797790527,
|
||
|
|
"logits/rejected": -3.9533722400665283,
|
||
|
|
"logps/chosen": -99.8465347290039,
|
||
|
|
"logps/rejected": -118.92730712890625,
|
||
|
|
"loss": 0.3884738922119141,
|
||
|
|
"rewards/accuracies": 0.776562511920929,
|
||
|
|
"rewards/chosen": -2.056798219680786,
|
||
|
|
"rewards/margins": 1.452993392944336,
|
||
|
|
"rewards/rejected": -3.509791612625122,
|
||
|
|
"step": 1520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6650340136054422,
|
||
|
|
"grad_norm": 0.6178652048110962,
|
||
|
|
"learning_rate": 9.18474195961107e-05,
|
||
|
|
"logits/chosen": -3.8090343475341797,
|
||
|
|
"logits/rejected": -3.734511613845825,
|
||
|
|
"logps/chosen": -100.30364990234375,
|
||
|
|
"logps/rejected": -121.00141906738281,
|
||
|
|
"loss": 0.36191649436950685,
|
||
|
|
"rewards/accuracies": 0.796875,
|
||
|
|
"rewards/chosen": -1.9928449392318726,
|
||
|
|
"rewards/margins": 1.523863673210144,
|
||
|
|
"rewards/rejected": -3.5167088508605957,
|
||
|
|
"step": 1530
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6759183673469389,
|
||
|
|
"grad_norm": 0.6442583799362183,
|
||
|
|
"learning_rate": 9.109947643979058e-05,
|
||
|
|
"logits/chosen": -3.819338321685791,
|
||
|
|
"logits/rejected": -3.7492592334747314,
|
||
|
|
"logps/chosen": -103.84706115722656,
|
||
|
|
"logps/rejected": -123.3454360961914,
|
||
|
|
"loss": 0.3575705289840698,
|
||
|
|
"rewards/accuracies": 0.793749988079071,
|
||
|
|
"rewards/chosen": -2.11049222946167,
|
||
|
|
"rewards/margins": 1.5337401628494263,
|
||
|
|
"rewards/rejected": -3.6442322731018066,
|
||
|
|
"step": 1540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.6868027210884353,
|
||
|
|
"grad_norm": 0.7383544445037842,
|
||
|
|
"learning_rate": 9.035153328347046e-05,
|
||
|
|
"logits/chosen": -3.8853554725646973,
|
||
|
|
"logits/rejected": -3.7873711585998535,
|
||
|
|
"logps/chosen": -103.81562805175781,
|
||
|
|
"logps/rejected": -122.89927673339844,
|
||
|
|
"loss": 0.37395687103271485,
|
||
|
|
"rewards/accuracies": 0.7679687738418579,
|
||
|
|
"rewards/chosen": -2.210036516189575,
|
||
|
|
"rewards/margins": 1.5190045833587646,
|
||
|
|
"rewards/rejected": -3.729041337966919,
|
||
|
|
"step": 1550
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.697687074829932,
|
||
|
|
"grad_norm": 0.5653782486915588,
|
||
|
|
"learning_rate": 8.960359012715034e-05,
|
||
|
|
"logits/chosen": -3.935488224029541,
|
||
|
|
"logits/rejected": -3.8435440063476562,
|
||
|
|
"logps/chosen": -100.53340148925781,
|
||
|
|
"logps/rejected": -120.791259765625,
|
||
|
|
"loss": 0.3584398031234741,
|
||
|
|
"rewards/accuracies": 0.78515625,
|
||
|
|
"rewards/chosen": -2.0043153762817383,
|
||
|
|
"rewards/margins": 1.5743240118026733,
|
||
|
|
"rewards/rejected": -3.578639268875122,
|
||
|
|
"step": 1560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7085714285714286,
|
||
|
|
"grad_norm": 0.8251403570175171,
|
||
|
|
"learning_rate": 8.885564697083022e-05,
|
||
|
|
"logits/chosen": -4.113719940185547,
|
||
|
|
"logits/rejected": -4.015976905822754,
|
||
|
|
"logps/chosen": -102.41825866699219,
|
||
|
|
"logps/rejected": -124.1919937133789,
|
||
|
|
"loss": 0.35360021591186525,
|
||
|
|
"rewards/accuracies": 0.8062499761581421,
|
||
|
|
"rewards/chosen": -2.1747488975524902,
|
||
|
|
"rewards/margins": 1.5947285890579224,
|
||
|
|
"rewards/rejected": -3.7694778442382812,
|
||
|
|
"step": 1570
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.719455782312925,
|
||
|
|
"grad_norm": 0.7069426774978638,
|
||
|
|
"learning_rate": 8.81077038145101e-05,
|
||
|
|
"logits/chosen": -4.1289448738098145,
|
||
|
|
"logits/rejected": -4.012777805328369,
|
||
|
|
"logps/chosen": -100.58708953857422,
|
||
|
|
"logps/rejected": -121.4145736694336,
|
||
|
|
"loss": 0.3635096549987793,
|
||
|
|
"rewards/accuracies": 0.7835937738418579,
|
||
|
|
"rewards/chosen": -2.133143424987793,
|
||
|
|
"rewards/margins": 1.5998557806015015,
|
||
|
|
"rewards/rejected": -3.732999324798584,
|
||
|
|
"step": 1580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7303401360544217,
|
||
|
|
"grad_norm": 0.8968989253044128,
|
||
|
|
"learning_rate": 8.735976065818999e-05,
|
||
|
|
"logits/chosen": -4.026947021484375,
|
||
|
|
"logits/rejected": -4.019469738006592,
|
||
|
|
"logps/chosen": -101.74604797363281,
|
||
|
|
"logps/rejected": -124.91915130615234,
|
||
|
|
"loss": 0.37527787685394287,
|
||
|
|
"rewards/accuracies": 0.778124988079071,
|
||
|
|
"rewards/chosen": -2.15088152885437,
|
||
|
|
"rewards/margins": 1.591313123703003,
|
||
|
|
"rewards/rejected": -3.742194652557373,
|
||
|
|
"step": 1590
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7412244897959184,
|
||
|
|
"grad_norm": 0.7351986169815063,
|
||
|
|
"learning_rate": 8.661181750186986e-05,
|
||
|
|
"logits/chosen": -3.981262683868408,
|
||
|
|
"logits/rejected": -3.917276382446289,
|
||
|
|
"logps/chosen": -103.64302825927734,
|
||
|
|
"logps/rejected": -124.09883880615234,
|
||
|
|
"loss": 0.37657463550567627,
|
||
|
|
"rewards/accuracies": 0.774218738079071,
|
||
|
|
"rewards/chosen": -2.378594160079956,
|
||
|
|
"rewards/margins": 1.506073236465454,
|
||
|
|
"rewards/rejected": -3.8846676349639893,
|
||
|
|
"step": 1600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7521088435374148,
|
||
|
|
"grad_norm": 0.5647110342979431,
|
||
|
|
"learning_rate": 8.586387434554974e-05,
|
||
|
|
"logits/chosen": -4.083320140838623,
|
||
|
|
"logits/rejected": -4.027539253234863,
|
||
|
|
"logps/chosen": -103.57328796386719,
|
||
|
|
"logps/rejected": -125.90059661865234,
|
||
|
|
"loss": 0.36530237197875975,
|
||
|
|
"rewards/accuracies": 0.784375011920929,
|
||
|
|
"rewards/chosen": -2.352818250656128,
|
||
|
|
"rewards/margins": 1.5950738191604614,
|
||
|
|
"rewards/rejected": -3.9478919506073,
|
||
|
|
"step": 1610
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7629931972789117,
|
||
|
|
"grad_norm": 0.7188398838043213,
|
||
|
|
"learning_rate": 8.511593118922962e-05,
|
||
|
|
"logits/chosen": -3.8719964027404785,
|
||
|
|
"logits/rejected": -3.833092212677002,
|
||
|
|
"logps/chosen": -104.86869812011719,
|
||
|
|
"logps/rejected": -124.95475006103516,
|
||
|
|
"loss": 0.36907384395599363,
|
||
|
|
"rewards/accuracies": 0.780468761920929,
|
||
|
|
"rewards/chosen": -2.335792064666748,
|
||
|
|
"rewards/margins": 1.5145189762115479,
|
||
|
|
"rewards/rejected": -3.850311279296875,
|
||
|
|
"step": 1620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7738775510204081,
|
||
|
|
"grad_norm": 0.6679319739341736,
|
||
|
|
"learning_rate": 8.43679880329095e-05,
|
||
|
|
"logits/chosen": -4.030211448669434,
|
||
|
|
"logits/rejected": -3.9728660583496094,
|
||
|
|
"logps/chosen": -103.6771011352539,
|
||
|
|
"logps/rejected": -124.9799575805664,
|
||
|
|
"loss": 0.3572399377822876,
|
||
|
|
"rewards/accuracies": 0.7945312261581421,
|
||
|
|
"rewards/chosen": -2.2580087184906006,
|
||
|
|
"rewards/margins": 1.571171522140503,
|
||
|
|
"rewards/rejected": -3.829180955886841,
|
||
|
|
"step": 1630
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7847619047619048,
|
||
|
|
"grad_norm": 0.6013081073760986,
|
||
|
|
"learning_rate": 8.362004487658939e-05,
|
||
|
|
"logits/chosen": -3.995105028152466,
|
||
|
|
"logits/rejected": -3.910709857940674,
|
||
|
|
"logps/chosen": -100.53841400146484,
|
||
|
|
"logps/rejected": -123.62651062011719,
|
||
|
|
"loss": 0.3458747625350952,
|
||
|
|
"rewards/accuracies": 0.8023437261581421,
|
||
|
|
"rewards/chosen": -2.2358031272888184,
|
||
|
|
"rewards/margins": 1.6609306335449219,
|
||
|
|
"rewards/rejected": -3.8967342376708984,
|
||
|
|
"step": 1640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.7956462585034014,
|
||
|
|
"grad_norm": 0.6696369051933289,
|
||
|
|
"learning_rate": 8.287210172026927e-05,
|
||
|
|
"logits/chosen": -3.9745001792907715,
|
||
|
|
"logits/rejected": -3.900158405303955,
|
||
|
|
"logps/chosen": -100.8124771118164,
|
||
|
|
"logps/rejected": -123.26814270019531,
|
||
|
|
"loss": 0.3555989027023315,
|
||
|
|
"rewards/accuracies": 0.79296875,
|
||
|
|
"rewards/chosen": -2.186405658721924,
|
||
|
|
"rewards/margins": 1.6339943408966064,
|
||
|
|
"rewards/rejected": -3.8204002380371094,
|
||
|
|
"step": 1650
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8065306122448979,
|
||
|
|
"grad_norm": 0.5695591568946838,
|
||
|
|
"learning_rate": 8.212415856394914e-05,
|
||
|
|
"logits/chosen": -4.0766448974609375,
|
||
|
|
"logits/rejected": -3.9804439544677734,
|
||
|
|
"logps/chosen": -99.67530822753906,
|
||
|
|
"logps/rejected": -120.61580657958984,
|
||
|
|
"loss": 0.3668109893798828,
|
||
|
|
"rewards/accuracies": 0.7890625,
|
||
|
|
"rewards/chosen": -2.070260524749756,
|
||
|
|
"rewards/margins": 1.568867802619934,
|
||
|
|
"rewards/rejected": -3.6391282081604004,
|
||
|
|
"step": 1660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8174149659863945,
|
||
|
|
"grad_norm": 0.6718661189079285,
|
||
|
|
"learning_rate": 8.137621540762902e-05,
|
||
|
|
"logits/chosen": -4.0458664894104,
|
||
|
|
"logits/rejected": -3.9342434406280518,
|
||
|
|
"logps/chosen": -103.21232604980469,
|
||
|
|
"logps/rejected": -124.14998626708984,
|
||
|
|
"loss": 0.3770064115524292,
|
||
|
|
"rewards/accuracies": 0.789843738079071,
|
||
|
|
"rewards/chosen": -2.1097495555877686,
|
||
|
|
"rewards/margins": 1.5439164638519287,
|
||
|
|
"rewards/rejected": -3.6536660194396973,
|
||
|
|
"step": 1670
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8282993197278912,
|
||
|
|
"grad_norm": 0.6736993193626404,
|
||
|
|
"learning_rate": 8.06282722513089e-05,
|
||
|
|
"logits/chosen": -4.07711124420166,
|
||
|
|
"logits/rejected": -4.0070013999938965,
|
||
|
|
"logps/chosen": -99.71659088134766,
|
||
|
|
"logps/rejected": -121.86555480957031,
|
||
|
|
"loss": 0.3560856580734253,
|
||
|
|
"rewards/accuracies": 0.7890625,
|
||
|
|
"rewards/chosen": -2.129600763320923,
|
||
|
|
"rewards/margins": 1.6453748941421509,
|
||
|
|
"rewards/rejected": -3.774975538253784,
|
||
|
|
"step": 1680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8391836734693876,
|
||
|
|
"grad_norm": 0.7004987597465515,
|
||
|
|
"learning_rate": 7.988032909498878e-05,
|
||
|
|
"logits/chosen": -3.9158096313476562,
|
||
|
|
"logits/rejected": -3.911224365234375,
|
||
|
|
"logps/chosen": -105.80472564697266,
|
||
|
|
"logps/rejected": -128.0396728515625,
|
||
|
|
"loss": 0.35362706184387205,
|
||
|
|
"rewards/accuracies": 0.8031250238418579,
|
||
|
|
"rewards/chosen": -2.293348789215088,
|
||
|
|
"rewards/margins": 1.6550325155258179,
|
||
|
|
"rewards/rejected": -3.9483814239501953,
|
||
|
|
"step": 1690
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8500680272108845,
|
||
|
|
"grad_norm": 0.5832870006561279,
|
||
|
|
"learning_rate": 7.913238593866867e-05,
|
||
|
|
"logits/chosen": -4.208107948303223,
|
||
|
|
"logits/rejected": -4.131274223327637,
|
||
|
|
"logps/chosen": -98.68458557128906,
|
||
|
|
"logps/rejected": -118.3831558227539,
|
||
|
|
"loss": 0.3670835256576538,
|
||
|
|
"rewards/accuracies": 0.760937511920929,
|
||
|
|
"rewards/chosen": -2.237819194793701,
|
||
|
|
"rewards/margins": 1.6039073467254639,
|
||
|
|
"rewards/rejected": -3.841726779937744,
|
||
|
|
"step": 1700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.860952380952381,
|
||
|
|
"grad_norm": 0.5559285879135132,
|
||
|
|
"learning_rate": 7.838444278234855e-05,
|
||
|
|
"logits/chosen": -4.1811747550964355,
|
||
|
|
"logits/rejected": -4.1075592041015625,
|
||
|
|
"logps/chosen": -99.44391632080078,
|
||
|
|
"logps/rejected": -122.75013732910156,
|
||
|
|
"loss": 0.3504934787750244,
|
||
|
|
"rewards/accuracies": 0.809374988079071,
|
||
|
|
"rewards/chosen": -2.2351126670837402,
|
||
|
|
"rewards/margins": 1.6141564846038818,
|
||
|
|
"rewards/rejected": -3.849269151687622,
|
||
|
|
"step": 1710
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8718367346938776,
|
||
|
|
"grad_norm": 0.5495217442512512,
|
||
|
|
"learning_rate": 7.763649962602843e-05,
|
||
|
|
"logits/chosen": -4.15755558013916,
|
||
|
|
"logits/rejected": -4.096221923828125,
|
||
|
|
"logps/chosen": -104.17137145996094,
|
||
|
|
"logps/rejected": -125.32877349853516,
|
||
|
|
"loss": 0.3607916355133057,
|
||
|
|
"rewards/accuracies": 0.7828124761581421,
|
||
|
|
"rewards/chosen": -2.3636012077331543,
|
||
|
|
"rewards/margins": 1.6084495782852173,
|
||
|
|
"rewards/rejected": -3.972050428390503,
|
||
|
|
"step": 1720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8827210884353742,
|
||
|
|
"grad_norm": 0.7562318444252014,
|
||
|
|
"learning_rate": 7.688855646970831e-05,
|
||
|
|
"logits/chosen": -4.053821563720703,
|
||
|
|
"logits/rejected": -3.9341418743133545,
|
||
|
|
"logps/chosen": -102.71726989746094,
|
||
|
|
"logps/rejected": -123.45503234863281,
|
||
|
|
"loss": 0.3550391674041748,
|
||
|
|
"rewards/accuracies": 0.7906249761581421,
|
||
|
|
"rewards/chosen": -2.264005422592163,
|
||
|
|
"rewards/margins": 1.6550270318984985,
|
||
|
|
"rewards/rejected": -3.919032573699951,
|
||
|
|
"step": 1730
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.8936054421768707,
|
||
|
|
"grad_norm": 0.6610931158065796,
|
||
|
|
"learning_rate": 7.61406133133882e-05,
|
||
|
|
"logits/chosen": -3.961141586303711,
|
||
|
|
"logits/rejected": -3.830481767654419,
|
||
|
|
"logps/chosen": -104.4319839477539,
|
||
|
|
"logps/rejected": -125.9029541015625,
|
||
|
|
"loss": 0.3494965314865112,
|
||
|
|
"rewards/accuracies": 0.7953125238418579,
|
||
|
|
"rewards/chosen": -2.1706981658935547,
|
||
|
|
"rewards/margins": 1.6067132949829102,
|
||
|
|
"rewards/rejected": -3.777411699295044,
|
||
|
|
"step": 1740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9044897959183673,
|
||
|
|
"grad_norm": 0.6473342776298523,
|
||
|
|
"learning_rate": 7.539267015706806e-05,
|
||
|
|
"logits/chosen": -3.913191556930542,
|
||
|
|
"logits/rejected": -3.8687243461608887,
|
||
|
|
"logps/chosen": -105.96980285644531,
|
||
|
|
"logps/rejected": -125.6346206665039,
|
||
|
|
"loss": 0.3611732482910156,
|
||
|
|
"rewards/accuracies": 0.7835937738418579,
|
||
|
|
"rewards/chosen": -2.0681533813476562,
|
||
|
|
"rewards/margins": 1.6147505044937134,
|
||
|
|
"rewards/rejected": -3.68290376663208,
|
||
|
|
"step": 1750
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.915374149659864,
|
||
|
|
"grad_norm": 0.7833436727523804,
|
||
|
|
"learning_rate": 7.464472700074794e-05,
|
||
|
|
"logits/chosen": -3.938861846923828,
|
||
|
|
"logits/rejected": -3.9061336517333984,
|
||
|
|
"logps/chosen": -97.59529876708984,
|
||
|
|
"logps/rejected": -119.07645416259766,
|
||
|
|
"loss": 0.36034407615661623,
|
||
|
|
"rewards/accuracies": 0.780468761920929,
|
||
|
|
"rewards/chosen": -1.8530477285385132,
|
||
|
|
"rewards/margins": 1.5701138973236084,
|
||
|
|
"rewards/rejected": -3.423161745071411,
|
||
|
|
"step": 1760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9262585034013604,
|
||
|
|
"grad_norm": 0.7200015783309937,
|
||
|
|
"learning_rate": 7.389678384442783e-05,
|
||
|
|
"logits/chosen": -3.8537850379943848,
|
||
|
|
"logits/rejected": -3.8197879791259766,
|
||
|
|
"logps/chosen": -104.39387512207031,
|
||
|
|
"logps/rejected": -122.6333236694336,
|
||
|
|
"loss": 0.3665923118591309,
|
||
|
|
"rewards/accuracies": 0.78125,
|
||
|
|
"rewards/chosen": -2.3217196464538574,
|
||
|
|
"rewards/margins": 1.548337697982788,
|
||
|
|
"rewards/rejected": -3.8700573444366455,
|
||
|
|
"step": 1770
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9371428571428573,
|
||
|
|
"grad_norm": 0.6586691737174988,
|
||
|
|
"learning_rate": 7.314884068810771e-05,
|
||
|
|
"logits/chosen": -3.951854705810547,
|
||
|
|
"logits/rejected": -4.002626419067383,
|
||
|
|
"logps/chosen": -106.01944732666016,
|
||
|
|
"logps/rejected": -127.8281021118164,
|
||
|
|
"loss": 0.36367824077606203,
|
||
|
|
"rewards/accuracies": 0.780468761920929,
|
||
|
|
"rewards/chosen": -2.4460396766662598,
|
||
|
|
"rewards/margins": 1.604691743850708,
|
||
|
|
"rewards/rejected": -4.050731658935547,
|
||
|
|
"step": 1780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9480272108843537,
|
||
|
|
"grad_norm": 0.5734717845916748,
|
||
|
|
"learning_rate": 7.240089753178759e-05,
|
||
|
|
"logits/chosen": -3.999734878540039,
|
||
|
|
"logits/rejected": -3.9233245849609375,
|
||
|
|
"logps/chosen": -100.71134948730469,
|
||
|
|
"logps/rejected": -121.8692626953125,
|
||
|
|
"loss": 0.3612796783447266,
|
||
|
|
"rewards/accuracies": 0.78125,
|
||
|
|
"rewards/chosen": -2.5755791664123535,
|
||
|
|
"rewards/margins": 1.5662591457366943,
|
||
|
|
"rewards/rejected": -4.141838550567627,
|
||
|
|
"step": 1790
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9589115646258504,
|
||
|
|
"grad_norm": 0.6537356376647949,
|
||
|
|
"learning_rate": 7.165295437546746e-05,
|
||
|
|
"logits/chosen": -3.9834160804748535,
|
||
|
|
"logits/rejected": -3.872994899749756,
|
||
|
|
"logps/chosen": -103.37413024902344,
|
||
|
|
"logps/rejected": -125.7213134765625,
|
||
|
|
"loss": 0.3468764781951904,
|
||
|
|
"rewards/accuracies": 0.819531261920929,
|
||
|
|
"rewards/chosen": -2.4130101203918457,
|
||
|
|
"rewards/margins": 1.6939111948013306,
|
||
|
|
"rewards/rejected": -4.106921195983887,
|
||
|
|
"step": 1800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.969795918367347,
|
||
|
|
"grad_norm": 0.6989219784736633,
|
||
|
|
"learning_rate": 7.090501121914734e-05,
|
||
|
|
"logits/chosen": -3.91550874710083,
|
||
|
|
"logits/rejected": -3.8702545166015625,
|
||
|
|
"logps/chosen": -106.2231674194336,
|
||
|
|
"logps/rejected": -126.7488784790039,
|
||
|
|
"loss": 0.36261188983917236,
|
||
|
|
"rewards/accuracies": 0.7945312261581421,
|
||
|
|
"rewards/chosen": -2.3560307025909424,
|
||
|
|
"rewards/margins": 1.6367266178131104,
|
||
|
|
"rewards/rejected": -3.9927573204040527,
|
||
|
|
"step": 1810
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.9806802721088435,
|
||
|
|
"grad_norm": 0.7157489657402039,
|
||
|
|
"learning_rate": 7.015706806282722e-05,
|
||
|
|
"logits/chosen": -3.9314968585968018,
|
||
|
|
"logits/rejected": -3.872746229171753,
|
||
|
|
"logps/chosen": -100.2810287475586,
|
||
|
|
"logps/rejected": -121.5164794921875,
|
||
|
|
"loss": 0.35974826812744143,
|
||
|
|
"rewards/accuracies": 0.778124988079071,
|
||
|
|
"rewards/chosen": -2.0922586917877197,
|
||
|
|
"rewards/margins": 1.629852533340454,
|
||
|
|
"rewards/rejected": -3.722111225128174,
|
||
|
|
"step": 1820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 1.99156462585034,
|
||
|
|
"grad_norm": 0.6526369452476501,
|
||
|
|
"learning_rate": 6.94091249065071e-05,
|
||
|
|
"logits/chosen": -3.94775652885437,
|
||
|
|
"logits/rejected": -3.8687806129455566,
|
||
|
|
"logps/chosen": -99.9976577758789,
|
||
|
|
"logps/rejected": -121.957763671875,
|
||
|
|
"loss": 0.3599375247955322,
|
||
|
|
"rewards/accuracies": 0.7945312261581421,
|
||
|
|
"rewards/chosen": -1.9912713766098022,
|
||
|
|
"rewards/margins": 1.6352510452270508,
|
||
|
|
"rewards/rejected": -3.6265225410461426,
|
||
|
|
"step": 1830
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.002176870748299,
|
||
|
|
"grad_norm": 0.49071577191352844,
|
||
|
|
"learning_rate": 6.866118175018699e-05,
|
||
|
|
"logits/chosen": -3.9457576274871826,
|
||
|
|
"logits/rejected": -3.855695962905884,
|
||
|
|
"logps/chosen": -103.30020904541016,
|
||
|
|
"logps/rejected": -127.69253540039062,
|
||
|
|
"loss": 0.32846436500549314,
|
||
|
|
"rewards/accuracies": 0.8221153616905212,
|
||
|
|
"rewards/chosen": -2.0800564289093018,
|
||
|
|
"rewards/margins": 1.746621012687683,
|
||
|
|
"rewards/rejected": -3.8266773223876953,
|
||
|
|
"step": 1840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.013061224489796,
|
||
|
|
"grad_norm": 0.5088359713554382,
|
||
|
|
"learning_rate": 6.791323859386687e-05,
|
||
|
|
"logits/chosen": -3.9414725303649902,
|
||
|
|
"logits/rejected": -3.9251086711883545,
|
||
|
|
"logps/chosen": -101.10342407226562,
|
||
|
|
"logps/rejected": -126.2572250366211,
|
||
|
|
"loss": 0.279949951171875,
|
||
|
|
"rewards/accuracies": 0.839062511920929,
|
||
|
|
"rewards/chosen": -2.04860258102417,
|
||
|
|
"rewards/margins": 1.9327900409698486,
|
||
|
|
"rewards/rejected": -3.9813923835754395,
|
||
|
|
"step": 1850
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0239455782312925,
|
||
|
|
"grad_norm": 0.5052030682563782,
|
||
|
|
"learning_rate": 6.716529543754675e-05,
|
||
|
|
"logits/chosen": -4.173396110534668,
|
||
|
|
"logits/rejected": -4.081063270568848,
|
||
|
|
"logps/chosen": -104.18453216552734,
|
||
|
|
"logps/rejected": -128.5143280029297,
|
||
|
|
"loss": 0.26949272155761717,
|
||
|
|
"rewards/accuracies": 0.858593761920929,
|
||
|
|
"rewards/chosen": -2.204545259475708,
|
||
|
|
"rewards/margins": 1.9530636072158813,
|
||
|
|
"rewards/rejected": -4.157608985900879,
|
||
|
|
"step": 1860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.034829931972789,
|
||
|
|
"grad_norm": 0.4783055782318115,
|
||
|
|
"learning_rate": 6.641735228122664e-05,
|
||
|
|
"logits/chosen": -4.18683385848999,
|
||
|
|
"logits/rejected": -4.042752265930176,
|
||
|
|
"logps/chosen": -104.73225402832031,
|
||
|
|
"logps/rejected": -129.3142852783203,
|
||
|
|
"loss": 0.2710374116897583,
|
||
|
|
"rewards/accuracies": 0.85546875,
|
||
|
|
"rewards/chosen": -2.1699516773223877,
|
||
|
|
"rewards/margins": 2.064490795135498,
|
||
|
|
"rewards/rejected": -4.234442234039307,
|
||
|
|
"step": 1870
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.045714285714286,
|
||
|
|
"grad_norm": 0.5429951548576355,
|
||
|
|
"learning_rate": 6.566940912490652e-05,
|
||
|
|
"logits/chosen": -4.225846290588379,
|
||
|
|
"logits/rejected": -4.173259735107422,
|
||
|
|
"logps/chosen": -102.9245834350586,
|
||
|
|
"logps/rejected": -129.71678161621094,
|
||
|
|
"loss": 0.2762880563735962,
|
||
|
|
"rewards/accuracies": 0.844531238079071,
|
||
|
|
"rewards/chosen": -2.279546022415161,
|
||
|
|
"rewards/margins": 2.050327777862549,
|
||
|
|
"rewards/rejected": -4.329874038696289,
|
||
|
|
"step": 1880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0565986394557823,
|
||
|
|
"grad_norm": 0.5694318413734436,
|
||
|
|
"learning_rate": 6.492146596858639e-05,
|
||
|
|
"logits/chosen": -4.403803825378418,
|
||
|
|
"logits/rejected": -4.27715539932251,
|
||
|
|
"logps/chosen": -103.4933853149414,
|
||
|
|
"logps/rejected": -131.231201171875,
|
||
|
|
"loss": 0.2582036733627319,
|
||
|
|
"rewards/accuracies": 0.8687499761581421,
|
||
|
|
"rewards/chosen": -2.3441028594970703,
|
||
|
|
"rewards/margins": 2.2010316848754883,
|
||
|
|
"rewards/rejected": -4.545134544372559,
|
||
|
|
"step": 1890
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.067482993197279,
|
||
|
|
"grad_norm": 0.488471657037735,
|
||
|
|
"learning_rate": 6.417352281226627e-05,
|
||
|
|
"logits/chosen": -4.374741554260254,
|
||
|
|
"logits/rejected": -4.235744476318359,
|
||
|
|
"logps/chosen": -108.23167419433594,
|
||
|
|
"logps/rejected": -136.00350952148438,
|
||
|
|
"loss": 0.2654814004898071,
|
||
|
|
"rewards/accuracies": 0.844531238079071,
|
||
|
|
"rewards/chosen": -2.7162108421325684,
|
||
|
|
"rewards/margins": 2.194584846496582,
|
||
|
|
"rewards/rejected": -4.910796165466309,
|
||
|
|
"step": 1900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.0783673469387756,
|
||
|
|
"grad_norm": 0.5405411720275879,
|
||
|
|
"learning_rate": 6.342557965594615e-05,
|
||
|
|
"logits/chosen": -4.263213157653809,
|
||
|
|
"logits/rejected": -4.2035136222839355,
|
||
|
|
"logps/chosen": -108.0687255859375,
|
||
|
|
"logps/rejected": -135.1525115966797,
|
||
|
|
"loss": 0.269286060333252,
|
||
|
|
"rewards/accuracies": 0.8460937738418579,
|
||
|
|
"rewards/chosen": -2.618056297302246,
|
||
|
|
"rewards/margins": 2.1149375438690186,
|
||
|
|
"rewards/rejected": -4.732994079589844,
|
||
|
|
"step": 1910
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.089251700680272,
|
||
|
|
"grad_norm": 0.5153743624687195,
|
||
|
|
"learning_rate": 6.267763649962603e-05,
|
||
|
|
"logits/chosen": -4.374856472015381,
|
||
|
|
"logits/rejected": -4.3279643058776855,
|
||
|
|
"logps/chosen": -102.6282958984375,
|
||
|
|
"logps/rejected": -129.84768676757812,
|
||
|
|
"loss": 0.27457327842712403,
|
||
|
|
"rewards/accuracies": 0.83984375,
|
||
|
|
"rewards/chosen": -2.5648117065429688,
|
||
|
|
"rewards/margins": 2.1569466590881348,
|
||
|
|
"rewards/rejected": -4.7217583656311035,
|
||
|
|
"step": 1920
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.100136054421769,
|
||
|
|
"grad_norm": 0.5591779947280884,
|
||
|
|
"learning_rate": 6.192969334330591e-05,
|
||
|
|
"logits/chosen": -4.222095489501953,
|
||
|
|
"logits/rejected": -4.156058311462402,
|
||
|
|
"logps/chosen": -107.1645736694336,
|
||
|
|
"logps/rejected": -131.6280059814453,
|
||
|
|
"loss": 0.2775254726409912,
|
||
|
|
"rewards/accuracies": 0.8218749761581421,
|
||
|
|
"rewards/chosen": -2.541388988494873,
|
||
|
|
"rewards/margins": 2.1209020614624023,
|
||
|
|
"rewards/rejected": -4.662291526794434,
|
||
|
|
"step": 1930
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1110204081632653,
|
||
|
|
"grad_norm": 0.47012007236480713,
|
||
|
|
"learning_rate": 6.118175018698578e-05,
|
||
|
|
"logits/chosen": -4.261131763458252,
|
||
|
|
"logits/rejected": -4.192044258117676,
|
||
|
|
"logps/chosen": -109.32353210449219,
|
||
|
|
"logps/rejected": -134.66297912597656,
|
||
|
|
"loss": 0.26659140586853025,
|
||
|
|
"rewards/accuracies": 0.8531249761581421,
|
||
|
|
"rewards/chosen": -2.73209810256958,
|
||
|
|
"rewards/margins": 2.093428134918213,
|
||
|
|
"rewards/rejected": -4.825526237487793,
|
||
|
|
"step": 1940
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1219047619047617,
|
||
|
|
"grad_norm": 0.5636943578720093,
|
||
|
|
"learning_rate": 6.0433807030665666e-05,
|
||
|
|
"logits/chosen": -4.38666296005249,
|
||
|
|
"logits/rejected": -4.33133602142334,
|
||
|
|
"logps/chosen": -107.5749282836914,
|
||
|
|
"logps/rejected": -132.30654907226562,
|
||
|
|
"loss": 0.27169456481933596,
|
||
|
|
"rewards/accuracies": 0.839062511920929,
|
||
|
|
"rewards/chosen": -2.6706109046936035,
|
||
|
|
"rewards/margins": 2.161822557449341,
|
||
|
|
"rewards/rejected": -4.832433223724365,
|
||
|
|
"step": 1950
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1327891156462586,
|
||
|
|
"grad_norm": 0.5739550590515137,
|
||
|
|
"learning_rate": 5.968586387434555e-05,
|
||
|
|
"logits/chosen": -4.306714057922363,
|
||
|
|
"logits/rejected": -4.2323102951049805,
|
||
|
|
"logps/chosen": -110.61419677734375,
|
||
|
|
"logps/rejected": -136.73745727539062,
|
||
|
|
"loss": 0.2676869869232178,
|
||
|
|
"rewards/accuracies": 0.85546875,
|
||
|
|
"rewards/chosen": -2.7505619525909424,
|
||
|
|
"rewards/margins": 2.1400463581085205,
|
||
|
|
"rewards/rejected": -4.890608310699463,
|
||
|
|
"step": 1960
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.143673469387755,
|
||
|
|
"grad_norm": 0.5009965896606445,
|
||
|
|
"learning_rate": 5.893792071802543e-05,
|
||
|
|
"logits/chosen": -4.33192253112793,
|
||
|
|
"logits/rejected": -4.311135768890381,
|
||
|
|
"logps/chosen": -108.64591979980469,
|
||
|
|
"logps/rejected": -132.796142578125,
|
||
|
|
"loss": 0.2648327112197876,
|
||
|
|
"rewards/accuracies": 0.8492187261581421,
|
||
|
|
"rewards/chosen": -2.6455886363983154,
|
||
|
|
"rewards/margins": 2.131373882293701,
|
||
|
|
"rewards/rejected": -4.776961803436279,
|
||
|
|
"step": 1970
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1545578231292515,
|
||
|
|
"grad_norm": 0.6320508718490601,
|
||
|
|
"learning_rate": 5.818997756170531e-05,
|
||
|
|
"logits/chosen": -4.376672267913818,
|
||
|
|
"logits/rejected": -4.221155643463135,
|
||
|
|
"logps/chosen": -107.09896087646484,
|
||
|
|
"logps/rejected": -135.03883361816406,
|
||
|
|
"loss": 0.2541360855102539,
|
||
|
|
"rewards/accuracies": 0.8570312261581421,
|
||
|
|
"rewards/chosen": -2.8258004188537598,
|
||
|
|
"rewards/margins": 2.2211556434631348,
|
||
|
|
"rewards/rejected": -5.046955585479736,
|
||
|
|
"step": 1980
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1654421768707484,
|
||
|
|
"grad_norm": 0.5602008700370789,
|
||
|
|
"learning_rate": 5.7442034405385194e-05,
|
||
|
|
"logits/chosen": -4.365457057952881,
|
||
|
|
"logits/rejected": -4.279487609863281,
|
||
|
|
"logps/chosen": -110.8012924194336,
|
||
|
|
"logps/rejected": -138.8103790283203,
|
||
|
|
"loss": 0.26124260425567625,
|
||
|
|
"rewards/accuracies": 0.8609374761581421,
|
||
|
|
"rewards/chosen": -3.039825439453125,
|
||
|
|
"rewards/margins": 2.225407838821411,
|
||
|
|
"rewards/rejected": -5.265233039855957,
|
||
|
|
"step": 1990
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.176326530612245,
|
||
|
|
"grad_norm": 0.5244850516319275,
|
||
|
|
"learning_rate": 5.6694091249065076e-05,
|
||
|
|
"logits/chosen": -4.355788707733154,
|
||
|
|
"logits/rejected": -4.297573566436768,
|
||
|
|
"logps/chosen": -113.49159240722656,
|
||
|
|
"logps/rejected": -140.8461456298828,
|
||
|
|
"loss": 0.2652238130569458,
|
||
|
|
"rewards/accuracies": 0.8359375,
|
||
|
|
"rewards/chosen": -3.1340157985687256,
|
||
|
|
"rewards/margins": 2.230769634246826,
|
||
|
|
"rewards/rejected": -5.364785194396973,
|
||
|
|
"step": 2000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.176326530612245,
|
||
|
|
"eval_logits/chosen": -4.392631530761719,
|
||
|
|
"eval_logits/rejected": -4.344852924346924,
|
||
|
|
"eval_logps/chosen": -113.7379150390625,
|
||
|
|
"eval_logps/rejected": -135.5451202392578,
|
||
|
|
"eval_loss": 0.46534663438796997,
|
||
|
|
"eval_rewards/accuracies": 0.7124999761581421,
|
||
|
|
"eval_rewards/chosen": -3.413275957107544,
|
||
|
|
"eval_rewards/margins": 1.556301474571228,
|
||
|
|
"eval_rewards/rejected": -4.969577789306641,
|
||
|
|
"eval_runtime": 104.78,
|
||
|
|
"eval_samples_per_second": 22.905,
|
||
|
|
"eval_steps_per_second": 2.863,
|
||
|
|
"step": 2000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.1872108843537417,
|
||
|
|
"grad_norm": 0.6399655342102051,
|
||
|
|
"learning_rate": 5.594614809274495e-05,
|
||
|
|
"logits/chosen": -4.25847053527832,
|
||
|
|
"logits/rejected": -4.194684028625488,
|
||
|
|
"logps/chosen": -115.2059555053711,
|
||
|
|
"logps/rejected": -140.93521118164062,
|
||
|
|
"loss": 0.2672285079956055,
|
||
|
|
"rewards/accuracies": 0.84375,
|
||
|
|
"rewards/chosen": -3.0837132930755615,
|
||
|
|
"rewards/margins": 2.1926894187927246,
|
||
|
|
"rewards/rejected": -5.276402950286865,
|
||
|
|
"step": 2010
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.198095238095238,
|
||
|
|
"grad_norm": 0.5022415518760681,
|
||
|
|
"learning_rate": 5.5198204936424834e-05,
|
||
|
|
"logits/chosen": -4.377929210662842,
|
||
|
|
"logits/rejected": -4.255377769470215,
|
||
|
|
"logps/chosen": -108.50428771972656,
|
||
|
|
"logps/rejected": -137.51651000976562,
|
||
|
|
"loss": 0.2487508773803711,
|
||
|
|
"rewards/accuracies": 0.8687499761581421,
|
||
|
|
"rewards/chosen": -2.958159923553467,
|
||
|
|
"rewards/margins": 2.2823288440704346,
|
||
|
|
"rewards/rejected": -5.240488529205322,
|
||
|
|
"step": 2020
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2089795918367345,
|
||
|
|
"grad_norm": 0.4892471134662628,
|
||
|
|
"learning_rate": 5.4450261780104716e-05,
|
||
|
|
"logits/chosen": -4.4286956787109375,
|
||
|
|
"logits/rejected": -4.3171610832214355,
|
||
|
|
"logps/chosen": -108.22352600097656,
|
||
|
|
"logps/rejected": -137.69073486328125,
|
||
|
|
"loss": 0.25264739990234375,
|
||
|
|
"rewards/accuracies": 0.850781261920929,
|
||
|
|
"rewards/chosen": -2.9794039726257324,
|
||
|
|
"rewards/margins": 2.2621421813964844,
|
||
|
|
"rewards/rejected": -5.241546154022217,
|
||
|
|
"step": 2030
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2198639455782314,
|
||
|
|
"grad_norm": 0.5124601125717163,
|
||
|
|
"learning_rate": 5.37023186237846e-05,
|
||
|
|
"logits/chosen": -4.357671737670898,
|
||
|
|
"logits/rejected": -4.335933685302734,
|
||
|
|
"logps/chosen": -110.78375244140625,
|
||
|
|
"logps/rejected": -140.46006774902344,
|
||
|
|
"loss": 0.24789047241210938,
|
||
|
|
"rewards/accuracies": 0.8515625,
|
||
|
|
"rewards/chosen": -2.977566957473755,
|
||
|
|
"rewards/margins": 2.320561170578003,
|
||
|
|
"rewards/rejected": -5.2981276512146,
|
||
|
|
"step": 2040
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.230748299319728,
|
||
|
|
"grad_norm": 0.590368926525116,
|
||
|
|
"learning_rate": 5.295437546746448e-05,
|
||
|
|
"logits/chosen": -4.396256446838379,
|
||
|
|
"logits/rejected": -4.374178409576416,
|
||
|
|
"logps/chosen": -108.7447280883789,
|
||
|
|
"logps/rejected": -136.3175506591797,
|
||
|
|
"loss": 0.27484884262084963,
|
||
|
|
"rewards/accuracies": 0.8359375,
|
||
|
|
"rewards/chosen": -2.8392646312713623,
|
||
|
|
"rewards/margins": 2.158670663833618,
|
||
|
|
"rewards/rejected": -4.9979352951049805,
|
||
|
|
"step": 2050
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2416326530612247,
|
||
|
|
"grad_norm": 0.5153351426124573,
|
||
|
|
"learning_rate": 5.220643231114436e-05,
|
||
|
|
"logits/chosen": -4.382789134979248,
|
||
|
|
"logits/rejected": -4.334689140319824,
|
||
|
|
"logps/chosen": -108.401611328125,
|
||
|
|
"logps/rejected": -137.27871704101562,
|
||
|
|
"loss": 0.2631440877914429,
|
||
|
|
"rewards/accuracies": 0.840624988079071,
|
||
|
|
"rewards/chosen": -2.8784780502319336,
|
||
|
|
"rewards/margins": 2.264145612716675,
|
||
|
|
"rewards/rejected": -5.1426239013671875,
|
||
|
|
"step": 2060
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.252517006802721,
|
||
|
|
"grad_norm": 0.5086898803710938,
|
||
|
|
"learning_rate": 5.145848915482424e-05,
|
||
|
|
"logits/chosen": -4.282557487487793,
|
||
|
|
"logits/rejected": -4.198935508728027,
|
||
|
|
"logps/chosen": -107.8048095703125,
|
||
|
|
"logps/rejected": -135.6783905029297,
|
||
|
|
"loss": 0.26320858001708985,
|
||
|
|
"rewards/accuracies": 0.8515625,
|
||
|
|
"rewards/chosen": -2.7925009727478027,
|
||
|
|
"rewards/margins": 2.1981430053710938,
|
||
|
|
"rewards/rejected": -4.990644454956055,
|
||
|
|
"step": 2070
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2634013605442176,
|
||
|
|
"grad_norm": 0.5398783683776855,
|
||
|
|
"learning_rate": 5.0710545998504114e-05,
|
||
|
|
"logits/chosen": -4.345008850097656,
|
||
|
|
"logits/rejected": -4.317435264587402,
|
||
|
|
"logps/chosen": -106.77925872802734,
|
||
|
|
"logps/rejected": -136.1848907470703,
|
||
|
|
"loss": 0.2555952787399292,
|
||
|
|
"rewards/accuracies": 0.8570312261581421,
|
||
|
|
"rewards/chosen": -2.866353988647461,
|
||
|
|
"rewards/margins": 2.281134843826294,
|
||
|
|
"rewards/rejected": -5.147488594055176,
|
||
|
|
"step": 2080
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2742857142857145,
|
||
|
|
"grad_norm": 0.4644974172115326,
|
||
|
|
"learning_rate": 4.9962602842183996e-05,
|
||
|
|
"logits/chosen": -4.256831169128418,
|
||
|
|
"logits/rejected": -4.2593584060668945,
|
||
|
|
"logps/chosen": -106.45500183105469,
|
||
|
|
"logps/rejected": -136.2468719482422,
|
||
|
|
"loss": 0.26251227855682374,
|
||
|
|
"rewards/accuracies": 0.84765625,
|
||
|
|
"rewards/chosen": -2.910574436187744,
|
||
|
|
"rewards/margins": 2.2922780513763428,
|
||
|
|
"rewards/rejected": -5.202852725982666,
|
||
|
|
"step": 2090
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.285170068027211,
|
||
|
|
"grad_norm": 0.38658151030540466,
|
||
|
|
"learning_rate": 4.921465968586388e-05,
|
||
|
|
"logits/chosen": -4.299098491668701,
|
||
|
|
"logits/rejected": -4.222846984863281,
|
||
|
|
"logps/chosen": -111.60084533691406,
|
||
|
|
"logps/rejected": -139.23606872558594,
|
||
|
|
"loss": 0.26093616485595705,
|
||
|
|
"rewards/accuracies": 0.8460937738418579,
|
||
|
|
"rewards/chosen": -2.9908206462860107,
|
||
|
|
"rewards/margins": 2.2845864295959473,
|
||
|
|
"rewards/rejected": -5.275406837463379,
|
||
|
|
"step": 2100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.2960544217687073,
|
||
|
|
"grad_norm": 0.5244455933570862,
|
||
|
|
"learning_rate": 4.846671652954376e-05,
|
||
|
|
"logits/chosen": -4.305111408233643,
|
||
|
|
"logits/rejected": -4.2441325187683105,
|
||
|
|
"logps/chosen": -105.7240982055664,
|
||
|
|
"logps/rejected": -132.33541870117188,
|
||
|
|
"loss": 0.26281836032867434,
|
||
|
|
"rewards/accuracies": 0.8578125238418579,
|
||
|
|
"rewards/chosen": -2.8944408893585205,
|
||
|
|
"rewards/margins": 2.207293748855591,
|
||
|
|
"rewards/rejected": -5.1017351150512695,
|
||
|
|
"step": 2110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.306938775510204,
|
||
|
|
"grad_norm": 0.5264010429382324,
|
||
|
|
"learning_rate": 4.7718773373223636e-05,
|
||
|
|
"logits/chosen": -4.359262943267822,
|
||
|
|
"logits/rejected": -4.287570476531982,
|
||
|
|
"logps/chosen": -107.4122085571289,
|
||
|
|
"logps/rejected": -135.27484130859375,
|
||
|
|
"loss": 0.279754638671875,
|
||
|
|
"rewards/accuracies": 0.821093738079071,
|
||
|
|
"rewards/chosen": -2.8419368267059326,
|
||
|
|
"rewards/margins": 2.1469409465789795,
|
||
|
|
"rewards/rejected": -4.98887825012207,
|
||
|
|
"step": 2120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.3178231292517006,
|
||
|
|
"grad_norm": 0.6190440058708191,
|
||
|
|
"learning_rate": 4.697083021690352e-05,
|
||
|
|
"logits/chosen": -4.3371100425720215,
|
||
|
|
"logits/rejected": -4.2610368728637695,
|
||
|
|
"logps/chosen": -109.62430572509766,
|
||
|
|
"logps/rejected": -136.06649780273438,
|
||
|
|
"loss": 0.27123911380767823,
|
||
|
|
"rewards/accuracies": 0.8414062261581421,
|
||
|
|
"rewards/chosen": -2.924973487854004,
|
||
|
|
"rewards/margins": 2.1363399028778076,
|
||
|
|
"rewards/rejected": -5.061313152313232,
|
||
|
|
"step": 2130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.328707482993197,
|
||
|
|
"grad_norm": 0.4766679108142853,
|
||
|
|
"learning_rate": 4.62228870605834e-05,
|
||
|
|
"logits/chosen": -4.276871681213379,
|
||
|
|
"logits/rejected": -4.208295822143555,
|
||
|
|
"logps/chosen": -110.64579010009766,
|
||
|
|
"logps/rejected": -139.74888610839844,
|
||
|
|
"loss": 0.24541153907775878,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": -2.862879991531372,
|
||
|
|
"rewards/margins": 2.3135013580322266,
|
||
|
|
"rewards/rejected": -5.176381587982178,
|
||
|
|
"step": 2140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.339591836734694,
|
||
|
|
"grad_norm": 0.4791225492954254,
|
||
|
|
"learning_rate": 4.5474943904263275e-05,
|
||
|
|
"logits/chosen": -4.355441093444824,
|
||
|
|
"logits/rejected": -4.288341522216797,
|
||
|
|
"logps/chosen": -105.79945373535156,
|
||
|
|
"logps/rejected": -131.28944396972656,
|
||
|
|
"loss": 0.27122857570648196,
|
||
|
|
"rewards/accuracies": 0.8414062261581421,
|
||
|
|
"rewards/chosen": -2.718742609024048,
|
||
|
|
"rewards/margins": 2.105710744857788,
|
||
|
|
"rewards/rejected": -4.824452877044678,
|
||
|
|
"step": 2150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.3504761904761904,
|
||
|
|
"grad_norm": 0.6803158521652222,
|
||
|
|
"learning_rate": 4.472700074794316e-05,
|
||
|
|
"logits/chosen": -4.416818141937256,
|
||
|
|
"logits/rejected": -4.399478435516357,
|
||
|
|
"logps/chosen": -105.3951187133789,
|
||
|
|
"logps/rejected": -134.6707763671875,
|
||
|
|
"loss": 0.25935161113739014,
|
||
|
|
"rewards/accuracies": 0.846875011920929,
|
||
|
|
"rewards/chosen": -2.693054676055908,
|
||
|
|
"rewards/margins": 2.30621337890625,
|
||
|
|
"rewards/rejected": -4.999267578125,
|
||
|
|
"step": 2160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.3613605442176873,
|
||
|
|
"grad_norm": 0.5753185749053955,
|
||
|
|
"learning_rate": 4.397905759162304e-05,
|
||
|
|
"logits/chosen": -4.324499607086182,
|
||
|
|
"logits/rejected": -4.302281856536865,
|
||
|
|
"logps/chosen": -108.54312896728516,
|
||
|
|
"logps/rejected": -138.02798461914062,
|
||
|
|
"loss": 0.24565551280975342,
|
||
|
|
"rewards/accuracies": 0.85546875,
|
||
|
|
"rewards/chosen": -2.722982883453369,
|
||
|
|
"rewards/margins": 2.3493704795837402,
|
||
|
|
"rewards/rejected": -5.072353363037109,
|
||
|
|
"step": 2170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.3722448979591837,
|
||
|
|
"grad_norm": 0.5277524590492249,
|
||
|
|
"learning_rate": 4.323111443530292e-05,
|
||
|
|
"logits/chosen": -4.220860481262207,
|
||
|
|
"logits/rejected": -4.199186325073242,
|
||
|
|
"logps/chosen": -109.83711242675781,
|
||
|
|
"logps/rejected": -138.1239013671875,
|
||
|
|
"loss": 0.2549403429031372,
|
||
|
|
"rewards/accuracies": 0.854687511920929,
|
||
|
|
"rewards/chosen": -2.7489871978759766,
|
||
|
|
"rewards/margins": 2.2629806995391846,
|
||
|
|
"rewards/rejected": -5.011967658996582,
|
||
|
|
"step": 2180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.38312925170068,
|
||
|
|
"grad_norm": 0.6214152574539185,
|
||
|
|
"learning_rate": 4.24831712789828e-05,
|
||
|
|
"logits/chosen": -4.419411659240723,
|
||
|
|
"logits/rejected": -4.302346706390381,
|
||
|
|
"logps/chosen": -109.27337646484375,
|
||
|
|
"logps/rejected": -137.83436584472656,
|
||
|
|
"loss": 0.25436248779296877,
|
||
|
|
"rewards/accuracies": 0.852343738079071,
|
||
|
|
"rewards/chosen": -2.788382053375244,
|
||
|
|
"rewards/margins": 2.3577487468719482,
|
||
|
|
"rewards/rejected": -5.1461310386657715,
|
||
|
|
"step": 2190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.394013605442177,
|
||
|
|
"grad_norm": 0.6792303323745728,
|
||
|
|
"learning_rate": 4.173522812266268e-05,
|
||
|
|
"logits/chosen": -4.336219787597656,
|
||
|
|
"logits/rejected": -4.238919258117676,
|
||
|
|
"logps/chosen": -107.77840423583984,
|
||
|
|
"logps/rejected": -135.3695831298828,
|
||
|
|
"loss": 0.26418342590332033,
|
||
|
|
"rewards/accuracies": 0.84375,
|
||
|
|
"rewards/chosen": -2.842252492904663,
|
||
|
|
"rewards/margins": 2.2352678775787354,
|
||
|
|
"rewards/rejected": -5.077520847320557,
|
||
|
|
"step": 2200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4048979591836734,
|
||
|
|
"grad_norm": 0.5071048736572266,
|
||
|
|
"learning_rate": 4.098728496634256e-05,
|
||
|
|
"logits/chosen": -4.275275230407715,
|
||
|
|
"logits/rejected": -4.160519599914551,
|
||
|
|
"logps/chosen": -108.19576263427734,
|
||
|
|
"logps/rejected": -135.44322204589844,
|
||
|
|
"loss": 0.25397777557373047,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": -2.940242052078247,
|
||
|
|
"rewards/margins": 2.3158531188964844,
|
||
|
|
"rewards/rejected": -5.256094932556152,
|
||
|
|
"step": 2210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4157823129251703,
|
||
|
|
"grad_norm": 0.5598253607749939,
|
||
|
|
"learning_rate": 4.0239341810022444e-05,
|
||
|
|
"logits/chosen": -4.34271240234375,
|
||
|
|
"logits/rejected": -4.281783103942871,
|
||
|
|
"logps/chosen": -111.84043884277344,
|
||
|
|
"logps/rejected": -139.2178497314453,
|
||
|
|
"loss": 0.25306150913238523,
|
||
|
|
"rewards/accuracies": 0.8453124761581421,
|
||
|
|
"rewards/chosen": -2.9429874420166016,
|
||
|
|
"rewards/margins": 2.3303146362304688,
|
||
|
|
"rewards/rejected": -5.27330207824707,
|
||
|
|
"step": 2220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4266666666666667,
|
||
|
|
"grad_norm": 0.4812939763069153,
|
||
|
|
"learning_rate": 3.949139865370232e-05,
|
||
|
|
"logits/chosen": -4.2378034591674805,
|
||
|
|
"logits/rejected": -4.214576244354248,
|
||
|
|
"logps/chosen": -108.81040954589844,
|
||
|
|
"logps/rejected": -136.83827209472656,
|
||
|
|
"loss": 0.26318118572235105,
|
||
|
|
"rewards/accuracies": 0.8421875238418579,
|
||
|
|
"rewards/chosen": -2.9642369747161865,
|
||
|
|
"rewards/margins": 2.3236169815063477,
|
||
|
|
"rewards/rejected": -5.287853717803955,
|
||
|
|
"step": 2230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.437551020408163,
|
||
|
|
"grad_norm": 0.5383220314979553,
|
||
|
|
"learning_rate": 3.87434554973822e-05,
|
||
|
|
"logits/chosen": -4.168697357177734,
|
||
|
|
"logits/rejected": -4.0632429122924805,
|
||
|
|
"logps/chosen": -113.34503173828125,
|
||
|
|
"logps/rejected": -140.87977600097656,
|
||
|
|
"loss": 0.2669438362121582,
|
||
|
|
"rewards/accuracies": 0.8453124761581421,
|
||
|
|
"rewards/chosen": -3.066909074783325,
|
||
|
|
"rewards/margins": 2.23907470703125,
|
||
|
|
"rewards/rejected": -5.305983066558838,
|
||
|
|
"step": 2240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4484353741496596,
|
||
|
|
"grad_norm": 0.4986330270767212,
|
||
|
|
"learning_rate": 3.799551234106208e-05,
|
||
|
|
"logits/chosen": -4.118273735046387,
|
||
|
|
"logits/rejected": -4.0370354652404785,
|
||
|
|
"logps/chosen": -110.60295104980469,
|
||
|
|
"logps/rejected": -138.21926879882812,
|
||
|
|
"loss": 0.26406409740448,
|
||
|
|
"rewards/accuracies": 0.850781261920929,
|
||
|
|
"rewards/chosen": -2.9518330097198486,
|
||
|
|
"rewards/margins": 2.2067174911499023,
|
||
|
|
"rewards/rejected": -5.158550262451172,
|
||
|
|
"step": 2250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4593197278911565,
|
||
|
|
"grad_norm": 0.5483224391937256,
|
||
|
|
"learning_rate": 3.724756918474196e-05,
|
||
|
|
"logits/chosen": -4.164062023162842,
|
||
|
|
"logits/rejected": -4.178997993469238,
|
||
|
|
"logps/chosen": -112.80091857910156,
|
||
|
|
"logps/rejected": -140.44825744628906,
|
||
|
|
"loss": 0.25186111927032473,
|
||
|
|
"rewards/accuracies": 0.8570312261581421,
|
||
|
|
"rewards/chosen": -3.0120620727539062,
|
||
|
|
"rewards/margins": 2.2925403118133545,
|
||
|
|
"rewards/rejected": -5.30460262298584,
|
||
|
|
"step": 2260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.470204081632653,
|
||
|
|
"grad_norm": 0.49677279591560364,
|
||
|
|
"learning_rate": 3.649962602842184e-05,
|
||
|
|
"logits/chosen": -4.339353561401367,
|
||
|
|
"logits/rejected": -4.249403953552246,
|
||
|
|
"logps/chosen": -110.08421325683594,
|
||
|
|
"logps/rejected": -137.5714569091797,
|
||
|
|
"loss": 0.25184576511383056,
|
||
|
|
"rewards/accuracies": 0.848437488079071,
|
||
|
|
"rewards/chosen": -3.0821781158447266,
|
||
|
|
"rewards/margins": 2.328827381134033,
|
||
|
|
"rewards/rejected": -5.411005020141602,
|
||
|
|
"step": 2270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.48108843537415,
|
||
|
|
"grad_norm": 0.5725961923599243,
|
||
|
|
"learning_rate": 3.5751682872101724e-05,
|
||
|
|
"logits/chosen": -4.2684502601623535,
|
||
|
|
"logits/rejected": -4.1795573234558105,
|
||
|
|
"logps/chosen": -113.55293273925781,
|
||
|
|
"logps/rejected": -142.6102752685547,
|
||
|
|
"loss": 0.24979727268218993,
|
||
|
|
"rewards/accuracies": 0.8492187261581421,
|
||
|
|
"rewards/chosen": -3.1018178462982178,
|
||
|
|
"rewards/margins": 2.409865617752075,
|
||
|
|
"rewards/rejected": -5.511682987213135,
|
||
|
|
"step": 2280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.4919727891156462,
|
||
|
|
"grad_norm": 0.5567153096199036,
|
||
|
|
"learning_rate": 3.5003739715781606e-05,
|
||
|
|
"logits/chosen": -4.302936553955078,
|
||
|
|
"logits/rejected": -4.220743656158447,
|
||
|
|
"logps/chosen": -107.4947738647461,
|
||
|
|
"logps/rejected": -135.80044555664062,
|
||
|
|
"loss": 0.2599829435348511,
|
||
|
|
"rewards/accuracies": 0.8359375,
|
||
|
|
"rewards/chosen": -2.999293804168701,
|
||
|
|
"rewards/margins": 2.2999939918518066,
|
||
|
|
"rewards/rejected": -5.299286842346191,
|
||
|
|
"step": 2290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5028571428571427,
|
||
|
|
"grad_norm": 0.46114829182624817,
|
||
|
|
"learning_rate": 3.425579655946148e-05,
|
||
|
|
"logits/chosen": -4.257116794586182,
|
||
|
|
"logits/rejected": -4.284517765045166,
|
||
|
|
"logps/chosen": -109.46214294433594,
|
||
|
|
"logps/rejected": -138.5498809814453,
|
||
|
|
"loss": 0.262731671333313,
|
||
|
|
"rewards/accuracies": 0.8414062261581421,
|
||
|
|
"rewards/chosen": -3.0271120071411133,
|
||
|
|
"rewards/margins": 2.3265421390533447,
|
||
|
|
"rewards/rejected": -5.353653907775879,
|
||
|
|
"step": 2300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5137414965986395,
|
||
|
|
"grad_norm": 0.5548444390296936,
|
||
|
|
"learning_rate": 3.350785340314136e-05,
|
||
|
|
"logits/chosen": -4.13088321685791,
|
||
|
|
"logits/rejected": -4.090167045593262,
|
||
|
|
"logps/chosen": -112.9273681640625,
|
||
|
|
"logps/rejected": -140.58865356445312,
|
||
|
|
"loss": 0.26635379791259767,
|
||
|
|
"rewards/accuracies": 0.840624988079071,
|
||
|
|
"rewards/chosen": -3.1592650413513184,
|
||
|
|
"rewards/margins": 2.2824602127075195,
|
||
|
|
"rewards/rejected": -5.441725730895996,
|
||
|
|
"step": 2310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.524625850340136,
|
||
|
|
"grad_norm": 0.6786127686500549,
|
||
|
|
"learning_rate": 3.275991024682124e-05,
|
||
|
|
"logits/chosen": -4.259942531585693,
|
||
|
|
"logits/rejected": -4.2501912117004395,
|
||
|
|
"logps/chosen": -113.3272476196289,
|
||
|
|
"logps/rejected": -140.26583862304688,
|
||
|
|
"loss": 0.26966152191162107,
|
||
|
|
"rewards/accuracies": 0.8382812738418579,
|
||
|
|
"rewards/chosen": -3.1748275756835938,
|
||
|
|
"rewards/margins": 2.2647013664245605,
|
||
|
|
"rewards/rejected": -5.439528942108154,
|
||
|
|
"step": 2320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.535510204081633,
|
||
|
|
"grad_norm": 0.5421922206878662,
|
||
|
|
"learning_rate": 3.201196709050112e-05,
|
||
|
|
"logits/chosen": -4.2332868576049805,
|
||
|
|
"logits/rejected": -4.159974098205566,
|
||
|
|
"logps/chosen": -112.0591049194336,
|
||
|
|
"logps/rejected": -142.06764221191406,
|
||
|
|
"loss": 0.26064703464508054,
|
||
|
|
"rewards/accuracies": 0.842968761920929,
|
||
|
|
"rewards/chosen": -3.2271761894226074,
|
||
|
|
"rewards/margins": 2.313660144805908,
|
||
|
|
"rewards/rejected": -5.540836334228516,
|
||
|
|
"step": 2330
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5463945578231293,
|
||
|
|
"grad_norm": 0.6206730008125305,
|
||
|
|
"learning_rate": 3.1264023934181e-05,
|
||
|
|
"logits/chosen": -4.249005317687988,
|
||
|
|
"logits/rejected": -4.177195072174072,
|
||
|
|
"logps/chosen": -110.71000671386719,
|
||
|
|
"logps/rejected": -138.38018798828125,
|
||
|
|
"loss": 0.264833927154541,
|
||
|
|
"rewards/accuracies": 0.8359375,
|
||
|
|
"rewards/chosen": -3.2622718811035156,
|
||
|
|
"rewards/margins": 2.270108699798584,
|
||
|
|
"rewards/rejected": -5.532380104064941,
|
||
|
|
"step": 2340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5572789115646257,
|
||
|
|
"grad_norm": 0.6249208450317383,
|
||
|
|
"learning_rate": 3.0516080777860885e-05,
|
||
|
|
"logits/chosen": -4.308166980743408,
|
||
|
|
"logits/rejected": -4.222630977630615,
|
||
|
|
"logps/chosen": -115.17570495605469,
|
||
|
|
"logps/rejected": -144.38851928710938,
|
||
|
|
"loss": 0.2524010896682739,
|
||
|
|
"rewards/accuracies": 0.856249988079071,
|
||
|
|
"rewards/chosen": -3.351501941680908,
|
||
|
|
"rewards/margins": 2.3318376541137695,
|
||
|
|
"rewards/rejected": -5.683339595794678,
|
||
|
|
"step": 2350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.5681632653061226,
|
||
|
|
"grad_norm": 0.6504024863243103,
|
||
|
|
"learning_rate": 2.9768137621540764e-05,
|
||
|
|
"logits/chosen": -4.296634674072266,
|
||
|
|
"logits/rejected": -4.280292987823486,
|
||
|
|
"logps/chosen": -111.20161437988281,
|
||
|
|
"logps/rejected": -141.90963745117188,
|
||
|
|
"loss": 0.26886260509490967,
|
||
|
|
"rewards/accuracies": 0.83984375,
|
||
|
|
"rewards/chosen": -3.2387752532958984,
|
||
|
|
"rewards/margins": 2.3250107765197754,
|
||
|
|
"rewards/rejected": -5.563786506652832,
|
||
|
|
"step": 2360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.579047619047619,
|
||
|
|
"grad_norm": 0.6845365762710571,
|
||
|
|
"learning_rate": 2.9020194465220646e-05,
|
||
|
|
"logits/chosen": -4.261755466461182,
|
||
|
|
"logits/rejected": -4.167389392852783,
|
||
|
|
"logps/chosen": -111.0313949584961,
|
||
|
|
"logps/rejected": -138.18850708007812,
|
||
|
|
"loss": 0.2643138885498047,
|
||
|
|
"rewards/accuracies": 0.8335937261581421,
|
||
|
|
"rewards/chosen": -3.0966439247131348,
|
||
|
|
"rewards/margins": 2.2732677459716797,
|
||
|
|
"rewards/rejected": -5.369911193847656,
|
||
|
|
"step": 2370
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.589931972789116,
|
||
|
|
"grad_norm": 0.4879046380519867,
|
||
|
|
"learning_rate": 2.827225130890053e-05,
|
||
|
|
"logits/chosen": -4.409466743469238,
|
||
|
|
"logits/rejected": -4.352431297302246,
|
||
|
|
"logps/chosen": -106.5313949584961,
|
||
|
|
"logps/rejected": -137.22837829589844,
|
||
|
|
"loss": 0.25514419078826905,
|
||
|
|
"rewards/accuracies": 0.848437488079071,
|
||
|
|
"rewards/chosen": -2.9418864250183105,
|
||
|
|
"rewards/margins": 2.351497173309326,
|
||
|
|
"rewards/rejected": -5.293383598327637,
|
||
|
|
"step": 2380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.6008163265306123,
|
||
|
|
"grad_norm": 0.5651640892028809,
|
||
|
|
"learning_rate": 2.7524308152580404e-05,
|
||
|
|
"logits/chosen": -4.4179205894470215,
|
||
|
|
"logits/rejected": -4.379713535308838,
|
||
|
|
"logps/chosen": -108.92497253417969,
|
||
|
|
"logps/rejected": -137.62344360351562,
|
||
|
|
"loss": 0.24242501258850097,
|
||
|
|
"rewards/accuracies": 0.8578125238418579,
|
||
|
|
"rewards/chosen": -3.073535203933716,
|
||
|
|
"rewards/margins": 2.418722152709961,
|
||
|
|
"rewards/rejected": -5.492257118225098,
|
||
|
|
"step": 2390
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.6117006802721088,
|
||
|
|
"grad_norm": 0.523777425289154,
|
||
|
|
"learning_rate": 2.6776364996260283e-05,
|
||
|
|
"logits/chosen": -4.350298881530762,
|
||
|
|
"logits/rejected": -4.247467517852783,
|
||
|
|
"logps/chosen": -112.7027359008789,
|
||
|
|
"logps/rejected": -142.613037109375,
|
||
|
|
"loss": 0.2534715890884399,
|
||
|
|
"rewards/accuracies": 0.856249988079071,
|
||
|
|
"rewards/chosen": -3.111325979232788,
|
||
|
|
"rewards/margins": 2.404831886291504,
|
||
|
|
"rewards/rejected": -5.516158103942871,
|
||
|
|
"step": 2400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.622585034013605,
|
||
|
|
"grad_norm": 0.5190083384513855,
|
||
|
|
"learning_rate": 2.6028421839940165e-05,
|
||
|
|
"logits/chosen": -4.415008544921875,
|
||
|
|
"logits/rejected": -4.2630157470703125,
|
||
|
|
"logps/chosen": -109.89335632324219,
|
||
|
|
"logps/rejected": -137.45790100097656,
|
||
|
|
"loss": 0.26002261638641355,
|
||
|
|
"rewards/accuracies": 0.8382812738418579,
|
||
|
|
"rewards/chosen": -3.0627522468566895,
|
||
|
|
"rewards/margins": 2.3217196464538574,
|
||
|
|
"rewards/rejected": -5.384472370147705,
|
||
|
|
"step": 2410
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.633469387755102,
|
||
|
|
"grad_norm": 0.5784986615180969,
|
||
|
|
"learning_rate": 2.5280478683620047e-05,
|
||
|
|
"logits/chosen": -4.250868797302246,
|
||
|
|
"logits/rejected": -4.1973700523376465,
|
||
|
|
"logps/chosen": -111.86470794677734,
|
||
|
|
"logps/rejected": -139.78781127929688,
|
||
|
|
"loss": 0.25365798473358153,
|
||
|
|
"rewards/accuracies": 0.852343738079071,
|
||
|
|
"rewards/chosen": -3.048133611679077,
|
||
|
|
"rewards/margins": 2.3121330738067627,
|
||
|
|
"rewards/rejected": -5.36026668548584,
|
||
|
|
"step": 2420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.6443537414965985,
|
||
|
|
"grad_norm": 0.5177122354507446,
|
||
|
|
"learning_rate": 2.4532535527299926e-05,
|
||
|
|
"logits/chosen": -4.4071784019470215,
|
||
|
|
"logits/rejected": -4.31978702545166,
|
||
|
|
"logps/chosen": -114.18182373046875,
|
||
|
|
"logps/rejected": -142.67727661132812,
|
||
|
|
"loss": 0.24829225540161132,
|
||
|
|
"rewards/accuracies": 0.8421875238418579,
|
||
|
|
"rewards/chosen": -3.116865873336792,
|
||
|
|
"rewards/margins": 2.388279438018799,
|
||
|
|
"rewards/rejected": -5.50514554977417,
|
||
|
|
"step": 2430
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.6552380952380954,
|
||
|
|
"grad_norm": 0.6798398494720459,
|
||
|
|
"learning_rate": 2.3784592370979808e-05,
|
||
|
|
"logits/chosen": -4.312591075897217,
|
||
|
|
"logits/rejected": -4.273346424102783,
|
||
|
|
"logps/chosen": -109.682861328125,
|
||
|
|
"logps/rejected": -138.36782836914062,
|
||
|
|
"loss": 0.2676241397857666,
|
||
|
|
"rewards/accuracies": 0.839062511920929,
|
||
|
|
"rewards/chosen": -2.952719211578369,
|
||
|
|
"rewards/margins": 2.2746853828430176,
|
||
|
|
"rewards/rejected": -5.227404594421387,
|
||
|
|
"step": 2440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.666122448979592,
|
||
|
|
"grad_norm": 0.6012817025184631,
|
||
|
|
"learning_rate": 2.3036649214659687e-05,
|
||
|
|
"logits/chosen": -4.368619918823242,
|
||
|
|
"logits/rejected": -4.296780586242676,
|
||
|
|
"logps/chosen": -110.58982849121094,
|
||
|
|
"logps/rejected": -139.80557250976562,
|
||
|
|
"loss": 0.25081255435943606,
|
||
|
|
"rewards/accuracies": 0.86328125,
|
||
|
|
"rewards/chosen": -2.887507915496826,
|
||
|
|
"rewards/margins": 2.3469045162200928,
|
||
|
|
"rewards/rejected": -5.23441219329834,
|
||
|
|
"step": 2450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.6770068027210883,
|
||
|
|
"grad_norm": 0.5708255171775818,
|
||
|
|
"learning_rate": 2.228870605833957e-05,
|
||
|
|
"logits/chosen": -4.390729904174805,
|
||
|
|
"logits/rejected": -4.355556964874268,
|
||
|
|
"logps/chosen": -107.79595947265625,
|
||
|
|
"logps/rejected": -134.82522583007812,
|
||
|
|
"loss": 0.27308039665222167,
|
||
|
|
"rewards/accuracies": 0.828125,
|
||
|
|
"rewards/chosen": -2.8849995136260986,
|
||
|
|
"rewards/margins": 2.2530252933502197,
|
||
|
|
"rewards/rejected": -5.13802433013916,
|
||
|
|
"step": 2460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.687891156462585,
|
||
|
|
"grad_norm": 0.47491082549095154,
|
||
|
|
"learning_rate": 2.1540762902019448e-05,
|
||
|
|
"logits/chosen": -4.335263729095459,
|
||
|
|
"logits/rejected": -4.283821105957031,
|
||
|
|
"logps/chosen": -108.29530334472656,
|
||
|
|
"logps/rejected": -135.4779815673828,
|
||
|
|
"loss": 0.2607786417007446,
|
||
|
|
"rewards/accuracies": 0.856249988079071,
|
||
|
|
"rewards/chosen": -2.8725428581237793,
|
||
|
|
"rewards/margins": 2.3250625133514404,
|
||
|
|
"rewards/rejected": -5.197605133056641,
|
||
|
|
"step": 2470
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.6987755102040816,
|
||
|
|
"grad_norm": 0.5339019298553467,
|
||
|
|
"learning_rate": 2.0792819745699327e-05,
|
||
|
|
"logits/chosen": -4.364335060119629,
|
||
|
|
"logits/rejected": -4.271590709686279,
|
||
|
|
"logps/chosen": -109.25032043457031,
|
||
|
|
"logps/rejected": -137.19566345214844,
|
||
|
|
"loss": 0.25407183170318604,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": -2.983041286468506,
|
||
|
|
"rewards/margins": 2.2924866676330566,
|
||
|
|
"rewards/rejected": -5.275527477264404,
|
||
|
|
"step": 2480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.7096598639455785,
|
||
|
|
"grad_norm": 0.5330259799957275,
|
||
|
|
"learning_rate": 2.004487658937921e-05,
|
||
|
|
"logits/chosen": -4.272704601287842,
|
||
|
|
"logits/rejected": -4.25288724899292,
|
||
|
|
"logps/chosen": -109.9069595336914,
|
||
|
|
"logps/rejected": -138.80812072753906,
|
||
|
|
"loss": 0.25796828269958494,
|
||
|
|
"rewards/accuracies": 0.858593761920929,
|
||
|
|
"rewards/chosen": -3.022810697555542,
|
||
|
|
"rewards/margins": 2.3227627277374268,
|
||
|
|
"rewards/rejected": -5.3455729484558105,
|
||
|
|
"step": 2490
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.720544217687075,
|
||
|
|
"grad_norm": 0.48291197419166565,
|
||
|
|
"learning_rate": 1.9296933433059088e-05,
|
||
|
|
"logits/chosen": -4.345968723297119,
|
||
|
|
"logits/rejected": -4.260931015014648,
|
||
|
|
"logps/chosen": -114.93940734863281,
|
||
|
|
"logps/rejected": -143.88821411132812,
|
||
|
|
"loss": 0.24913032054901124,
|
||
|
|
"rewards/accuracies": 0.844531238079071,
|
||
|
|
"rewards/chosen": -3.0745816230773926,
|
||
|
|
"rewards/margins": 2.3651044368743896,
|
||
|
|
"rewards/rejected": -5.439685821533203,
|
||
|
|
"step": 2500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 2.720544217687075,
|
||
|
|
"eval_logits/chosen": -4.365705490112305,
|
||
|
|
"eval_logits/rejected": -4.315591335296631,
|
||
|
|
"eval_logps/chosen": -113.0962142944336,
|
||
|
|
"eval_logps/rejected": -135.48565673828125,
|
||
|
|
"eval_loss": 0.45733407139778137,
|
||
|
|
"eval_rewards/accuracies": 0.721666693687439,
|
||
|
|
"eval_rewards/chosen": -3.3491051197052,
|
||
|
|
"eval_rewards/margins": 1.6145259141921997,
|
||
|
|
"eval_rewards/rejected": -4.9636311531066895,
|
||
|
|
"eval_runtime": 104.6509,
|
||
|
|
"eval_samples_per_second": 22.933,
|
||
|
|
"eval_steps_per_second": 2.867,
|
||
|
|
"step": 2500
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 10,
|
||
|
|
"max_steps": 2757,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 3,
|
||
|
|
"save_steps": 500,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"EarlyStoppingCallback": {
|
||
|
|
"args": {
|
||
|
|
"early_stopping_patience": 3,
|
||
|
|
"early_stopping_threshold": 0.0
|
||
|
|
},
|
||
|
|
"attributes": {
|
||
|
|
"early_stopping_patience_counter": 0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": false
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 8,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|