2946 lines
102 KiB
JSON
2946 lines
102 KiB
JSON
{
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 1.3129888447236826,
|
|
"eval_steps": 10,
|
|
"global_step": 320,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.008206180279523016,
|
|
"grad_norm": 72.29982535271158,
|
|
"learning_rate": 2.040816326530612e-08,
|
|
"logits/chosen": -2.1581597328186035,
|
|
"logits/rejected": -2.159653902053833,
|
|
"logps/chosen": -25.12261962890625,
|
|
"logps/rejected": -43.09302520751953,
|
|
"loss": 0.6931,
|
|
"rewards/accuracies": 0.0,
|
|
"rewards/chosen": 0.0,
|
|
"rewards/margins": 0.0,
|
|
"rewards/rejected": 0.0,
|
|
"step": 2
|
|
},
|
|
{
|
|
"epoch": 0.016412360559046033,
|
|
"grad_norm": 96.10955860968517,
|
|
"learning_rate": 4.081632653061224e-08,
|
|
"logits/chosen": -2.0733723640441895,
|
|
"logits/rejected": -2.0702476501464844,
|
|
"logps/chosen": -24.637685775756836,
|
|
"logps/rejected": -41.43503189086914,
|
|
"loss": 0.6963,
|
|
"rewards/accuracies": 0.453125,
|
|
"rewards/chosen": 0.008512299507856369,
|
|
"rewards/margins": -0.013849787414073944,
|
|
"rewards/rejected": 0.022362088784575462,
|
|
"step": 4
|
|
},
|
|
{
|
|
"epoch": 0.02461854083856905,
|
|
"grad_norm": 68.94505414065074,
|
|
"learning_rate": 6.122448979591837e-08,
|
|
"logits/chosen": -2.118948459625244,
|
|
"logits/rejected": -2.109750270843506,
|
|
"logps/chosen": -20.342554092407227,
|
|
"logps/rejected": -29.361812591552734,
|
|
"loss": 0.6878,
|
|
"rewards/accuracies": 0.484375,
|
|
"rewards/chosen": 0.028929362073540688,
|
|
"rewards/margins": 0.029945004731416702,
|
|
"rewards/rejected": -0.0010156440548598766,
|
|
"step": 6
|
|
},
|
|
{
|
|
"epoch": 0.032824721118092065,
|
|
"grad_norm": 77.97306429217313,
|
|
"learning_rate": 8.163265306122448e-08,
|
|
"logits/chosen": -2.0574400424957275,
|
|
"logits/rejected": -2.0550637245178223,
|
|
"logps/chosen": -27.057085037231445,
|
|
"logps/rejected": -39.5283088684082,
|
|
"loss": 0.7013,
|
|
"rewards/accuracies": 0.40625,
|
|
"rewards/chosen": -0.014948742464184761,
|
|
"rewards/margins": -0.01355709508061409,
|
|
"rewards/rejected": -0.0013916483148932457,
|
|
"step": 8
|
|
},
|
|
{
|
|
"epoch": 0.04103090139761508,
|
|
"grad_norm": 67.40204277563684,
|
|
"learning_rate": 1.0204081632653061e-07,
|
|
"logits/chosen": -2.0784454345703125,
|
|
"logits/rejected": -2.072957992553711,
|
|
"logps/chosen": -25.560945510864258,
|
|
"logps/rejected": -31.469083786010742,
|
|
"loss": 0.6968,
|
|
"rewards/accuracies": 0.453125,
|
|
"rewards/chosen": -0.004186006262898445,
|
|
"rewards/margins": 0.010740835219621658,
|
|
"rewards/rejected": -0.014926840551197529,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.04103090139761508,
|
|
"eval_logits/chosen": -2.0721328258514404,
|
|
"eval_logits/rejected": -2.069610834121704,
|
|
"eval_logps/chosen": -24.65077018737793,
|
|
"eval_logps/rejected": -31.145498275756836,
|
|
"eval_loss": 0.6956828236579895,
|
|
"eval_rewards/accuracies": 0.40668201446533203,
|
|
"eval_rewards/chosen": -0.005178123712539673,
|
|
"eval_rewards/margins": -0.0014642790192738175,
|
|
"eval_rewards/rejected": -0.003713843412697315,
|
|
"eval_runtime": 391.6625,
|
|
"eval_samples_per_second": 4.427,
|
|
"eval_steps_per_second": 1.108,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.0492370816771381,
|
|
"grad_norm": 75.23897935562478,
|
|
"learning_rate": 1.2244897959183673e-07,
|
|
"logits/chosen": -2.105541229248047,
|
|
"logits/rejected": -2.110215187072754,
|
|
"logps/chosen": -25.037534713745117,
|
|
"logps/rejected": -51.45681381225586,
|
|
"loss": 0.691,
|
|
"rewards/accuracies": 0.515625,
|
|
"rewards/chosen": -0.02754499390721321,
|
|
"rewards/margins": -0.003392305225133896,
|
|
"rewards/rejected": -0.024152684956789017,
|
|
"step": 12
|
|
},
|
|
{
|
|
"epoch": 0.057443261956661114,
|
|
"grad_norm": 62.15854308442268,
|
|
"learning_rate": 1.4285714285714285e-07,
|
|
"logits/chosen": -2.0920844078063965,
|
|
"logits/rejected": -2.0932013988494873,
|
|
"logps/chosen": -30.580596923828125,
|
|
"logps/rejected": -43.68434143066406,
|
|
"loss": 0.693,
|
|
"rewards/accuracies": 0.578125,
|
|
"rewards/chosen": -0.03211332485079765,
|
|
"rewards/margins": 0.012334156781435013,
|
|
"rewards/rejected": -0.044447485357522964,
|
|
"step": 14
|
|
},
|
|
{
|
|
"epoch": 0.06564944223618413,
|
|
"grad_norm": 70.35846634784929,
|
|
"learning_rate": 1.6326530612244896e-07,
|
|
"logits/chosen": -2.058406114578247,
|
|
"logits/rejected": -2.0612599849700928,
|
|
"logps/chosen": -24.555591583251953,
|
|
"logps/rejected": -47.54515838623047,
|
|
"loss": 0.6833,
|
|
"rewards/accuracies": 0.546875,
|
|
"rewards/chosen": -0.06411759555339813,
|
|
"rewards/margins": 0.02048414759337902,
|
|
"rewards/rejected": -0.0846017450094223,
|
|
"step": 16
|
|
},
|
|
{
|
|
"epoch": 0.07385562251570714,
|
|
"grad_norm": 53.416196200679686,
|
|
"learning_rate": 1.836734693877551e-07,
|
|
"logits/chosen": -2.0910069942474365,
|
|
"logits/rejected": -2.0841658115386963,
|
|
"logps/chosen": -24.07425308227539,
|
|
"logps/rejected": -23.863197326660156,
|
|
"loss": 0.6845,
|
|
"rewards/accuracies": 0.515625,
|
|
"rewards/chosen": -0.09408427029848099,
|
|
"rewards/margins": 0.020395735278725624,
|
|
"rewards/rejected": -0.11447998881340027,
|
|
"step": 18
|
|
},
|
|
{
|
|
"epoch": 0.08206180279523016,
|
|
"grad_norm": 81.67543770250495,
|
|
"learning_rate": 2.0408163265306121e-07,
|
|
"logits/chosen": -2.0422399044036865,
|
|
"logits/rejected": -2.0412869453430176,
|
|
"logps/chosen": -23.748153686523438,
|
|
"logps/rejected": -41.940765380859375,
|
|
"loss": 0.6754,
|
|
"rewards/accuracies": 0.546875,
|
|
"rewards/chosen": -0.08865831792354584,
|
|
"rewards/margins": 0.042564887553453445,
|
|
"rewards/rejected": -0.13122320175170898,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.08206180279523016,
|
|
"eval_logits/chosen": -2.073920726776123,
|
|
"eval_logits/rejected": -2.071418285369873,
|
|
"eval_logps/chosen": -24.912931442260742,
|
|
"eval_logps/rejected": -31.505783081054688,
|
|
"eval_loss": 0.6749772429466248,
|
|
"eval_rewards/accuracies": 0.559907853603363,
|
|
"eval_rewards/chosen": -0.13625794649124146,
|
|
"eval_rewards/margins": 0.04759809002280235,
|
|
"eval_rewards/rejected": -0.1838560253381729,
|
|
"eval_runtime": 388.9647,
|
|
"eval_samples_per_second": 4.458,
|
|
"eval_steps_per_second": 1.116,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.09026798307475317,
|
|
"grad_norm": 58.1111337202034,
|
|
"learning_rate": 2.2448979591836733e-07,
|
|
"logits/chosen": -2.141969680786133,
|
|
"logits/rejected": -2.139296054840088,
|
|
"logps/chosen": -28.556930541992188,
|
|
"logps/rejected": -33.20477294921875,
|
|
"loss": 0.6809,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": -0.1592944860458374,
|
|
"rewards/margins": 0.038385238498449326,
|
|
"rewards/rejected": -0.19767972826957703,
|
|
"step": 22
|
|
},
|
|
{
|
|
"epoch": 0.0984741633542762,
|
|
"grad_norm": 59.55094100468827,
|
|
"learning_rate": 2.4489795918367347e-07,
|
|
"logits/chosen": -2.0954370498657227,
|
|
"logits/rejected": -2.0884788036346436,
|
|
"logps/chosen": -24.139230728149414,
|
|
"logps/rejected": -28.622846603393555,
|
|
"loss": 0.6507,
|
|
"rewards/accuracies": 0.640625,
|
|
"rewards/chosen": -0.26595112681388855,
|
|
"rewards/margins": 0.08658240735530853,
|
|
"rewards/rejected": -0.3525335192680359,
|
|
"step": 24
|
|
},
|
|
{
|
|
"epoch": 0.1066803436337992,
|
|
"grad_norm": 57.19794639459796,
|
|
"learning_rate": 2.653061224489796e-07,
|
|
"logits/chosen": -2.137075662612915,
|
|
"logits/rejected": -2.142050266265869,
|
|
"logps/chosen": -29.15459442138672,
|
|
"logps/rejected": -42.837894439697266,
|
|
"loss": 0.6393,
|
|
"rewards/accuracies": 0.578125,
|
|
"rewards/chosen": -0.3813813328742981,
|
|
"rewards/margins": 0.10138601809740067,
|
|
"rewards/rejected": -0.48276740312576294,
|
|
"step": 26
|
|
},
|
|
{
|
|
"epoch": 0.11488652391332223,
|
|
"grad_norm": 49.34854220939512,
|
|
"learning_rate": 2.857142857142857e-07,
|
|
"logits/chosen": -2.078376531600952,
|
|
"logits/rejected": -2.0755605697631836,
|
|
"logps/chosen": -25.93030548095703,
|
|
"logps/rejected": -38.2554817199707,
|
|
"loss": 0.6617,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": -0.49894633889198303,
|
|
"rewards/margins": 0.13289065659046173,
|
|
"rewards/rejected": -0.631837010383606,
|
|
"step": 28
|
|
},
|
|
{
|
|
"epoch": 0.12309270419284524,
|
|
"grad_norm": 57.358296790419175,
|
|
"learning_rate": 3.0612244897959183e-07,
|
|
"logits/chosen": -2.1170706748962402,
|
|
"logits/rejected": -2.123375415802002,
|
|
"logps/chosen": -21.049701690673828,
|
|
"logps/rejected": -53.651187896728516,
|
|
"loss": 0.6385,
|
|
"rewards/accuracies": 0.546875,
|
|
"rewards/chosen": -0.6054391264915466,
|
|
"rewards/margins": 0.20123842358589172,
|
|
"rewards/rejected": -0.806677520275116,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.12309270419284524,
|
|
"eval_logits/chosen": -2.072676420211792,
|
|
"eval_logits/rejected": -2.0702009201049805,
|
|
"eval_logps/chosen": -25.986181259155273,
|
|
"eval_logps/rejected": -32.866546630859375,
|
|
"eval_loss": 0.6259192824363708,
|
|
"eval_rewards/accuracies": 0.5967742204666138,
|
|
"eval_rewards/chosen": -0.672883927822113,
|
|
"eval_rewards/margins": 0.1913554072380066,
|
|
"eval_rewards/rejected": -0.8642393946647644,
|
|
"eval_runtime": 383.4842,
|
|
"eval_samples_per_second": 4.522,
|
|
"eval_steps_per_second": 1.132,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.13129888447236826,
|
|
"grad_norm": 51.59090790007893,
|
|
"learning_rate": 3.265306122448979e-07,
|
|
"logits/chosen": -2.065584182739258,
|
|
"logits/rejected": -2.059347629547119,
|
|
"logps/chosen": -21.224966049194336,
|
|
"logps/rejected": -31.50775718688965,
|
|
"loss": 0.6031,
|
|
"rewards/accuracies": 0.609375,
|
|
"rewards/chosen": -0.6435321569442749,
|
|
"rewards/margins": 0.2527439296245575,
|
|
"rewards/rejected": -0.8962759971618652,
|
|
"step": 32
|
|
},
|
|
{
|
|
"epoch": 0.13950506475189126,
|
|
"grad_norm": 48.81049837247277,
|
|
"learning_rate": 3.4693877551020406e-07,
|
|
"logits/chosen": -2.044010639190674,
|
|
"logits/rejected": -2.04213285446167,
|
|
"logps/chosen": -21.99913787841797,
|
|
"logps/rejected": -41.94310760498047,
|
|
"loss": 0.5998,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": -0.8395999670028687,
|
|
"rewards/margins": 0.3455100953578949,
|
|
"rewards/rejected": -1.1851099729537964,
|
|
"step": 34
|
|
},
|
|
{
|
|
"epoch": 0.14771124503141428,
|
|
"grad_norm": 51.08931493056027,
|
|
"learning_rate": 3.673469387755102e-07,
|
|
"logits/chosen": -2.0850181579589844,
|
|
"logits/rejected": -2.0847809314727783,
|
|
"logps/chosen": -31.404415130615234,
|
|
"logps/rejected": -38.55030059814453,
|
|
"loss": 0.5904,
|
|
"rewards/accuracies": 0.6875,
|
|
"rewards/chosen": -1.2278621196746826,
|
|
"rewards/margins": 0.26516327261924744,
|
|
"rewards/rejected": -1.493025541305542,
|
|
"step": 36
|
|
},
|
|
{
|
|
"epoch": 0.1559174253109373,
|
|
"grad_norm": 59.35651114893968,
|
|
"learning_rate": 3.877551020408163e-07,
|
|
"logits/chosen": -2.114025831222534,
|
|
"logits/rejected": -2.1199615001678467,
|
|
"logps/chosen": -24.901527404785156,
|
|
"logps/rejected": -50.99317932128906,
|
|
"loss": 0.5923,
|
|
"rewards/accuracies": 0.640625,
|
|
"rewards/chosen": -1.1870838403701782,
|
|
"rewards/margins": 0.41857296228408813,
|
|
"rewards/rejected": -1.6056568622589111,
|
|
"step": 38
|
|
},
|
|
{
|
|
"epoch": 0.16412360559046033,
|
|
"grad_norm": 46.92283583915801,
|
|
"learning_rate": 4.0816326530612243e-07,
|
|
"logits/chosen": -2.109259843826294,
|
|
"logits/rejected": -2.1076252460479736,
|
|
"logps/chosen": -27.03592300415039,
|
|
"logps/rejected": -42.91762161254883,
|
|
"loss": 0.5115,
|
|
"rewards/accuracies": 0.671875,
|
|
"rewards/chosen": -1.1980061531066895,
|
|
"rewards/margins": 0.5265197157859802,
|
|
"rewards/rejected": -1.7245259284973145,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.16412360559046033,
|
|
"eval_logits/chosen": -2.0738561153411865,
|
|
"eval_logits/rejected": -2.0714197158813477,
|
|
"eval_logps/chosen": -27.073177337646484,
|
|
"eval_logps/rejected": -34.44426345825195,
|
|
"eval_loss": 0.5755711197853088,
|
|
"eval_rewards/accuracies": 0.6682027578353882,
|
|
"eval_rewards/chosen": -1.2163803577423096,
|
|
"eval_rewards/margins": 0.43671703338623047,
|
|
"eval_rewards/rejected": -1.6530975103378296,
|
|
"eval_runtime": 383.5104,
|
|
"eval_samples_per_second": 4.521,
|
|
"eval_steps_per_second": 1.132,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.17232978586998332,
|
|
"grad_norm": 53.83140343188913,
|
|
"learning_rate": 4.285714285714285e-07,
|
|
"logits/chosen": -2.0352187156677246,
|
|
"logits/rejected": -2.037541627883911,
|
|
"logps/chosen": -32.00979232788086,
|
|
"logps/rejected": -34.965049743652344,
|
|
"loss": 0.5958,
|
|
"rewards/accuracies": 0.609375,
|
|
"rewards/chosen": -1.6347800493240356,
|
|
"rewards/margins": 0.25150731205940247,
|
|
"rewards/rejected": -1.8862874507904053,
|
|
"step": 42
|
|
},
|
|
{
|
|
"epoch": 0.18053596614950634,
|
|
"grad_norm": 47.972074857166355,
|
|
"learning_rate": 4.4897959183673465e-07,
|
|
"logits/chosen": -2.075744390487671,
|
|
"logits/rejected": -2.080559253692627,
|
|
"logps/chosen": -27.046289443969727,
|
|
"logps/rejected": -43.155860900878906,
|
|
"loss": 0.5475,
|
|
"rewards/accuracies": 0.625,
|
|
"rewards/chosen": -1.4715561866760254,
|
|
"rewards/margins": 0.3980383574962616,
|
|
"rewards/rejected": -1.8695944547653198,
|
|
"step": 44
|
|
},
|
|
{
|
|
"epoch": 0.18874214642902937,
|
|
"grad_norm": 49.55763090785352,
|
|
"learning_rate": 4.693877551020408e-07,
|
|
"logits/chosen": -2.0551533699035645,
|
|
"logits/rejected": -2.0602352619171143,
|
|
"logps/chosen": -33.108123779296875,
|
|
"logps/rejected": -42.747528076171875,
|
|
"loss": 0.5183,
|
|
"rewards/accuracies": 0.640625,
|
|
"rewards/chosen": -1.2049014568328857,
|
|
"rewards/margins": 0.6831133961677551,
|
|
"rewards/rejected": -1.888014793395996,
|
|
"step": 46
|
|
},
|
|
{
|
|
"epoch": 0.1969483267085524,
|
|
"grad_norm": 36.14217009270528,
|
|
"learning_rate": 4.897959183673469e-07,
|
|
"logits/chosen": -2.178698778152466,
|
|
"logits/rejected": -2.178926467895508,
|
|
"logps/chosen": -23.19949722290039,
|
|
"logps/rejected": -47.667236328125,
|
|
"loss": 0.4744,
|
|
"rewards/accuracies": 0.703125,
|
|
"rewards/chosen": -1.3382686376571655,
|
|
"rewards/margins": 0.744179368019104,
|
|
"rewards/rejected": -2.0824477672576904,
|
|
"step": 48
|
|
},
|
|
{
|
|
"epoch": 0.20515450698807539,
|
|
"grad_norm": 44.92936419641385,
|
|
"learning_rate": 4.999935398141225e-07,
|
|
"logits/chosen": -2.0898728370666504,
|
|
"logits/rejected": -2.086491107940674,
|
|
"logps/chosen": -28.257888793945312,
|
|
"logps/rejected": -45.15388870239258,
|
|
"loss": 0.493,
|
|
"rewards/accuracies": 0.796875,
|
|
"rewards/chosen": -1.5506025552749634,
|
|
"rewards/margins": 0.6771446466445923,
|
|
"rewards/rejected": -2.2277474403381348,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.20515450698807539,
|
|
"eval_logits/chosen": -2.0701448917388916,
|
|
"eval_logits/rejected": -2.067728042602539,
|
|
"eval_logps/chosen": -26.948427200317383,
|
|
"eval_logps/rejected": -35.04188919067383,
|
|
"eval_loss": 0.49889206886291504,
|
|
"eval_rewards/accuracies": 0.7396313548088074,
|
|
"eval_rewards/chosen": -1.154005527496338,
|
|
"eval_rewards/margins": 0.797903835773468,
|
|
"eval_rewards/rejected": -1.9519096612930298,
|
|
"eval_runtime": 383.3286,
|
|
"eval_samples_per_second": 4.524,
|
|
"eval_steps_per_second": 1.132,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.2133606872675984,
|
|
"grad_norm": 39.90879769142747,
|
|
"learning_rate": 4.999418603303176e-07,
|
|
"logits/chosen": -2.044602394104004,
|
|
"logits/rejected": -2.043879985809326,
|
|
"logps/chosen": -20.71453094482422,
|
|
"logps/rejected": -39.783756256103516,
|
|
"loss": 0.4715,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.9839560985565186,
|
|
"rewards/margins": 0.9620079398155212,
|
|
"rewards/rejected": -1.9459640979766846,
|
|
"step": 52
|
|
},
|
|
{
|
|
"epoch": 0.22156686754712143,
|
|
"grad_norm": 37.33476295758473,
|
|
"learning_rate": 4.998385120460602e-07,
|
|
"logits/chosen": -2.080956220626831,
|
|
"logits/rejected": -2.0807886123657227,
|
|
"logps/chosen": -26.196861267089844,
|
|
"logps/rejected": -39.63607406616211,
|
|
"loss": 0.51,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -1.140933871269226,
|
|
"rewards/margins": 0.8466982841491699,
|
|
"rewards/rejected": -1.9876320362091064,
|
|
"step": 54
|
|
},
|
|
{
|
|
"epoch": 0.22977304782664446,
|
|
"grad_norm": 45.971175914221135,
|
|
"learning_rate": 4.996835163258461e-07,
|
|
"logits/chosen": -2.098681688308716,
|
|
"logits/rejected": -2.0977959632873535,
|
|
"logps/chosen": -27.12312126159668,
|
|
"logps/rejected": -44.47397994995117,
|
|
"loss": 0.5346,
|
|
"rewards/accuracies": 0.671875,
|
|
"rewards/chosen": -1.1570792198181152,
|
|
"rewards/margins": 0.7375643253326416,
|
|
"rewards/rejected": -1.8946435451507568,
|
|
"step": 56
|
|
},
|
|
{
|
|
"epoch": 0.23797922810616745,
|
|
"grad_norm": 37.460312981010176,
|
|
"learning_rate": 4.994769052108987e-07,
|
|
"logits/chosen": -2.0555946826934814,
|
|
"logits/rejected": -2.0529699325561523,
|
|
"logps/chosen": -23.68335723876953,
|
|
"logps/rejected": -39.27627944946289,
|
|
"loss": 0.3812,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -1.066185712814331,
|
|
"rewards/margins": 1.065974235534668,
|
|
"rewards/rejected": -2.13215970993042,
|
|
"step": 58
|
|
},
|
|
{
|
|
"epoch": 0.24618540838569047,
|
|
"grad_norm": 34.303283256725926,
|
|
"learning_rate": 4.992187214125447e-07,
|
|
"logits/chosen": -2.10715651512146,
|
|
"logits/rejected": -2.1082205772399902,
|
|
"logps/chosen": -23.896142959594727,
|
|
"logps/rejected": -50.47355270385742,
|
|
"loss": 0.3707,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -0.9780565500259399,
|
|
"rewards/margins": 1.4628773927688599,
|
|
"rewards/rejected": -2.4409339427948,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.24618540838569047,
|
|
"eval_logits/chosen": -2.0698177814483643,
|
|
"eval_logits/rejected": -2.0673775672912598,
|
|
"eval_logps/chosen": -26.22644805908203,
|
|
"eval_logps/rejected": -34.93669128417969,
|
|
"eval_loss": 0.44575488567352295,
|
|
"eval_rewards/accuracies": 0.7695852518081665,
|
|
"eval_rewards/chosen": -0.7930165529251099,
|
|
"eval_rewards/margins": 1.106292963027954,
|
|
"eval_rewards/rejected": -1.8993093967437744,
|
|
"eval_runtime": 383.2498,
|
|
"eval_samples_per_second": 4.524,
|
|
"eval_steps_per_second": 1.132,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.2543915886652135,
|
|
"grad_norm": 34.73750630342744,
|
|
"learning_rate": 4.98909018303385e-07,
|
|
"logits/chosen": -2.096700429916382,
|
|
"logits/rejected": -2.0969526767730713,
|
|
"logps/chosen": -28.021564483642578,
|
|
"logps/rejected": -44.515228271484375,
|
|
"loss": 0.4361,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -0.700480580329895,
|
|
"rewards/margins": 1.3456040620803833,
|
|
"rewards/rejected": -2.046084403991699,
|
|
"step": 62
|
|
},
|
|
{
|
|
"epoch": 0.2625977689447365,
|
|
"grad_norm": 32.81062577153495,
|
|
"learning_rate": 4.985478599062611e-07,
|
|
"logits/chosen": -2.087684154510498,
|
|
"logits/rejected": -2.088871717453003,
|
|
"logps/chosen": -26.94491958618164,
|
|
"logps/rejected": -34.76374435424805,
|
|
"loss": 0.4479,
|
|
"rewards/accuracies": 0.734375,
|
|
"rewards/chosen": -0.528798520565033,
|
|
"rewards/margins": 0.9001777172088623,
|
|
"rewards/rejected": -1.42897629737854,
|
|
"step": 64
|
|
},
|
|
{
|
|
"epoch": 0.27080394922425954,
|
|
"grad_norm": 29.052321436937024,
|
|
"learning_rate": 4.981353208810206e-07,
|
|
"logits/chosen": -2.0988636016845703,
|
|
"logits/rejected": -2.104152202606201,
|
|
"logps/chosen": -20.718381881713867,
|
|
"logps/rejected": -51.553409576416016,
|
|
"loss": 0.4029,
|
|
"rewards/accuracies": 0.828125,
|
|
"rewards/chosen": -0.7514103055000305,
|
|
"rewards/margins": 1.5007989406585693,
|
|
"rewards/rejected": -2.252209186553955,
|
|
"step": 66
|
|
},
|
|
{
|
|
"epoch": 0.2790101295037825,
|
|
"grad_norm": 37.90898283460309,
|
|
"learning_rate": 4.976714865090826e-07,
|
|
"logits/chosen": -2.0126729011535645,
|
|
"logits/rejected": -2.007772445678711,
|
|
"logps/chosen": -27.219005584716797,
|
|
"logps/rejected": -26.84417152404785,
|
|
"loss": 0.45,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.7038444876670837,
|
|
"rewards/margins": 0.9012500047683716,
|
|
"rewards/rejected": -1.6050945520401,
|
|
"step": 68
|
|
},
|
|
{
|
|
"epoch": 0.28721630978330553,
|
|
"grad_norm": 31.01097368955644,
|
|
"learning_rate": 4.971564526758087e-07,
|
|
"logits/chosen": -2.0806331634521484,
|
|
"logits/rejected": -2.090015411376953,
|
|
"logps/chosen": -26.418888092041016,
|
|
"logps/rejected": -61.58074951171875,
|
|
"loss": 0.3921,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.6639503240585327,
|
|
"rewards/margins": 2.018562078475952,
|
|
"rewards/rejected": -2.6825122833251953,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.28721630978330553,
|
|
"eval_logits/chosen": -2.0629022121429443,
|
|
"eval_logits/rejected": -2.0604288578033447,
|
|
"eval_logps/chosen": -25.214706420898438,
|
|
"eval_logps/rejected": -34.3575553894043,
|
|
"eval_loss": 0.4037153124809265,
|
|
"eval_rewards/accuracies": 0.7776497602462769,
|
|
"eval_rewards/chosen": -0.2871449589729309,
|
|
"eval_rewards/margins": 1.322598934173584,
|
|
"eval_rewards/rejected": -1.6097438335418701,
|
|
"eval_runtime": 383.2946,
|
|
"eval_samples_per_second": 4.524,
|
|
"eval_steps_per_second": 1.132,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.29542249006282856,
|
|
"grad_norm": 35.944172042258,
|
|
"learning_rate": 4.965903258506806e-07,
|
|
"logits/chosen": -1.982763409614563,
|
|
"logits/rejected": -1.9825458526611328,
|
|
"logps/chosen": -26.120790481567383,
|
|
"logps/rejected": -47.724666595458984,
|
|
"loss": 0.3965,
|
|
"rewards/accuracies": 0.765625,
|
|
"rewards/chosen": -0.5441683530807495,
|
|
"rewards/margins": 1.4394950866699219,
|
|
"rewards/rejected": -1.9836633205413818,
|
|
"step": 72
|
|
},
|
|
{
|
|
"epoch": 0.3036286703423516,
|
|
"grad_norm": 30.076414397329636,
|
|
"learning_rate": 4.959732230652907e-07,
|
|
"logits/chosen": -2.0936832427978516,
|
|
"logits/rejected": -2.0875070095062256,
|
|
"logps/chosen": -21.951000213623047,
|
|
"logps/rejected": -35.42924499511719,
|
|
"loss": 0.3786,
|
|
"rewards/accuracies": 0.828125,
|
|
"rewards/chosen": -0.6129394769668579,
|
|
"rewards/margins": 1.359676718711853,
|
|
"rewards/rejected": -1.9726160764694214,
|
|
"step": 74
|
|
},
|
|
{
|
|
"epoch": 0.3118348506218746,
|
|
"grad_norm": 31.92687509033969,
|
|
"learning_rate": 4.953052718891494e-07,
|
|
"logits/chosen": -2.0927772521972656,
|
|
"logits/rejected": -2.0937466621398926,
|
|
"logps/chosen": -24.76601791381836,
|
|
"logps/rejected": -40.51527786254883,
|
|
"loss": 0.3999,
|
|
"rewards/accuracies": 0.765625,
|
|
"rewards/chosen": -0.39051324129104614,
|
|
"rewards/margins": 1.7075785398483276,
|
|
"rewards/rejected": -2.0980916023254395,
|
|
"step": 76
|
|
},
|
|
{
|
|
"epoch": 0.3200410309013976,
|
|
"grad_norm": 39.97329898507408,
|
|
"learning_rate": 4.945866104033126e-07,
|
|
"logits/chosen": -2.07487416267395,
|
|
"logits/rejected": -2.0698864459991455,
|
|
"logps/chosen": -28.91614532470703,
|
|
"logps/rejected": -32.64835739135742,
|
|
"loss": 0.3902,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.8651710152626038,
|
|
"rewards/margins": 1.3195717334747314,
|
|
"rewards/rejected": -2.1847424507141113,
|
|
"step": 78
|
|
},
|
|
{
|
|
"epoch": 0.32824721118092065,
|
|
"grad_norm": 35.23935142184965,
|
|
"learning_rate": 4.938173871718379e-07,
|
|
"logits/chosen": -2.1075708866119385,
|
|
"logits/rejected": -2.108260154724121,
|
|
"logps/chosen": -30.674692153930664,
|
|
"logps/rejected": -45.75242233276367,
|
|
"loss": 0.4197,
|
|
"rewards/accuracies": 0.765625,
|
|
"rewards/chosen": -0.8922288417816162,
|
|
"rewards/margins": 1.6468966007232666,
|
|
"rewards/rejected": -2.5391252040863037,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.32824721118092065,
|
|
"eval_logits/chosen": -2.0568766593933105,
|
|
"eval_logits/rejected": -2.0545151233673096,
|
|
"eval_logps/chosen": -26.29727554321289,
|
|
"eval_logps/rejected": -36.041969299316406,
|
|
"eval_loss": 0.3652815520763397,
|
|
"eval_rewards/accuracies": 0.7937787771224976,
|
|
"eval_rewards/chosen": -0.8284297585487366,
|
|
"eval_rewards/margins": 1.6235177516937256,
|
|
"eval_rewards/rejected": -2.4519472122192383,
|
|
"eval_runtime": 383.0876,
|
|
"eval_samples_per_second": 4.526,
|
|
"eval_steps_per_second": 1.133,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.3364533914604437,
|
|
"grad_norm": 27.98699458231919,
|
|
"learning_rate": 4.929977612110723e-07,
|
|
"logits/chosen": -2.0166802406311035,
|
|
"logits/rejected": -2.009007215499878,
|
|
"logps/chosen": -27.796003341674805,
|
|
"logps/rejected": -37.83144760131836,
|
|
"loss": 0.3548,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -1.278414249420166,
|
|
"rewards/margins": 1.387697696685791,
|
|
"rewards/rejected": -2.666111946105957,
|
|
"step": 82
|
|
},
|
|
{
|
|
"epoch": 0.34465957173996664,
|
|
"grad_norm": 29.909892482214758,
|
|
"learning_rate": 4.921279019567806e-07,
|
|
"logits/chosen": -2.060328483581543,
|
|
"logits/rejected": -2.0567514896392822,
|
|
"logps/chosen": -22.3341064453125,
|
|
"logps/rejected": -27.066940307617188,
|
|
"loss": 0.3357,
|
|
"rewards/accuracies": 0.828125,
|
|
"rewards/chosen": -1.3317604064941406,
|
|
"rewards/margins": 1.3481371402740479,
|
|
"rewards/rejected": -2.6798975467681885,
|
|
"step": 84
|
|
},
|
|
{
|
|
"epoch": 0.35286575201948966,
|
|
"grad_norm": 28.894675574314675,
|
|
"learning_rate": 4.912079892291184e-07,
|
|
"logits/chosen": -2.0049288272857666,
|
|
"logits/rejected": -2.0062108039855957,
|
|
"logps/chosen": -30.222366333007812,
|
|
"logps/rejected": -37.637451171875,
|
|
"loss": 0.3523,
|
|
"rewards/accuracies": 0.890625,
|
|
"rewards/chosen": -1.1566193103790283,
|
|
"rewards/margins": 1.645595669746399,
|
|
"rewards/rejected": -2.8022148609161377,
|
|
"step": 86
|
|
},
|
|
{
|
|
"epoch": 0.3610719322990127,
|
|
"grad_norm": 33.499254927144385,
|
|
"learning_rate": 4.902382131954594e-07,
|
|
"logits/chosen": -2.0440762042999268,
|
|
"logits/rejected": -2.041065216064453,
|
|
"logps/chosen": -28.95600700378418,
|
|
"logps/rejected": -32.45362091064453,
|
|
"loss": 0.3969,
|
|
"rewards/accuracies": 0.859375,
|
|
"rewards/chosen": -1.4144505262374878,
|
|
"rewards/margins": 1.526777982711792,
|
|
"rewards/rejected": -2.9412283897399902,
|
|
"step": 88
|
|
},
|
|
{
|
|
"epoch": 0.3692781125785357,
|
|
"grad_norm": 36.672895433055324,
|
|
"learning_rate": 4.892187743310834e-07,
|
|
"logits/chosen": -2.1121721267700195,
|
|
"logits/rejected": -2.1142220497131348,
|
|
"logps/chosen": -33.17682647705078,
|
|
"logps/rejected": -55.22991180419922,
|
|
"loss": 0.3216,
|
|
"rewards/accuracies": 0.859375,
|
|
"rewards/chosen": -0.8327025175094604,
|
|
"rewards/margins": 2.4360129833221436,
|
|
"rewards/rejected": -3.2687156200408936,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.3692781125785357,
|
|
"eval_logits/chosen": -2.052072048187256,
|
|
"eval_logits/rejected": -2.0496795177459717,
|
|
"eval_logps/chosen": -25.994354248046875,
|
|
"eval_logps/rejected": -35.990543365478516,
|
|
"eval_loss": 0.33759820461273193,
|
|
"eval_rewards/accuracies": 0.8029953837394714,
|
|
"eval_rewards/chosen": -0.6769699454307556,
|
|
"eval_rewards/margins": 1.7492659091949463,
|
|
"eval_rewards/rejected": -2.426236152648926,
|
|
"eval_runtime": 382.9061,
|
|
"eval_samples_per_second": 4.529,
|
|
"eval_steps_per_second": 1.133,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.37748429285805873,
|
|
"grad_norm": 27.350771381278054,
|
|
"learning_rate": 4.881498833777333e-07,
|
|
"logits/chosen": -2.1142969131469727,
|
|
"logits/rejected": -2.1127796173095703,
|
|
"logps/chosen": -27.94344711303711,
|
|
"logps/rejected": -36.454994201660156,
|
|
"loss": 0.3337,
|
|
"rewards/accuracies": 0.828125,
|
|
"rewards/chosen": -0.4834941327571869,
|
|
"rewards/margins": 1.6755785942077637,
|
|
"rewards/rejected": -2.1590728759765625,
|
|
"step": 92
|
|
},
|
|
{
|
|
"epoch": 0.38569047313758176,
|
|
"grad_norm": 21.635416471698683,
|
|
"learning_rate": 4.870317613000496e-07,
|
|
"logits/chosen": -2.016702890396118,
|
|
"logits/rejected": -2.017988681793213,
|
|
"logps/chosen": -29.998384475708008,
|
|
"logps/rejected": -37.61553192138672,
|
|
"loss": 0.3289,
|
|
"rewards/accuracies": 0.890625,
|
|
"rewards/chosen": -0.3838534653186798,
|
|
"rewards/margins": 1.9279775619506836,
|
|
"rewards/rejected": -2.311830997467041,
|
|
"step": 94
|
|
},
|
|
{
|
|
"epoch": 0.3938966534171048,
|
|
"grad_norm": 32.26464274668422,
|
|
"learning_rate": 4.858646392398927e-07,
|
|
"logits/chosen": -2.1012330055236816,
|
|
"logits/rejected": -2.1029739379882812,
|
|
"logps/chosen": -24.2839298248291,
|
|
"logps/rejected": -52.811363220214844,
|
|
"loss": 0.3288,
|
|
"rewards/accuracies": 0.765625,
|
|
"rewards/chosen": -0.1970413625240326,
|
|
"rewards/margins": 2.4278342723846436,
|
|
"rewards/rejected": -2.624875545501709,
|
|
"step": 96
|
|
},
|
|
{
|
|
"epoch": 0.40210283369662775,
|
|
"grad_norm": 24.11098061020347,
|
|
"learning_rate": 4.846487584685594e-07,
|
|
"logits/chosen": -2.0802366733551025,
|
|
"logits/rejected": -2.073582649230957,
|
|
"logps/chosen": -24.776100158691406,
|
|
"logps/rejected": -30.90226936340332,
|
|
"loss": 0.3069,
|
|
"rewards/accuracies": 0.796875,
|
|
"rewards/chosen": -0.5730211734771729,
|
|
"rewards/margins": 1.4600439071655273,
|
|
"rewards/rejected": -2.0330650806427,
|
|
"step": 98
|
|
},
|
|
{
|
|
"epoch": 0.41030901397615077,
|
|
"grad_norm": 24.386233800584957,
|
|
"learning_rate": 4.833843703369075e-07,
|
|
"logits/chosen": -2.056903839111328,
|
|
"logits/rejected": -2.0560696125030518,
|
|
"logps/chosen": -25.943012237548828,
|
|
"logps/rejected": -47.84954071044922,
|
|
"loss": 0.2756,
|
|
"rewards/accuracies": 0.890625,
|
|
"rewards/chosen": -0.26147836446762085,
|
|
"rewards/margins": 2.6134910583496094,
|
|
"rewards/rejected": -2.874969482421875,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.41030901397615077,
|
|
"eval_logits/chosen": -2.048205852508545,
|
|
"eval_logits/rejected": -2.0459136962890625,
|
|
"eval_logps/chosen": -25.047216415405273,
|
|
"eval_logps/rejected": -35.27001953125,
|
|
"eval_loss": 0.3195771872997284,
|
|
"eval_rewards/accuracies": 0.804147481918335,
|
|
"eval_rewards/chosen": -0.20340144634246826,
|
|
"eval_rewards/margins": 1.8625727891921997,
|
|
"eval_rewards/rejected": -2.065974235534668,
|
|
"eval_runtime": 383.1037,
|
|
"eval_samples_per_second": 4.526,
|
|
"eval_steps_per_second": 1.133,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.4185151942556738,
|
|
"grad_norm": 20.26348597387277,
|
|
"learning_rate": 4.82071736223395e-07,
|
|
"logits/chosen": -2.0507657527923584,
|
|
"logits/rejected": -2.044565200805664,
|
|
"logps/chosen": -23.989782333374023,
|
|
"logps/rejected": -31.029855728149414,
|
|
"loss": 0.2811,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -0.2594943940639496,
|
|
"rewards/margins": 1.7268887758255005,
|
|
"rewards/rejected": -1.9863829612731934,
|
|
"step": 102
|
|
},
|
|
{
|
|
"epoch": 0.4267213745351968,
|
|
"grad_norm": 19.535827187227056,
|
|
"learning_rate": 4.807111274800475e-07,
|
|
"logits/chosen": -2.0995054244995117,
|
|
"logits/rejected": -2.091494083404541,
|
|
"logps/chosen": -20.125661849975586,
|
|
"logps/rejected": -26.041656494140625,
|
|
"loss": 0.2717,
|
|
"rewards/accuracies": 0.765625,
|
|
"rewards/chosen": -0.21148215234279633,
|
|
"rewards/margins": 1.814781665802002,
|
|
"rewards/rejected": -2.02626371383667,
|
|
"step": 104
|
|
},
|
|
{
|
|
"epoch": 0.43492755481471984,
|
|
"grad_norm": 33.73160324782582,
|
|
"learning_rate": 4.793028253763632e-07,
|
|
"logits/chosen": -2.0714871883392334,
|
|
"logits/rejected": -2.073652744293213,
|
|
"logps/chosen": -29.02633285522461,
|
|
"logps/rejected": -45.91259765625,
|
|
"loss": 0.3285,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -0.4463399350643158,
|
|
"rewards/margins": 2.3804402351379395,
|
|
"rewards/rejected": -2.826780080795288,
|
|
"step": 106
|
|
},
|
|
{
|
|
"epoch": 0.44313373509424286,
|
|
"grad_norm": 27.500884484426393,
|
|
"learning_rate": 4.778471210411683e-07,
|
|
"logits/chosen": -2.1328721046447754,
|
|
"logits/rejected": -2.128079414367676,
|
|
"logps/chosen": -26.338151931762695,
|
|
"logps/rejected": -39.00925064086914,
|
|
"loss": 0.2889,
|
|
"rewards/accuracies": 0.859375,
|
|
"rewards/chosen": -0.20531219244003296,
|
|
"rewards/margins": 2.1164348125457764,
|
|
"rewards/rejected": -2.321747064590454,
|
|
"step": 108
|
|
},
|
|
{
|
|
"epoch": 0.4513399153737659,
|
|
"grad_norm": 19.395398725174257,
|
|
"learning_rate": 4.763443154024334e-07,
|
|
"logits/chosen": -2.010795831680298,
|
|
"logits/rejected": -2.004256010055542,
|
|
"logps/chosen": -21.2674560546875,
|
|
"logps/rejected": -29.27115249633789,
|
|
"loss": 0.2402,
|
|
"rewards/accuracies": 0.828125,
|
|
"rewards/chosen": -0.35923194885253906,
|
|
"rewards/margins": 2.286337375640869,
|
|
"rewards/rejected": -2.645569324493408,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.4513399153737659,
|
|
"eval_logits/chosen": -2.047652244567871,
|
|
"eval_logits/rejected": -2.0454225540161133,
|
|
"eval_logps/chosen": -25.388172149658203,
|
|
"eval_logps/rejected": -36.19011306762695,
|
|
"eval_loss": 0.29904791712760925,
|
|
"eval_rewards/accuracies": 0.8064516186714172,
|
|
"eval_rewards/chosen": -0.37387850880622864,
|
|
"eval_rewards/margins": 2.152141571044922,
|
|
"eval_rewards/rejected": -2.526020050048828,
|
|
"eval_runtime": 382.9313,
|
|
"eval_samples_per_second": 4.528,
|
|
"eval_steps_per_second": 1.133,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.4595460956532889,
|
|
"grad_norm": 19.214229030497684,
|
|
"learning_rate": 4.74794719125065e-07,
|
|
"logits/chosen": -2.0643627643585205,
|
|
"logits/rejected": -2.0607378482818604,
|
|
"logps/chosen": -28.60050392150879,
|
|
"logps/rejected": -30.703428268432617,
|
|
"loss": 0.2887,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -0.28185003995895386,
|
|
"rewards/margins": 1.8889483213424683,
|
|
"rewards/rejected": -2.1707985401153564,
|
|
"step": 112
|
|
},
|
|
{
|
|
"epoch": 0.4677522759328119,
|
|
"grad_norm": 26.967072996600148,
|
|
"learning_rate": 4.731986525466836e-07,
|
|
"logits/chosen": -2.0277886390686035,
|
|
"logits/rejected": -2.0253236293792725,
|
|
"logps/chosen": -28.50118064880371,
|
|
"logps/rejected": -39.31520462036133,
|
|
"loss": 0.2738,
|
|
"rewards/accuracies": 0.765625,
|
|
"rewards/chosen": -0.7399206757545471,
|
|
"rewards/margins": 2.0028390884399414,
|
|
"rewards/rejected": -2.7427597045898438,
|
|
"step": 114
|
|
},
|
|
{
|
|
"epoch": 0.4759584562123349,
|
|
"grad_norm": 23.78521498468923,
|
|
"learning_rate": 4.7155644561140293e-07,
|
|
"logits/chosen": -2.0169668197631836,
|
|
"logits/rejected": -2.0144312381744385,
|
|
"logps/chosen": -17.960556030273438,
|
|
"logps/rejected": -44.15462875366211,
|
|
"loss": 0.308,
|
|
"rewards/accuracies": 0.796875,
|
|
"rewards/chosen": -0.5678286552429199,
|
|
"rewards/margins": 2.510011672973633,
|
|
"rewards/rejected": -3.077840805053711,
|
|
"step": 116
|
|
},
|
|
{
|
|
"epoch": 0.4841646364918579,
|
|
"grad_norm": 24.124459622722974,
|
|
"learning_rate": 4.698684378016222e-07,
|
|
"logits/chosen": -1.9945940971374512,
|
|
"logits/rejected": -1.9939186573028564,
|
|
"logps/chosen": -27.118621826171875,
|
|
"logps/rejected": -47.901485443115234,
|
|
"loss": 0.3078,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -0.848014235496521,
|
|
"rewards/margins": 2.41487455368042,
|
|
"rewards/rejected": -3.2628891468048096,
|
|
"step": 118
|
|
},
|
|
{
|
|
"epoch": 0.49237081677138095,
|
|
"grad_norm": 21.029226154824478,
|
|
"learning_rate": 4.681349780678478e-07,
|
|
"logits/chosen": -2.069658041000366,
|
|
"logits/rejected": -2.066652774810791,
|
|
"logps/chosen": -19.98298454284668,
|
|
"logps/rejected": -35.198486328125,
|
|
"loss": 0.2684,
|
|
"rewards/accuracies": 0.828125,
|
|
"rewards/chosen": -0.4263419508934021,
|
|
"rewards/margins": 2.379479169845581,
|
|
"rewards/rejected": -2.805821418762207,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.49237081677138095,
|
|
"eval_logits/chosen": -2.0455994606018066,
|
|
"eval_logits/rejected": -2.0433595180511475,
|
|
"eval_logps/chosen": -25.59235954284668,
|
|
"eval_logps/rejected": -36.88100051879883,
|
|
"eval_loss": 0.2864527106285095,
|
|
"eval_rewards/accuracies": 0.8122119903564453,
|
|
"eval_rewards/chosen": -0.47597208619117737,
|
|
"eval_rewards/margins": 2.3954925537109375,
|
|
"eval_rewards/rejected": -2.871464729309082,
|
|
"eval_runtime": 382.8408,
|
|
"eval_samples_per_second": 4.529,
|
|
"eval_steps_per_second": 1.134,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.500576997050904,
|
|
"grad_norm": 22.882841908986034,
|
|
"learning_rate": 4.6635642475655643e-07,
|
|
"logits/chosen": -2.0903265476226807,
|
|
"logits/rejected": -2.091627836227417,
|
|
"logps/chosen": -27.477466583251953,
|
|
"logps/rejected": -39.44534683227539,
|
|
"loss": 0.3338,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.455081045627594,
|
|
"rewards/margins": 2.4550111293792725,
|
|
"rewards/rejected": -2.9100921154022217,
|
|
"step": 122
|
|
},
|
|
{
|
|
"epoch": 0.508783177330427,
|
|
"grad_norm": 25.08346996002156,
|
|
"learning_rate": 4.6453314553611724e-07,
|
|
"logits/chosen": -2.058607816696167,
|
|
"logits/rejected": -2.0590226650238037,
|
|
"logps/chosen": -26.310922622680664,
|
|
"logps/rejected": -41.810604095458984,
|
|
"loss": 0.3438,
|
|
"rewards/accuracies": 0.828125,
|
|
"rewards/chosen": -0.7647377848625183,
|
|
"rewards/margins": 2.2874107360839844,
|
|
"rewards/rejected": -3.0521485805511475,
|
|
"step": 124
|
|
},
|
|
{
|
|
"epoch": 0.51698935760995,
|
|
"grad_norm": 19.245560055238066,
|
|
"learning_rate": 4.626655173207856e-07,
|
|
"logits/chosen": -2.0874435901641846,
|
|
"logits/rejected": -2.0808544158935547,
|
|
"logps/chosen": -34.59534454345703,
|
|
"logps/rejected": -37.33736801147461,
|
|
"loss": 0.2242,
|
|
"rewards/accuracies": 0.890625,
|
|
"rewards/chosen": -0.46927499771118164,
|
|
"rewards/margins": 2.6450273990631104,
|
|
"rewards/rejected": -3.114302396774292,
|
|
"step": 126
|
|
},
|
|
{
|
|
"epoch": 0.525195537889473,
|
|
"grad_norm": 26.78927560536762,
|
|
"learning_rate": 4.607539261927868e-07,
|
|
"logits/chosen": -2.038646936416626,
|
|
"logits/rejected": -2.036555290222168,
|
|
"logps/chosen": -25.151229858398438,
|
|
"logps/rejected": -41.385738372802734,
|
|
"loss": 0.258,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -0.7142379879951477,
|
|
"rewards/margins": 2.4793665409088135,
|
|
"rewards/rejected": -3.1936047077178955,
|
|
"step": 128
|
|
},
|
|
{
|
|
"epoch": 0.5334017181689961,
|
|
"grad_norm": 28.211129877400257,
|
|
"learning_rate": 4.587987673225031e-07,
|
|
"logits/chosen": -2.0317800045013428,
|
|
"logits/rejected": -2.029090642929077,
|
|
"logps/chosen": -23.62847137451172,
|
|
"logps/rejected": -41.40718460083008,
|
|
"loss": 0.314,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.7062588930130005,
|
|
"rewards/margins": 2.260392904281616,
|
|
"rewards/rejected": -2.9666519165039062,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.5334017181689961,
|
|
"eval_logits/chosen": -2.046064615249634,
|
|
"eval_logits/rejected": -2.0438313484191895,
|
|
"eval_logps/chosen": -24.87871742248535,
|
|
"eval_logps/rejected": -36.286964416503906,
|
|
"eval_loss": 0.2757984399795532,
|
|
"eval_rewards/accuracies": 0.8225806355476379,
|
|
"eval_rewards/chosen": -0.11915161460638046,
|
|
"eval_rewards/margins": 2.4552958011627197,
|
|
"eval_rewards/rejected": -2.5744473934173584,
|
|
"eval_runtime": 382.9112,
|
|
"eval_samples_per_second": 4.528,
|
|
"eval_steps_per_second": 1.133,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.5416078984485191,
|
|
"grad_norm": 19.634258213686227,
|
|
"learning_rate": 4.568004448867836e-07,
|
|
"logits/chosen": -1.9866001605987549,
|
|
"logits/rejected": -1.983536720275879,
|
|
"logps/chosen": -31.62383460998535,
|
|
"logps/rejected": -39.09525680541992,
|
|
"loss": 0.2353,
|
|
"rewards/accuracies": 0.828125,
|
|
"rewards/chosen": -0.31665635108947754,
|
|
"rewards/margins": 2.6990866661071777,
|
|
"rewards/rejected": -3.0157430171966553,
|
|
"step": 132
|
|
},
|
|
{
|
|
"epoch": 0.5498140787280421,
|
|
"grad_norm": 18.53508760026935,
|
|
"learning_rate": 4.547593719853908e-07,
|
|
"logits/chosen": -2.117701530456543,
|
|
"logits/rejected": -2.1072885990142822,
|
|
"logps/chosen": -18.488718032836914,
|
|
"logps/rejected": -28.42927360534668,
|
|
"loss": 0.2533,
|
|
"rewards/accuracies": 0.796875,
|
|
"rewards/chosen": -0.2543734908103943,
|
|
"rewards/margins": 1.9832816123962402,
|
|
"rewards/rejected": -2.2376551628112793,
|
|
"step": 134
|
|
},
|
|
{
|
|
"epoch": 0.558020259007565,
|
|
"grad_norm": 16.806992595132815,
|
|
"learning_rate": 4.526759705556037e-07,
|
|
"logits/chosen": -2.1042609214782715,
|
|
"logits/rejected": -2.106780767440796,
|
|
"logps/chosen": -25.879810333251953,
|
|
"logps/rejected": -48.42536163330078,
|
|
"loss": 0.1949,
|
|
"rewards/accuracies": 0.828125,
|
|
"rewards/chosen": 0.10162217915058136,
|
|
"rewards/margins": 3.190650224685669,
|
|
"rewards/rejected": -3.0890283584594727,
|
|
"step": 136
|
|
},
|
|
{
|
|
"epoch": 0.566226439287088,
|
|
"grad_norm": 29.13623442121838,
|
|
"learning_rate": 4.5055067128499336e-07,
|
|
"logits/chosen": -2.1067519187927246,
|
|
"logits/rejected": -2.0996322631835938,
|
|
"logps/chosen": -24.035615921020508,
|
|
"logps/rejected": -33.068504333496094,
|
|
"loss": 0.3053,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -0.14914892613887787,
|
|
"rewards/margins": 2.2678146362304688,
|
|
"rewards/rejected": -2.4169633388519287,
|
|
"step": 138
|
|
},
|
|
{
|
|
"epoch": 0.5744326195666111,
|
|
"grad_norm": 14.892524709426025,
|
|
"learning_rate": 4.483839135223899e-07,
|
|
"logits/chosen": -2.0495522022247314,
|
|
"logits/rejected": -2.049654960632324,
|
|
"logps/chosen": -22.117572784423828,
|
|
"logps/rejected": -45.10372543334961,
|
|
"loss": 0.213,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -0.1762857437133789,
|
|
"rewards/margins": 3.072883129119873,
|
|
"rewards/rejected": -3.249169111251831,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.5744326195666111,
|
|
"eval_logits/chosen": -2.043985605239868,
|
|
"eval_logits/rejected": -2.041775703430176,
|
|
"eval_logps/chosen": -24.56585693359375,
|
|
"eval_logps/rejected": -36.1696662902832,
|
|
"eval_loss": 0.26956623792648315,
|
|
"eval_rewards/accuracies": 0.8214285969734192,
|
|
"eval_rewards/chosen": 0.03727945312857628,
|
|
"eval_rewards/margins": 2.55307674407959,
|
|
"eval_rewards/rejected": -2.5157971382141113,
|
|
"eval_runtime": 382.8923,
|
|
"eval_samples_per_second": 4.529,
|
|
"eval_steps_per_second": 1.133,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.5826387998461341,
|
|
"grad_norm": 26.214083971395798,
|
|
"learning_rate": 4.461761451870586e-07,
|
|
"logits/chosen": -2.0746335983276367,
|
|
"logits/rejected": -2.0725696086883545,
|
|
"logps/chosen": -28.98033332824707,
|
|
"logps/rejected": -33.45741653442383,
|
|
"loss": 0.2844,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.3142589330673218,
|
|
"rewards/margins": 2.325613498687744,
|
|
"rewards/rejected": -2.6398725509643555,
|
|
"step": 142
|
|
},
|
|
{
|
|
"epoch": 0.5908449801256571,
|
|
"grad_norm": 19.801373156192714,
|
|
"learning_rate": 4.4392782267610495e-07,
|
|
"logits/chosen": -2.026793956756592,
|
|
"logits/rejected": -2.02146315574646,
|
|
"logps/chosen": -27.090116500854492,
|
|
"logps/rejected": -36.820072174072266,
|
|
"loss": 0.2405,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 0.05273272842168808,
|
|
"rewards/margins": 2.6351218223571777,
|
|
"rewards/rejected": -2.5823891162872314,
|
|
"step": 144
|
|
},
|
|
{
|
|
"epoch": 0.5990511604051801,
|
|
"grad_norm": 16.912833507875153,
|
|
"learning_rate": 4.416394107701263e-07,
|
|
"logits/chosen": -2.0721688270568848,
|
|
"logits/rejected": -2.0710277557373047,
|
|
"logps/chosen": -29.055322647094727,
|
|
"logps/rejected": -35.73569107055664,
|
|
"loss": 0.22,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.04920143634080887,
|
|
"rewards/margins": 2.790663957595825,
|
|
"rewards/rejected": -2.7414627075195312,
|
|
"step": 146
|
|
},
|
|
{
|
|
"epoch": 0.6072573406847032,
|
|
"grad_norm": 18.48986000965074,
|
|
"learning_rate": 4.393113825371312e-07,
|
|
"logits/chosen": -2.088228225708008,
|
|
"logits/rejected": -2.0804646015167236,
|
|
"logps/chosen": -22.193986892700195,
|
|
"logps/rejected": -28.600019454956055,
|
|
"loss": 0.2584,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -0.23887695372104645,
|
|
"rewards/margins": 2.352029323577881,
|
|
"rewards/rejected": -2.5909059047698975,
|
|
"step": 148
|
|
},
|
|
{
|
|
"epoch": 0.6154635209642262,
|
|
"grad_norm": 20.195764906566346,
|
|
"learning_rate": 4.3694421923474523e-07,
|
|
"logits/chosen": -2.0307724475860596,
|
|
"logits/rejected": -2.0267581939697266,
|
|
"logps/chosen": -25.377315521240234,
|
|
"logps/rejected": -26.69452667236328,
|
|
"loss": 0.2466,
|
|
"rewards/accuracies": 0.859375,
|
|
"rewards/chosen": 0.02581462636590004,
|
|
"rewards/margins": 2.615370512008667,
|
|
"rewards/rejected": -2.5895559787750244,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.6154635209642262,
|
|
"eval_logits/chosen": -2.0484068393707275,
|
|
"eval_logits/rejected": -2.046243906021118,
|
|
"eval_logps/chosen": -24.641366958618164,
|
|
"eval_logps/rejected": -36.52920150756836,
|
|
"eval_loss": 0.2627074420452118,
|
|
"eval_rewards/accuracies": 0.8179723620414734,
|
|
"eval_rewards/chosen": -0.00047618892858736217,
|
|
"eval_rewards/margins": 2.6950886249542236,
|
|
"eval_rewards/rejected": -2.6955649852752686,
|
|
"eval_runtime": 383.0874,
|
|
"eval_samples_per_second": 4.526,
|
|
"eval_steps_per_second": 1.133,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.6236697012437492,
|
|
"grad_norm": 19.84271445630352,
|
|
"learning_rate": 4.3453841021072367e-07,
|
|
"logits/chosen": -2.0682671070098877,
|
|
"logits/rejected": -2.0678606033325195,
|
|
"logps/chosen": -29.11172866821289,
|
|
"logps/rejected": -43.93864059448242,
|
|
"loss": 0.2393,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -0.21894294023513794,
|
|
"rewards/margins": 2.8336007595062256,
|
|
"rewards/rejected": -3.0525436401367188,
|
|
"step": 152
|
|
},
|
|
{
|
|
"epoch": 0.6318758815232722,
|
|
"grad_norm": 23.803208123467336,
|
|
"learning_rate": 4.32094452801792e-07,
|
|
"logits/chosen": -2.120655059814453,
|
|
"logits/rejected": -2.113632917404175,
|
|
"logps/chosen": -23.940746307373047,
|
|
"logps/rejected": -40.54450988769531,
|
|
"loss": 0.2101,
|
|
"rewards/accuracies": 0.890625,
|
|
"rewards/chosen": 0.22472774982452393,
|
|
"rewards/margins": 3.2544140815734863,
|
|
"rewards/rejected": -3.029686450958252,
|
|
"step": 154
|
|
},
|
|
{
|
|
"epoch": 0.6400820618027953,
|
|
"grad_norm": 27.831688353015224,
|
|
"learning_rate": 4.29612852230835e-07,
|
|
"logits/chosen": -2.1510727405548096,
|
|
"logits/rejected": -2.1424570083618164,
|
|
"logps/chosen": -29.35157012939453,
|
|
"logps/rejected": -39.825775146484375,
|
|
"loss": 0.3136,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -0.25963878631591797,
|
|
"rewards/margins": 2.4727556705474854,
|
|
"rewards/rejected": -2.7323946952819824,
|
|
"step": 156
|
|
},
|
|
{
|
|
"epoch": 0.6482882420823183,
|
|
"grad_norm": 16.129032160167743,
|
|
"learning_rate": 4.270941215024551e-07,
|
|
"logits/chosen": -2.0342090129852295,
|
|
"logits/rejected": -2.030592918395996,
|
|
"logps/chosen": -25.4481201171875,
|
|
"logps/rejected": -30.034128189086914,
|
|
"loss": 0.2383,
|
|
"rewards/accuracies": 0.859375,
|
|
"rewards/chosen": -0.3485499918460846,
|
|
"rewards/margins": 2.4843502044677734,
|
|
"rewards/rejected": -2.832900047302246,
|
|
"step": 158
|
|
},
|
|
{
|
|
"epoch": 0.6564944223618413,
|
|
"grad_norm": 22.363661877277583,
|
|
"learning_rate": 4.2453878129692257e-07,
|
|
"logits/chosen": -2.0489790439605713,
|
|
"logits/rejected": -2.0505762100219727,
|
|
"logps/chosen": -20.61872673034668,
|
|
"logps/rejected": -39.31790542602539,
|
|
"loss": 0.236,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -0.36551517248153687,
|
|
"rewards/margins": 2.786931037902832,
|
|
"rewards/rejected": -3.1524462699890137,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.6564944223618413,
|
|
"eval_logits/chosen": -2.048050880432129,
|
|
"eval_logits/rejected": -2.0458953380584717,
|
|
"eval_logps/chosen": -25.27681541442871,
|
|
"eval_logps/rejected": -37.62437438964844,
|
|
"eval_loss": 0.2534183859825134,
|
|
"eval_rewards/accuracies": 0.8260368704795837,
|
|
"eval_rewards/chosen": -0.31819990277290344,
|
|
"eval_rewards/margins": 2.9249520301818848,
|
|
"eval_rewards/rejected": -3.243151903152466,
|
|
"eval_runtime": 383.1245,
|
|
"eval_samples_per_second": 4.526,
|
|
"eval_steps_per_second": 1.133,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.6647006026413643,
|
|
"grad_norm": 16.96720912691383,
|
|
"learning_rate": 4.2194735986253894e-07,
|
|
"logits/chosen": -2.0488674640655518,
|
|
"logits/rejected": -2.04327130317688,
|
|
"logps/chosen": -29.53526496887207,
|
|
"logps/rejected": -29.405216217041016,
|
|
"loss": 0.2198,
|
|
"rewards/accuracies": 0.890625,
|
|
"rewards/chosen": -0.7364621162414551,
|
|
"rewards/margins": 2.2279775142669678,
|
|
"rewards/rejected": -2.9644393920898438,
|
|
"step": 162
|
|
},
|
|
{
|
|
"epoch": 0.6729067829208873,
|
|
"grad_norm": 21.306381619736708,
|
|
"learning_rate": 4.193203929064353e-07,
|
|
"logits/chosen": -2.098163366317749,
|
|
"logits/rejected": -2.09775710105896,
|
|
"logps/chosen": -27.31165313720703,
|
|
"logps/rejected": -49.05406951904297,
|
|
"loss": 0.2626,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": -0.6689335107803345,
|
|
"rewards/margins": 3.3616225719451904,
|
|
"rewards/rejected": -4.030555725097656,
|
|
"step": 164
|
|
},
|
|
{
|
|
"epoch": 0.6811129632004103,
|
|
"grad_norm": 16.744872149563246,
|
|
"learning_rate": 4.1665842348382974e-07,
|
|
"logits/chosen": -2.076477289199829,
|
|
"logits/rejected": -2.078810453414917,
|
|
"logps/chosen": -21.170326232910156,
|
|
"logps/rejected": -47.55445098876953,
|
|
"loss": 0.2274,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -0.5238662958145142,
|
|
"rewards/margins": 3.466320514678955,
|
|
"rewards/rejected": -3.9901866912841797,
|
|
"step": 166
|
|
},
|
|
{
|
|
"epoch": 0.6893191434799333,
|
|
"grad_norm": 14.185305520407544,
|
|
"learning_rate": 4.139620018857648e-07,
|
|
"logits/chosen": -2.003122091293335,
|
|
"logits/rejected": -2.000248908996582,
|
|
"logps/chosen": -22.870935440063477,
|
|
"logps/rejected": -39.74565505981445,
|
|
"loss": 0.2638,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.6974708437919617,
|
|
"rewards/margins": 2.8220083713531494,
|
|
"rewards/rejected": -3.519479513168335,
|
|
"step": 168
|
|
},
|
|
{
|
|
"epoch": 0.6975253237594563,
|
|
"grad_norm": 18.022997094570332,
|
|
"learning_rate": 4.1123168552534983e-07,
|
|
"logits/chosen": -2.025099277496338,
|
|
"logits/rejected": -2.0219833850860596,
|
|
"logps/chosen": -28.584171295166016,
|
|
"logps/rejected": -38.055633544921875,
|
|
"loss": 0.2503,
|
|
"rewards/accuracies": 0.859375,
|
|
"rewards/chosen": -0.9480264186859131,
|
|
"rewards/margins": 2.6562623977661133,
|
|
"rewards/rejected": -3.6042890548706055,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.6975253237594563,
|
|
"eval_logits/chosen": -2.047826051712036,
|
|
"eval_logits/rejected": -2.0456759929656982,
|
|
"eval_logps/chosen": -25.28836441040039,
|
|
"eval_logps/rejected": -37.87636184692383,
|
|
"eval_loss": 0.2495637983083725,
|
|
"eval_rewards/accuracies": 0.8271889686584473,
|
|
"eval_rewards/chosen": -0.32397639751434326,
|
|
"eval_rewards/margins": 3.0451700687408447,
|
|
"eval_rewards/rejected": -3.3691465854644775,
|
|
"eval_runtime": 383.2321,
|
|
"eval_samples_per_second": 4.525,
|
|
"eval_steps_per_second": 1.132,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.7057315040389793,
|
|
"grad_norm": 30.201529803311132,
|
|
"learning_rate": 4.084680388225302e-07,
|
|
"logits/chosen": -2.0860209465026855,
|
|
"logits/rejected": -2.0855419635772705,
|
|
"logps/chosen": -25.88998031616211,
|
|
"logps/rejected": -35.96123504638672,
|
|
"loss": 0.2004,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -0.12960225343704224,
|
|
"rewards/margins": 3.401216745376587,
|
|
"rewards/rejected": -3.5308187007904053,
|
|
"step": 172
|
|
},
|
|
{
|
|
"epoch": 0.7139376843185024,
|
|
"grad_norm": 13.521536443635728,
|
|
"learning_rate": 4.0567163308740925e-07,
|
|
"logits/chosen": -2.061061143875122,
|
|
"logits/rejected": -2.0628788471221924,
|
|
"logps/chosen": -28.78243637084961,
|
|
"logps/rejected": -46.390567779541016,
|
|
"loss": 0.1995,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -0.46569639444351196,
|
|
"rewards/margins": 3.489924430847168,
|
|
"rewards/rejected": -3.9556210041046143,
|
|
"step": 174
|
|
},
|
|
{
|
|
"epoch": 0.7221438645980254,
|
|
"grad_norm": 12.241119865111436,
|
|
"learning_rate": 4.028430464021445e-07,
|
|
"logits/chosen": -2.0593080520629883,
|
|
"logits/rejected": -2.0476720333099365,
|
|
"logps/chosen": -25.62759780883789,
|
|
"logps/rejected": -28.63144302368164,
|
|
"loss": 0.2311,
|
|
"rewards/accuracies": 0.828125,
|
|
"rewards/chosen": -0.2619730532169342,
|
|
"rewards/margins": 2.6522488594055176,
|
|
"rewards/rejected": -2.91422176361084,
|
|
"step": 176
|
|
},
|
|
{
|
|
"epoch": 0.7303500448775484,
|
|
"grad_norm": 19.69681362187802,
|
|
"learning_rate": 3.9998286350144517e-07,
|
|
"logits/chosen": -2.127859592437744,
|
|
"logits/rejected": -2.124366283416748,
|
|
"logps/chosen": -27.991050720214844,
|
|
"logps/rejected": -32.49789810180664,
|
|
"loss": 0.1931,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -0.4937363862991333,
|
|
"rewards/margins": 2.826824188232422,
|
|
"rewards/rejected": -3.3205604553222656,
|
|
"step": 178
|
|
},
|
|
{
|
|
"epoch": 0.7385562251570714,
|
|
"grad_norm": 18.465447655230893,
|
|
"learning_rate": 3.970916756516936e-07,
|
|
"logits/chosen": -2.0612165927886963,
|
|
"logits/rejected": -2.0634703636169434,
|
|
"logps/chosen": -31.92318344116211,
|
|
"logps/rejected": -47.82712173461914,
|
|
"loss": 0.1869,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -0.8367382287979126,
|
|
"rewards/margins": 3.4318110942840576,
|
|
"rewards/rejected": -4.26854944229126,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.7385562251570714,
|
|
"eval_logits/chosen": -2.047463893890381,
|
|
"eval_logits/rejected": -2.0453338623046875,
|
|
"eval_logps/chosen": -25.764009475708008,
|
|
"eval_logps/rejected": -38.632667541503906,
|
|
"eval_loss": 0.24548786878585815,
|
|
"eval_rewards/accuracies": 0.8248847723007202,
|
|
"eval_rewards/chosen": -0.5617985725402832,
|
|
"eval_rewards/margins": 3.185500383377075,
|
|
"eval_rewards/rejected": -3.7472991943359375,
|
|
"eval_runtime": 383.1735,
|
|
"eval_samples_per_second": 4.525,
|
|
"eval_steps_per_second": 1.133,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.7467624054365944,
|
|
"grad_norm": 28.11697685801471,
|
|
"learning_rate": 3.941700805287168e-07,
|
|
"logits/chosen": -2.100497245788574,
|
|
"logits/rejected": -2.0966544151306152,
|
|
"logps/chosen": -22.02035140991211,
|
|
"logps/rejected": -34.0717658996582,
|
|
"loss": 0.2677,
|
|
"rewards/accuracies": 0.796875,
|
|
"rewards/chosen": -0.21246086061000824,
|
|
"rewards/margins": 2.8322882652282715,
|
|
"rewards/rejected": -3.0447492599487305,
|
|
"step": 182
|
|
},
|
|
{
|
|
"epoch": 0.7549685857161175,
|
|
"grad_norm": 21.185462762042597,
|
|
"learning_rate": 3.912186820942329e-07,
|
|
"logits/chosen": -2.07576847076416,
|
|
"logits/rejected": -2.076803684234619,
|
|
"logps/chosen": -28.578994750976562,
|
|
"logps/rejected": -53.69257354736328,
|
|
"loss": 0.2491,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.507973849773407,
|
|
"rewards/margins": 4.095354080200195,
|
|
"rewards/rejected": -4.603327751159668,
|
|
"step": 184
|
|
},
|
|
{
|
|
"epoch": 0.7631747659956405,
|
|
"grad_norm": 15.467370455908538,
|
|
"learning_rate": 3.8823809047099844e-07,
|
|
"logits/chosen": -2.091827630996704,
|
|
"logits/rejected": -2.0913453102111816,
|
|
"logps/chosen": -22.403928756713867,
|
|
"logps/rejected": -48.54361343383789,
|
|
"loss": 0.214,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -0.5807015895843506,
|
|
"rewards/margins": 3.9955556392669678,
|
|
"rewards/rejected": -4.57625675201416,
|
|
"step": 186
|
|
},
|
|
{
|
|
"epoch": 0.7713809462751635,
|
|
"grad_norm": 21.449176063404654,
|
|
"learning_rate": 3.8522892181668145e-07,
|
|
"logits/chosen": -2.073751449584961,
|
|
"logits/rejected": -2.0747601985931396,
|
|
"logps/chosen": -25.900894165039062,
|
|
"logps/rejected": -46.20444107055664,
|
|
"loss": 0.2199,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.7566344141960144,
|
|
"rewards/margins": 3.398766040802002,
|
|
"rewards/rejected": -4.15540075302124,
|
|
"step": 188
|
|
},
|
|
{
|
|
"epoch": 0.7795871265546865,
|
|
"grad_norm": 17.55165479512368,
|
|
"learning_rate": 3.821917981964873e-07,
|
|
"logits/chosen": -2.0060808658599854,
|
|
"logits/rejected": -2.0017528533935547,
|
|
"logps/chosen": -22.145849227905273,
|
|
"logps/rejected": -36.78887939453125,
|
|
"loss": 0.2346,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -0.17031463980674744,
|
|
"rewards/margins": 2.9387521743774414,
|
|
"rewards/rejected": -3.1090667247772217,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.7795871265546865,
|
|
"eval_logits/chosen": -2.0468969345092773,
|
|
"eval_logits/rejected": -2.044738292694092,
|
|
"eval_logps/chosen": -24.78700065612793,
|
|
"eval_logps/rejected": -37.58818054199219,
|
|
"eval_loss": 0.2414015829563141,
|
|
"eval_rewards/accuracies": 0.8329492807388306,
|
|
"eval_rewards/chosen": -0.07329300791025162,
|
|
"eval_rewards/margins": 3.1517624855041504,
|
|
"eval_rewards/rejected": -3.225055456161499,
|
|
"eval_runtime": 383.2411,
|
|
"eval_samples_per_second": 4.525,
|
|
"eval_steps_per_second": 1.132,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.7877933068342096,
|
|
"grad_norm": 17.368985354953686,
|
|
"learning_rate": 3.79127347454564e-07,
|
|
"logits/chosen": -2.0460448265075684,
|
|
"logits/rejected": -2.0370802879333496,
|
|
"logps/chosen": -26.563547134399414,
|
|
"logps/rejected": -38.11757278442383,
|
|
"loss": 0.2211,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -0.37561261653900146,
|
|
"rewards/margins": 2.6861519813537598,
|
|
"rewards/rejected": -3.0617644786834717,
|
|
"step": 192
|
|
},
|
|
{
|
|
"epoch": 0.7959994871137326,
|
|
"grad_norm": 24.935734742960335,
|
|
"learning_rate": 3.760362030842113e-07,
|
|
"logits/chosen": -2.0645647048950195,
|
|
"logits/rejected": -2.0566651821136475,
|
|
"logps/chosen": -25.446151733398438,
|
|
"logps/rejected": -29.0418758392334,
|
|
"loss": 0.2469,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -0.27131855487823486,
|
|
"rewards/margins": 2.571014165878296,
|
|
"rewards/rejected": -2.842332124710083,
|
|
"step": 194
|
|
},
|
|
{
|
|
"epoch": 0.8042056673932555,
|
|
"grad_norm": 30.86901898716295,
|
|
"learning_rate": 3.7291900409692346e-07,
|
|
"logits/chosen": -2.1032328605651855,
|
|
"logits/rejected": -2.1033365726470947,
|
|
"logps/chosen": -25.201629638671875,
|
|
"logps/rejected": -39.323097229003906,
|
|
"loss": 0.2264,
|
|
"rewards/accuracies": 0.828125,
|
|
"rewards/chosen": -0.08912299573421478,
|
|
"rewards/margins": 3.298488140106201,
|
|
"rewards/rejected": -3.387611150741577,
|
|
"step": 196
|
|
},
|
|
{
|
|
"epoch": 0.8124118476727785,
|
|
"grad_norm": 21.758022770529394,
|
|
"learning_rate": 3.6977639489029056e-07,
|
|
"logits/chosen": -2.0734946727752686,
|
|
"logits/rejected": -2.0731263160705566,
|
|
"logps/chosen": -23.40597915649414,
|
|
"logps/rejected": -54.18486022949219,
|
|
"loss": 0.2027,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.06441287696361542,
|
|
"rewards/margins": 3.9613521099090576,
|
|
"rewards/rejected": -3.896939277648926,
|
|
"step": 198
|
|
},
|
|
{
|
|
"epoch": 0.8206180279523015,
|
|
"grad_norm": 24.294166941995968,
|
|
"learning_rate": 3.666090251147864e-07,
|
|
"logits/chosen": -2.0556201934814453,
|
|
"logits/rejected": -2.056826114654541,
|
|
"logps/chosen": -22.517078399658203,
|
|
"logps/rejected": -50.51237487792969,
|
|
"loss": 0.2229,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -0.15975169837474823,
|
|
"rewards/margins": 4.061436176300049,
|
|
"rewards/rejected": -4.221188068389893,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.8206180279523015,
|
|
"eval_logits/chosen": -2.0442752838134766,
|
|
"eval_logits/rejected": -2.0421106815338135,
|
|
"eval_logps/chosen": -24.409955978393555,
|
|
"eval_logps/rejected": -37.24555206298828,
|
|
"eval_loss": 0.23997244238853455,
|
|
"eval_rewards/accuracies": 0.8341013789176941,
|
|
"eval_rewards/chosen": 0.1152293011546135,
|
|
"eval_rewards/margins": 3.168970823287964,
|
|
"eval_rewards/rejected": -3.053741216659546,
|
|
"eval_runtime": 383.2231,
|
|
"eval_samples_per_second": 4.525,
|
|
"eval_steps_per_second": 1.132,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.8288242082318246,
|
|
"grad_norm": 16.34097836225302,
|
|
"learning_rate": 3.6341754953947074e-07,
|
|
"logits/chosen": -1.9979509115219116,
|
|
"logits/rejected": -1.9910781383514404,
|
|
"logps/chosen": -19.41321563720703,
|
|
"logps/rejected": -31.22482681274414,
|
|
"loss": 0.1755,
|
|
"rewards/accuracies": 0.859375,
|
|
"rewards/chosen": 0.12338539958000183,
|
|
"rewards/margins": 3.176830291748047,
|
|
"rewards/rejected": -3.0534448623657227,
|
|
"step": 202
|
|
},
|
|
{
|
|
"epoch": 0.8370303885113476,
|
|
"grad_norm": 12.694366662014156,
|
|
"learning_rate": 3.6020262791663334e-07,
|
|
"logits/chosen": -2.0971240997314453,
|
|
"logits/rejected": -2.098155975341797,
|
|
"logps/chosen": -28.991458892822266,
|
|
"logps/rejected": -45.771728515625,
|
|
"loss": 0.1939,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 0.10578002035617828,
|
|
"rewards/margins": 3.6688270568847656,
|
|
"rewards/rejected": -3.563046932220459,
|
|
"step": 204
|
|
},
|
|
{
|
|
"epoch": 0.8452365687908706,
|
|
"grad_norm": 14.104950698059591,
|
|
"learning_rate": 3.569649248454077e-07,
|
|
"logits/chosen": -2.0265486240386963,
|
|
"logits/rejected": -2.023569107055664,
|
|
"logps/chosen": -19.130268096923828,
|
|
"logps/rejected": -45.600608825683594,
|
|
"loss": 0.1853,
|
|
"rewards/accuracies": 0.921875,
|
|
"rewards/chosen": 0.10110447555780411,
|
|
"rewards/margins": 3.614145040512085,
|
|
"rewards/rejected": -3.513040542602539,
|
|
"step": 206
|
|
},
|
|
{
|
|
"epoch": 0.8534427490703936,
|
|
"grad_norm": 17.487336280708007,
|
|
"learning_rate": 3.53705109634383e-07,
|
|
"logits/chosen": -2.0617876052856445,
|
|
"logits/rejected": -2.059702157974243,
|
|
"logps/chosen": -22.653236389160156,
|
|
"logps/rejected": -33.451725006103516,
|
|
"loss": 0.2036,
|
|
"rewards/accuracies": 0.859375,
|
|
"rewards/chosen": -0.21188408136367798,
|
|
"rewards/margins": 2.7427585124969482,
|
|
"rewards/rejected": -2.9546422958374023,
|
|
"step": 208
|
|
},
|
|
{
|
|
"epoch": 0.8616489293499167,
|
|
"grad_norm": 9.723870774955257,
|
|
"learning_rate": 3.5042385616324236e-07,
|
|
"logits/chosen": -2.030812978744507,
|
|
"logits/rejected": -2.0279781818389893,
|
|
"logps/chosen": -24.24312973022461,
|
|
"logps/rejected": -41.24238204956055,
|
|
"loss": 0.1987,
|
|
"rewards/accuracies": 0.859375,
|
|
"rewards/chosen": -0.33003199100494385,
|
|
"rewards/margins": 3.4097635746002197,
|
|
"rewards/rejected": -3.739795684814453,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.8616489293499167,
|
|
"eval_logits/chosen": -2.044862985610962,
|
|
"eval_logits/rejected": -2.042698621749878,
|
|
"eval_logps/chosen": -24.65873146057129,
|
|
"eval_logps/rejected": -37.76519012451172,
|
|
"eval_loss": 0.23567576706409454,
|
|
"eval_rewards/accuracies": 0.8375576138496399,
|
|
"eval_rewards/chosen": -0.009159128181636333,
|
|
"eval_rewards/margins": 3.304400682449341,
|
|
"eval_rewards/rejected": -3.3135595321655273,
|
|
"eval_runtime": 383.2234,
|
|
"eval_samples_per_second": 4.525,
|
|
"eval_steps_per_second": 1.132,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.8698551096294397,
|
|
"grad_norm": 21.492517186963823,
|
|
"learning_rate": 3.471218427434564e-07,
|
|
"logits/chosen": -2.080448865890503,
|
|
"logits/rejected": -2.077627182006836,
|
|
"logps/chosen": -28.046445846557617,
|
|
"logps/rejected": -39.041099548339844,
|
|
"loss": 0.2058,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -0.2396007478237152,
|
|
"rewards/margins": 3.291498899459839,
|
|
"rewards/rejected": -3.531099557876587,
|
|
"step": 212
|
|
},
|
|
{
|
|
"epoch": 0.8780612899089627,
|
|
"grad_norm": 13.459743982109678,
|
|
"learning_rate": 3.4379975197806025e-07,
|
|
"logits/chosen": -2.0183067321777344,
|
|
"logits/rejected": -2.0159990787506104,
|
|
"logps/chosen": -30.474184036254883,
|
|
"logps/rejected": -43.47400665283203,
|
|
"loss": 0.1623,
|
|
"rewards/accuracies": 0.890625,
|
|
"rewards/chosen": -0.49888333678245544,
|
|
"rewards/margins": 3.724670886993408,
|
|
"rewards/rejected": -4.2235541343688965,
|
|
"step": 214
|
|
},
|
|
{
|
|
"epoch": 0.8862674701884857,
|
|
"grad_norm": 21.39701909954645,
|
|
"learning_rate": 3.404582706205438e-07,
|
|
"logits/chosen": -2.0548624992370605,
|
|
"logits/rejected": -2.0510313510894775,
|
|
"logps/chosen": -25.710582733154297,
|
|
"logps/rejected": -47.18820571899414,
|
|
"loss": 0.2419,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -0.6691687107086182,
|
|
"rewards/margins": 3.2068655490875244,
|
|
"rewards/rejected": -3.8760342597961426,
|
|
"step": 216
|
|
},
|
|
{
|
|
"epoch": 0.8944736504680088,
|
|
"grad_norm": 15.742259895322631,
|
|
"learning_rate": 3.370980894328836e-07,
|
|
"logits/chosen": -2.0632827281951904,
|
|
"logits/rejected": -2.0607898235321045,
|
|
"logps/chosen": -29.016246795654297,
|
|
"logps/rejected": -29.224483489990234,
|
|
"loss": 0.231,
|
|
"rewards/accuracies": 0.859375,
|
|
"rewards/chosen": -0.02165369875729084,
|
|
"rewards/margins": 2.8292150497436523,
|
|
"rewards/rejected": -2.8508687019348145,
|
|
"step": 218
|
|
},
|
|
{
|
|
"epoch": 0.9026798307475318,
|
|
"grad_norm": 15.763481233552632,
|
|
"learning_rate": 3.337199030427465e-07,
|
|
"logits/chosen": -2.0306642055511475,
|
|
"logits/rejected": -2.0277583599090576,
|
|
"logps/chosen": -24.812223434448242,
|
|
"logps/rejected": -42.606937408447266,
|
|
"loss": 0.2242,
|
|
"rewards/accuracies": 0.796875,
|
|
"rewards/chosen": -0.5038759112358093,
|
|
"rewards/margins": 3.393310070037842,
|
|
"rewards/rejected": -3.897186040878296,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.9026798307475318,
|
|
"eval_logits/chosen": -2.0421266555786133,
|
|
"eval_logits/rejected": -2.0399889945983887,
|
|
"eval_logps/chosen": -24.929790496826172,
|
|
"eval_logps/rejected": -38.282798767089844,
|
|
"eval_loss": 0.23096635937690735,
|
|
"eval_rewards/accuracies": 0.8398617506027222,
|
|
"eval_rewards/chosen": -0.144687682390213,
|
|
"eval_rewards/margins": 3.4276747703552246,
|
|
"eval_rewards/rejected": -3.5723624229431152,
|
|
"eval_runtime": 383.2137,
|
|
"eval_samples_per_second": 4.525,
|
|
"eval_steps_per_second": 1.133,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.9108860110270548,
|
|
"grad_norm": 21.58130767879211,
|
|
"learning_rate": 3.303244097998937e-07,
|
|
"logits/chosen": -2.059748888015747,
|
|
"logits/rejected": -2.055283546447754,
|
|
"logps/chosen": -28.7065372467041,
|
|
"logps/rejected": -32.10883331298828,
|
|
"loss": 0.2483,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -0.3254169821739197,
|
|
"rewards/margins": 2.9877233505249023,
|
|
"rewards/rejected": -3.313140392303467,
|
|
"step": 222
|
|
},
|
|
{
|
|
"epoch": 0.9190921913065778,
|
|
"grad_norm": 22.43367663307141,
|
|
"learning_rate": 3.2691231163181577e-07,
|
|
"logits/chosen": -2.03068470954895,
|
|
"logits/rejected": -2.029240608215332,
|
|
"logps/chosen": -30.380455017089844,
|
|
"logps/rejected": -51.02494430541992,
|
|
"loss": 0.2287,
|
|
"rewards/accuracies": 0.828125,
|
|
"rewards/chosen": -0.5234890580177307,
|
|
"rewards/margins": 3.933694362640381,
|
|
"rewards/rejected": -4.457183361053467,
|
|
"step": 224
|
|
},
|
|
{
|
|
"epoch": 0.9272983715861007,
|
|
"grad_norm": 18.763839267544366,
|
|
"learning_rate": 3.2348431389862775e-07,
|
|
"logits/chosen": -2.0352532863616943,
|
|
"logits/rejected": -2.0309362411499023,
|
|
"logps/chosen": -26.602890014648438,
|
|
"logps/rejected": -50.593746185302734,
|
|
"loss": 0.2013,
|
|
"rewards/accuracies": 0.890625,
|
|
"rewards/chosen": -0.3150665760040283,
|
|
"rewards/margins": 4.31168270111084,
|
|
"rewards/rejected": -4.626749515533447,
|
|
"step": 226
|
|
},
|
|
{
|
|
"epoch": 0.9355045518656238,
|
|
"grad_norm": 12.546473609091086,
|
|
"learning_rate": 3.2004112524725485e-07,
|
|
"logits/chosen": -2.102660894393921,
|
|
"logits/rejected": -2.098243474960327,
|
|
"logps/chosen": -18.035892486572266,
|
|
"logps/rejected": -29.32090187072754,
|
|
"loss": 0.22,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -0.6185750365257263,
|
|
"rewards/margins": 2.496063232421875,
|
|
"rewards/rejected": -3.114638328552246,
|
|
"step": 228
|
|
},
|
|
{
|
|
"epoch": 0.9437107321451468,
|
|
"grad_norm": 15.34640763010342,
|
|
"learning_rate": 3.16583457464939e-07,
|
|
"logits/chosen": -2.1066555976867676,
|
|
"logits/rejected": -2.102388381958008,
|
|
"logps/chosen": -22.569570541381836,
|
|
"logps/rejected": -35.02971267700195,
|
|
"loss": 0.2468,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 0.08270835876464844,
|
|
"rewards/margins": 3.306253433227539,
|
|
"rewards/rejected": -3.2235450744628906,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.9437107321451468,
|
|
"eval_logits/chosen": -2.0403828620910645,
|
|
"eval_logits/rejected": -2.0383219718933105,
|
|
"eval_logps/chosen": -25.11418342590332,
|
|
"eval_logps/rejected": -38.591522216796875,
|
|
"eval_loss": 0.22682493925094604,
|
|
"eval_rewards/accuracies": 0.8364055156707764,
|
|
"eval_rewards/chosen": -0.23688402771949768,
|
|
"eval_rewards/margins": 3.4898440837860107,
|
|
"eval_rewards/rejected": -3.7267279624938965,
|
|
"eval_runtime": 383.2213,
|
|
"eval_samples_per_second": 4.525,
|
|
"eval_steps_per_second": 1.133,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.9519169124246698,
|
|
"grad_norm": 20.317292755951325,
|
|
"learning_rate": 3.1311202533209516e-07,
|
|
"logits/chosen": -2.066338300704956,
|
|
"logits/rejected": -2.0640790462493896,
|
|
"logps/chosen": -18.971635818481445,
|
|
"logps/rejected": -37.304718017578125,
|
|
"loss": 0.2079,
|
|
"rewards/accuracies": 0.78125,
|
|
"rewards/chosen": -0.2257971614599228,
|
|
"rewards/margins": 3.128026008605957,
|
|
"rewards/rejected": -3.353823184967041,
|
|
"step": 232
|
|
},
|
|
{
|
|
"epoch": 0.9601230927041928,
|
|
"grad_norm": 21.602603549490688,
|
|
"learning_rate": 3.096275464745501e-07,
|
|
"logits/chosen": -2.0534098148345947,
|
|
"logits/rejected": -2.059464931488037,
|
|
"logps/chosen": -27.803630828857422,
|
|
"logps/rejected": -51.089698791503906,
|
|
"loss": 0.1848,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -0.1761884093284607,
|
|
"rewards/margins": 3.8570480346679688,
|
|
"rewards/rejected": -4.033236503601074,
|
|
"step": 234
|
|
},
|
|
{
|
|
"epoch": 0.9683292729837158,
|
|
"grad_norm": 25.9781205793325,
|
|
"learning_rate": 3.061307412151922e-07,
|
|
"logits/chosen": -2.063019037246704,
|
|
"logits/rejected": -2.0604867935180664,
|
|
"logps/chosen": -30.086076736450195,
|
|
"logps/rejected": -38.32453536987305,
|
|
"loss": 0.2619,
|
|
"rewards/accuracies": 0.796875,
|
|
"rewards/chosen": -0.35281646251678467,
|
|
"rewards/margins": 3.2657690048217773,
|
|
"rewards/rejected": -3.6185855865478516,
|
|
"step": 236
|
|
},
|
|
{
|
|
"epoch": 0.9765354532632389,
|
|
"grad_norm": 21.253354056577106,
|
|
"learning_rate": 3.0262233242506414e-07,
|
|
"logits/chosen": -2.0784614086151123,
|
|
"logits/rejected": -2.0770914554595947,
|
|
"logps/chosen": -21.169662475585938,
|
|
"logps/rejected": -38.09800720214844,
|
|
"loss": 0.2181,
|
|
"rewards/accuracies": 0.890625,
|
|
"rewards/chosen": 0.11809039115905762,
|
|
"rewards/margins": 3.477996587753296,
|
|
"rewards/rejected": -3.359905958175659,
|
|
"step": 238
|
|
},
|
|
{
|
|
"epoch": 0.9847416335427619,
|
|
"grad_norm": 19.6268618703958,
|
|
"learning_rate": 2.9910304537392837e-07,
|
|
"logits/chosen": -1.9946519136428833,
|
|
"logits/rejected": -1.9943212270736694,
|
|
"logps/chosen": -25.262042999267578,
|
|
"logps/rejected": -37.486366271972656,
|
|
"loss": 0.218,
|
|
"rewards/accuracies": 0.796875,
|
|
"rewards/chosen": -0.47168225049972534,
|
|
"rewards/margins": 3.64669132232666,
|
|
"rewards/rejected": -4.118373394012451,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.9847416335427619,
|
|
"eval_logits/chosen": -2.043826103210449,
|
|
"eval_logits/rejected": -2.0417795181274414,
|
|
"eval_logps/chosen": -24.943058013916016,
|
|
"eval_logps/rejected": -38.50017166137695,
|
|
"eval_loss": 0.22486171126365662,
|
|
"eval_rewards/accuracies": 0.8387096524238586,
|
|
"eval_rewards/chosen": -0.1513207107782364,
|
|
"eval_rewards/margins": 3.5297298431396484,
|
|
"eval_rewards/rejected": -3.6810505390167236,
|
|
"eval_runtime": 383.2373,
|
|
"eval_samples_per_second": 4.525,
|
|
"eval_steps_per_second": 1.132,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.9929478138222849,
|
|
"grad_norm": 14.682865028202697,
|
|
"learning_rate": 2.955736075803371e-07,
|
|
"logits/chosen": -2.0690362453460693,
|
|
"logits/rejected": -2.070676326751709,
|
|
"logps/chosen": -24.48103904724121,
|
|
"logps/rejected": -47.019752502441406,
|
|
"loss": 0.1536,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 0.06840521097183228,
|
|
"rewards/margins": 4.232391357421875,
|
|
"rewards/rejected": -4.163986682891846,
|
|
"step": 242
|
|
},
|
|
{
|
|
"epoch": 1.001153994101808,
|
|
"grad_norm": 23.69289209855517,
|
|
"learning_rate": 2.9203474866123756e-07,
|
|
"logits/chosen": -2.050558090209961,
|
|
"logits/rejected": -2.045598030090332,
|
|
"logps/chosen": -26.37921142578125,
|
|
"logps/rejected": -40.620906829833984,
|
|
"loss": 0.2308,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -0.41915470361709595,
|
|
"rewards/margins": 3.705007314682007,
|
|
"rewards/rejected": -4.124162197113037,
|
|
"step": 244
|
|
},
|
|
{
|
|
"epoch": 1.009360174381331,
|
|
"grad_norm": 17.446487056986353,
|
|
"learning_rate": 2.884872001811425e-07,
|
|
"logits/chosen": -1.9810688495635986,
|
|
"logits/rejected": -1.9750444889068604,
|
|
"logps/chosen": -21.791378021240234,
|
|
"logps/rejected": -34.808170318603516,
|
|
"loss": 0.1961,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -0.5856488943099976,
|
|
"rewards/margins": 3.6117939949035645,
|
|
"rewards/rejected": -4.197443008422852,
|
|
"step": 246
|
|
},
|
|
{
|
|
"epoch": 1.017566354660854,
|
|
"grad_norm": 10.922108855871292,
|
|
"learning_rate": 2.849316955008996e-07,
|
|
"logits/chosen": -2.0840566158294678,
|
|
"logits/rejected": -2.0829105377197266,
|
|
"logps/chosen": -20.662445068359375,
|
|
"logps/rejected": -45.1578254699707,
|
|
"loss": 0.1894,
|
|
"rewards/accuracies": 0.828125,
|
|
"rewards/chosen": -0.044170308858156204,
|
|
"rewards/margins": 4.373213768005371,
|
|
"rewards/rejected": -4.417383670806885,
|
|
"step": 248
|
|
},
|
|
{
|
|
"epoch": 1.025772534940377,
|
|
"grad_norm": 7.713913555670714,
|
|
"learning_rate": 2.8136896962608785e-07,
|
|
"logits/chosen": -2.0515904426574707,
|
|
"logits/rejected": -2.051260232925415,
|
|
"logps/chosen": -25.802305221557617,
|
|
"logps/rejected": -45.2052116394043,
|
|
"loss": 0.1496,
|
|
"rewards/accuracies": 0.890625,
|
|
"rewards/chosen": -0.37995997071266174,
|
|
"rewards/margins": 4.365400314331055,
|
|
"rewards/rejected": -4.745360374450684,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 1.025772534940377,
|
|
"eval_logits/chosen": -2.042832612991333,
|
|
"eval_logits/rejected": -2.040806293487549,
|
|
"eval_logps/chosen": -25.576231002807617,
|
|
"eval_logps/rejected": -39.28297805786133,
|
|
"eval_loss": 0.2241121530532837,
|
|
"eval_rewards/accuracies": 0.8352534770965576,
|
|
"eval_rewards/chosen": -0.4679082930088043,
|
|
"eval_rewards/margins": 3.604546546936035,
|
|
"eval_rewards/rejected": -4.072454452514648,
|
|
"eval_runtime": 383.2982,
|
|
"eval_samples_per_second": 4.524,
|
|
"eval_steps_per_second": 1.132,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 1.0339787152199,
|
|
"grad_norm": 13.7751082342933,
|
|
"learning_rate": 2.777997590550758e-07,
|
|
"logits/chosen": -2.155553102493286,
|
|
"logits/rejected": -2.1575121879577637,
|
|
"logps/chosen": -27.36958122253418,
|
|
"logps/rejected": -42.855403900146484,
|
|
"loss": 0.1621,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -0.4879440367221832,
|
|
"rewards/margins": 3.895385980606079,
|
|
"rewards/rejected": -4.38332986831665,
|
|
"step": 252
|
|
},
|
|
{
|
|
"epoch": 1.042184895499423,
|
|
"grad_norm": 16.01605616077575,
|
|
"learning_rate": 2.742248016267692e-07,
|
|
"logits/chosen": -2.085308790206909,
|
|
"logits/rejected": -2.082420825958252,
|
|
"logps/chosen": -27.64177703857422,
|
|
"logps/rejected": -43.7150993347168,
|
|
"loss": 0.1802,
|
|
"rewards/accuracies": 0.890625,
|
|
"rewards/chosen": -0.7673605680465698,
|
|
"rewards/margins": 4.039832592010498,
|
|
"rewards/rejected": -4.807192325592041,
|
|
"step": 254
|
|
},
|
|
{
|
|
"epoch": 1.050391075778946,
|
|
"grad_norm": 10.928437797325595,
|
|
"learning_rate": 2.706448363680831e-07,
|
|
"logits/chosen": -2.0981714725494385,
|
|
"logits/rejected": -2.0977556705474854,
|
|
"logps/chosen": -25.607622146606445,
|
|
"logps/rejected": -46.82646179199219,
|
|
"loss": 0.1185,
|
|
"rewards/accuracies": 0.96875,
|
|
"rewards/chosen": -0.45319244265556335,
|
|
"rewards/margins": 4.636317253112793,
|
|
"rewards/rejected": -5.089509010314941,
|
|
"step": 256
|
|
},
|
|
{
|
|
"epoch": 1.058597256058469,
|
|
"grad_norm": 8.863927238249516,
|
|
"learning_rate": 2.6706060334116775e-07,
|
|
"logits/chosen": -2.0557029247283936,
|
|
"logits/rejected": -2.058215379714966,
|
|
"logps/chosen": -24.904468536376953,
|
|
"logps/rejected": -52.995845794677734,
|
|
"loss": 0.1153,
|
|
"rewards/accuracies": 0.953125,
|
|
"rewards/chosen": -0.11815226078033447,
|
|
"rewards/margins": 5.022463321685791,
|
|
"rewards/rejected": -5.140615940093994,
|
|
"step": 258
|
|
},
|
|
{
|
|
"epoch": 1.0668034363379921,
|
|
"grad_norm": 6.912079383205965,
|
|
"learning_rate": 2.634728434904204e-07,
|
|
"logits/chosen": -2.0943830013275146,
|
|
"logits/rejected": -2.0922083854675293,
|
|
"logps/chosen": -27.497676849365234,
|
|
"logps/rejected": -48.15060806274414,
|
|
"loss": 0.1458,
|
|
"rewards/accuracies": 0.890625,
|
|
"rewards/chosen": -0.2815995514392853,
|
|
"rewards/margins": 4.458585739135742,
|
|
"rewards/rejected": -4.740184783935547,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 1.0668034363379921,
|
|
"eval_logits/chosen": -2.0431203842163086,
|
|
"eval_logits/rejected": -2.0411202907562256,
|
|
"eval_logps/chosen": -25.558095932006836,
|
|
"eval_logps/rejected": -39.460453033447266,
|
|
"eval_loss": 0.2211485058069229,
|
|
"eval_rewards/accuracies": 0.8364055156707764,
|
|
"eval_rewards/chosen": -0.4588410556316376,
|
|
"eval_rewards/margins": 3.702350378036499,
|
|
"eval_rewards/rejected": -4.161191463470459,
|
|
"eval_runtime": 383.5467,
|
|
"eval_samples_per_second": 4.521,
|
|
"eval_steps_per_second": 1.132,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 1.0750096166175152,
|
|
"grad_norm": 15.832890184225922,
|
|
"learning_rate": 2.5988229848931483e-07,
|
|
"logits/chosen": -2.0925068855285645,
|
|
"logits/rejected": -2.0973916053771973,
|
|
"logps/chosen": -35.409889221191406,
|
|
"logps/rejected": -56.27671813964844,
|
|
"loss": 0.1349,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": -0.4806308150291443,
|
|
"rewards/margins": 4.840456008911133,
|
|
"rewards/rejected": -5.321086883544922,
|
|
"step": 262
|
|
},
|
|
{
|
|
"epoch": 1.0832157968970382,
|
|
"grad_norm": 10.693776832442103,
|
|
"learning_rate": 2.562897105870801e-07,
|
|
"logits/chosen": -2.0722289085388184,
|
|
"logits/rejected": -2.0738039016723633,
|
|
"logps/chosen": -22.13025665283203,
|
|
"logps/rejected": -36.53873825073242,
|
|
"loss": 0.1738,
|
|
"rewards/accuracies": 0.828125,
|
|
"rewards/chosen": -0.3039347231388092,
|
|
"rewards/margins": 3.8571736812591553,
|
|
"rewards/rejected": -4.161108016967773,
|
|
"step": 264
|
|
},
|
|
{
|
|
"epoch": 1.0914219771765612,
|
|
"grad_norm": 19.182488661584227,
|
|
"learning_rate": 2.5269582245526096e-07,
|
|
"logits/chosen": -2.0609676837921143,
|
|
"logits/rejected": -2.0527615547180176,
|
|
"logps/chosen": -25.158451080322266,
|
|
"logps/rejected": -37.255897521972656,
|
|
"loss": 0.1578,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -0.35766327381134033,
|
|
"rewards/margins": 3.5247013568878174,
|
|
"rewards/rejected": -3.882364273071289,
|
|
"step": 266
|
|
},
|
|
{
|
|
"epoch": 1.0996281574560842,
|
|
"grad_norm": 9.44923086833568,
|
|
"learning_rate": 2.4910137703418926e-07,
|
|
"logits/chosen": -2.0382041931152344,
|
|
"logits/rejected": -2.0330657958984375,
|
|
"logps/chosen": -27.744306564331055,
|
|
"logps/rejected": -37.20608901977539,
|
|
"loss": 0.1527,
|
|
"rewards/accuracies": 0.890625,
|
|
"rewards/chosen": 0.0014753900468349457,
|
|
"rewards/margins": 4.225948333740234,
|
|
"rewards/rejected": -4.224472522735596,
|
|
"step": 268
|
|
},
|
|
{
|
|
"epoch": 1.107834337735607,
|
|
"grad_norm": 10.39281903644009,
|
|
"learning_rate": 2.4550711737940205e-07,
|
|
"logits/chosen": -2.0400283336639404,
|
|
"logits/rejected": -2.0371220111846924,
|
|
"logps/chosen": -26.864028930664062,
|
|
"logps/rejected": -52.77448272705078,
|
|
"loss": 0.1428,
|
|
"rewards/accuracies": 0.921875,
|
|
"rewards/chosen": -0.7589187026023865,
|
|
"rewards/margins": 4.930204391479492,
|
|
"rewards/rejected": -5.689122200012207,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 1.107834337735607,
|
|
"eval_logits/chosen": -2.043691635131836,
|
|
"eval_logits/rejected": -2.0416359901428223,
|
|
"eval_logps/chosen": -25.027318954467773,
|
|
"eval_logps/rejected": -38.982704162597656,
|
|
"eval_loss": 0.22408966720104218,
|
|
"eval_rewards/accuracies": 0.8364055156707764,
|
|
"eval_rewards/chosen": -0.19345209002494812,
|
|
"eval_rewards/margins": 3.7288661003112793,
|
|
"eval_rewards/rejected": -3.9223177433013916,
|
|
"eval_runtime": 383.6577,
|
|
"eval_samples_per_second": 4.52,
|
|
"eval_steps_per_second": 1.131,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 1.11604051801513,
|
|
"grad_norm": 6.39182395545924,
|
|
"learning_rate": 2.419137865080337e-07,
|
|
"logits/chosen": -2.069648027420044,
|
|
"logits/rejected": -2.0665249824523926,
|
|
"logps/chosen": -22.426170349121094,
|
|
"logps/rejected": -28.420143127441406,
|
|
"loss": 0.1774,
|
|
"rewards/accuracies": 0.765625,
|
|
"rewards/chosen": -0.1105562299489975,
|
|
"rewards/margins": 3.127821683883667,
|
|
"rewards/rejected": -3.238377571105957,
|
|
"step": 272
|
|
},
|
|
{
|
|
"epoch": 1.124246698294653,
|
|
"grad_norm": 11.027723946191974,
|
|
"learning_rate": 2.383221272452178e-07,
|
|
"logits/chosen": -2.0779690742492676,
|
|
"logits/rejected": -2.087660312652588,
|
|
"logps/chosen": -24.797609329223633,
|
|
"logps/rejected": -65.99850463867188,
|
|
"loss": 0.1436,
|
|
"rewards/accuracies": 0.921875,
|
|
"rewards/chosen": -0.11033198237419128,
|
|
"rewards/margins": 5.425482749938965,
|
|
"rewards/rejected": -5.53581428527832,
|
|
"step": 274
|
|
},
|
|
{
|
|
"epoch": 1.132452878574176,
|
|
"grad_norm": 17.805145954377792,
|
|
"learning_rate": 2.3473288207052741e-07,
|
|
"logits/chosen": -2.08833384513855,
|
|
"logits/rejected": -2.0850491523742676,
|
|
"logps/chosen": -25.857656478881836,
|
|
"logps/rejected": -32.381996154785156,
|
|
"loss": 0.1588,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": -0.1286620795726776,
|
|
"rewards/margins": 3.531426191329956,
|
|
"rewards/rejected": -3.660088062286377,
|
|
"step": 276
|
|
},
|
|
{
|
|
"epoch": 1.1406590588536991,
|
|
"grad_norm": 17.682382824702852,
|
|
"learning_rate": 2.3114679296448726e-07,
|
|
"logits/chosen": -2.131256341934204,
|
|
"logits/rejected": -2.125654697418213,
|
|
"logps/chosen": -24.452953338623047,
|
|
"logps/rejected": -31.313426971435547,
|
|
"loss": 0.2115,
|
|
"rewards/accuracies": 0.828125,
|
|
"rewards/chosen": -0.033705681562423706,
|
|
"rewards/margins": 3.4805080890655518,
|
|
"rewards/rejected": -3.514213800430298,
|
|
"step": 278
|
|
},
|
|
{
|
|
"epoch": 1.1488652391332221,
|
|
"grad_norm": 16.145295646746973,
|
|
"learning_rate": 2.2756460125518942e-07,
|
|
"logits/chosen": -2.0623483657836914,
|
|
"logits/rejected": -2.056093692779541,
|
|
"logps/chosen": -21.44537353515625,
|
|
"logps/rejected": -35.502716064453125,
|
|
"loss": 0.1726,
|
|
"rewards/accuracies": 0.859375,
|
|
"rewards/chosen": -0.28275609016418457,
|
|
"rewards/margins": 3.6864452362060547,
|
|
"rewards/rejected": -3.96920108795166,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 1.1488652391332221,
|
|
"eval_logits/chosen": -2.045440196990967,
|
|
"eval_logits/rejected": -2.0434725284576416,
|
|
"eval_logps/chosen": -25.390390396118164,
|
|
"eval_logps/rejected": -39.54319763183594,
|
|
"eval_loss": 0.2192622274160385,
|
|
"eval_rewards/accuracies": 0.8352534770965576,
|
|
"eval_rewards/chosen": -0.37498798966407776,
|
|
"eval_rewards/margins": 3.8275747299194336,
|
|
"eval_rewards/rejected": -4.2025628089904785,
|
|
"eval_runtime": 383.5119,
|
|
"eval_samples_per_second": 4.521,
|
|
"eval_steps_per_second": 1.132,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 1.1570714194127452,
|
|
"grad_norm": 18.875320195670874,
|
|
"learning_rate": 2.2398704746504318e-07,
|
|
"logits/chosen": -1.9832574129104614,
|
|
"logits/rejected": -1.9829756021499634,
|
|
"logps/chosen": -31.6018123626709,
|
|
"logps/rejected": -41.83821105957031,
|
|
"loss": 0.1804,
|
|
"rewards/accuracies": 0.859375,
|
|
"rewards/chosen": -0.4063544273376465,
|
|
"rewards/margins": 4.4296417236328125,
|
|
"rewards/rejected": -4.835995674133301,
|
|
"step": 282
|
|
},
|
|
{
|
|
"epoch": 1.1652775996922682,
|
|
"grad_norm": 14.790336639999394,
|
|
"learning_rate": 2.20414871157692e-07,
|
|
"logits/chosen": -2.0224382877349854,
|
|
"logits/rejected": -2.027156352996826,
|
|
"logps/chosen": -25.788434982299805,
|
|
"logps/rejected": -50.23531723022461,
|
|
"loss": 0.1406,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -0.5073741674423218,
|
|
"rewards/margins": 4.580756664276123,
|
|
"rewards/rejected": -5.088130474090576,
|
|
"step": 284
|
|
},
|
|
{
|
|
"epoch": 1.1734837799717912,
|
|
"grad_norm": 14.288848495922638,
|
|
"learning_rate": 2.1684881078512867e-07,
|
|
"logits/chosen": -2.0773634910583496,
|
|
"logits/rejected": -2.0686299800872803,
|
|
"logps/chosen": -25.211549758911133,
|
|
"logps/rejected": -30.360422134399414,
|
|
"loss": 0.1824,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -0.5571584105491638,
|
|
"rewards/margins": 3.3241891860961914,
|
|
"rewards/rejected": -3.881347417831421,
|
|
"step": 286
|
|
},
|
|
{
|
|
"epoch": 1.1816899602513142,
|
|
"grad_norm": 17.171906139091153,
|
|
"learning_rate": 2.1328960353503978e-07,
|
|
"logits/chosen": -2.085387945175171,
|
|
"logits/rejected": -2.0828468799591064,
|
|
"logps/chosen": -23.766998291015625,
|
|
"logps/rejected": -33.4593620300293,
|
|
"loss": 0.1488,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -0.5340551137924194,
|
|
"rewards/margins": 3.7335057258605957,
|
|
"rewards/rejected": -4.267560958862305,
|
|
"step": 288
|
|
},
|
|
{
|
|
"epoch": 1.1898961405308373,
|
|
"grad_norm": 13.043172605616002,
|
|
"learning_rate": 2.0973798517841173e-07,
|
|
"logits/chosen": -2.018440008163452,
|
|
"logits/rejected": -2.0149004459381104,
|
|
"logps/chosen": -31.094905853271484,
|
|
"logps/rejected": -34.58868408203125,
|
|
"loss": 0.1415,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -0.8485018610954285,
|
|
"rewards/margins": 3.8137242794036865,
|
|
"rewards/rejected": -4.66222620010376,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 1.1898961405308373,
|
|
"eval_logits/chosen": -2.0464136600494385,
|
|
"eval_logits/rejected": -2.044499158859253,
|
|
"eval_logps/chosen": -25.900218963623047,
|
|
"eval_logps/rejected": -40.12266159057617,
|
|
"eval_loss": 0.21777255833148956,
|
|
"eval_rewards/accuracies": 0.8364055156707764,
|
|
"eval_rewards/chosen": -0.6299027800559998,
|
|
"eval_rewards/margins": 3.8623909950256348,
|
|
"eval_rewards/rejected": -4.492293834686279,
|
|
"eval_runtime": 383.3374,
|
|
"eval_samples_per_second": 4.523,
|
|
"eval_steps_per_second": 1.132,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 1.1981023208103603,
|
|
"grad_norm": 20.638807271142973,
|
|
"learning_rate": 2.0619468991743042e-07,
|
|
"logits/chosen": -2.061304807662964,
|
|
"logits/rejected": -2.0597338676452637,
|
|
"logps/chosen": -24.799753189086914,
|
|
"logps/rejected": -57.012210845947266,
|
|
"loss": 0.1738,
|
|
"rewards/accuracies": 0.921875,
|
|
"rewards/chosen": -1.1362073421478271,
|
|
"rewards/margins": 5.481666564941406,
|
|
"rewards/rejected": -6.617873191833496,
|
|
"step": 292
|
|
},
|
|
{
|
|
"epoch": 1.2063085010898833,
|
|
"grad_norm": 15.936474764605512,
|
|
"learning_rate": 2.026604502337039e-07,
|
|
"logits/chosen": -2.047367572784424,
|
|
"logits/rejected": -2.044090747833252,
|
|
"logps/chosen": -27.630414962768555,
|
|
"logps/rejected": -50.361202239990234,
|
|
"loss": 0.1453,
|
|
"rewards/accuracies": 0.890625,
|
|
"rewards/chosen": 0.04436264932155609,
|
|
"rewards/margins": 4.908329486846924,
|
|
"rewards/rejected": -4.863966941833496,
|
|
"step": 294
|
|
},
|
|
{
|
|
"epoch": 1.2145146813694063,
|
|
"grad_norm": 15.408621452000459,
|
|
"learning_rate": 1.9913599673684159e-07,
|
|
"logits/chosen": -2.0377581119537354,
|
|
"logits/rejected": -2.0361804962158203,
|
|
"logps/chosen": -22.501596450805664,
|
|
"logps/rejected": -40.86521530151367,
|
|
"loss": 0.1384,
|
|
"rewards/accuracies": 0.890625,
|
|
"rewards/chosen": -0.5628668069839478,
|
|
"rewards/margins": 3.820146083831787,
|
|
"rewards/rejected": -4.383012771606445,
|
|
"step": 296
|
|
},
|
|
{
|
|
"epoch": 1.2227208616489293,
|
|
"grad_norm": 12.285332508425714,
|
|
"learning_rate": 1.9562205801342034e-07,
|
|
"logits/chosen": -2.061429738998413,
|
|
"logits/rejected": -2.0591816902160645,
|
|
"logps/chosen": -29.394742965698242,
|
|
"logps/rejected": -43.32746505737305,
|
|
"loss": 0.1526,
|
|
"rewards/accuracies": 0.921875,
|
|
"rewards/chosen": -0.384346067905426,
|
|
"rewards/margins": 4.4763689041137695,
|
|
"rewards/rejected": -4.860714912414551,
|
|
"step": 298
|
|
},
|
|
{
|
|
"epoch": 1.2309270419284524,
|
|
"grad_norm": 5.583982281384929,
|
|
"learning_rate": 1.9211936047636867e-07,
|
|
"logits/chosen": -2.0710768699645996,
|
|
"logits/rejected": -2.0669403076171875,
|
|
"logps/chosen": -20.023929595947266,
|
|
"logps/rejected": -34.831573486328125,
|
|
"loss": 0.1595,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": -0.22750067710876465,
|
|
"rewards/margins": 4.123189926147461,
|
|
"rewards/rejected": -4.350690841674805,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 1.2309270419284524,
|
|
"eval_logits/chosen": -2.0440096855163574,
|
|
"eval_logits/rejected": -2.042032480239868,
|
|
"eval_logps/chosen": -24.7576961517334,
|
|
"eval_logps/rejected": -38.71461486816406,
|
|
"eval_loss": 0.2164347618818283,
|
|
"eval_rewards/accuracies": 0.8387096524238586,
|
|
"eval_rewards/chosen": -0.058640193194150925,
|
|
"eval_rewards/margins": 3.729631185531616,
|
|
"eval_rewards/rejected": -3.788270950317383,
|
|
"eval_runtime": 383.42,
|
|
"eval_samples_per_second": 4.522,
|
|
"eval_steps_per_second": 1.132,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 1.2391332222079754,
|
|
"grad_norm": 11.637212026135893,
|
|
"learning_rate": 1.886286282148002e-07,
|
|
"logits/chosen": -2.0720925331115723,
|
|
"logits/rejected": -2.06878399848938,
|
|
"logps/chosen": -21.090686798095703,
|
|
"logps/rejected": -28.20726776123047,
|
|
"loss": 0.156,
|
|
"rewards/accuracies": 0.859375,
|
|
"rewards/chosen": 0.15111806988716125,
|
|
"rewards/margins": 3.5836377143859863,
|
|
"rewards/rejected": -3.4325199127197266,
|
|
"step": 302
|
|
},
|
|
{
|
|
"epoch": 1.2473394024874984,
|
|
"grad_norm": 6.71334576605166,
|
|
"learning_rate": 1.8515058284432743e-07,
|
|
"logits/chosen": -2.1127841472625732,
|
|
"logits/rejected": -2.111618757247925,
|
|
"logps/chosen": -23.5858097076416,
|
|
"logps/rejected": -42.50680923461914,
|
|
"loss": 0.1521,
|
|
"rewards/accuracies": 0.890625,
|
|
"rewards/chosen": -0.2578216791152954,
|
|
"rewards/margins": 4.158515453338623,
|
|
"rewards/rejected": -4.416337013244629,
|
|
"step": 304
|
|
},
|
|
{
|
|
"epoch": 1.2555455827670214,
|
|
"grad_norm": 16.588064814380605,
|
|
"learning_rate": 1.8168594335788728e-07,
|
|
"logits/chosen": -2.042862892150879,
|
|
"logits/rejected": -2.0439772605895996,
|
|
"logps/chosen": -24.17473030090332,
|
|
"logps/rejected": -53.81690216064453,
|
|
"loss": 0.177,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -0.44858402013778687,
|
|
"rewards/margins": 4.805813789367676,
|
|
"rewards/rejected": -5.254397392272949,
|
|
"step": 306
|
|
},
|
|
{
|
|
"epoch": 1.2637517630465445,
|
|
"grad_norm": 6.647490191841678,
|
|
"learning_rate": 1.7823542597710832e-07,
|
|
"logits/chosen": -2.0013082027435303,
|
|
"logits/rejected": -2.004103183746338,
|
|
"logps/chosen": -23.131669998168945,
|
|
"logps/rejected": -43.91994857788086,
|
|
"loss": 0.1689,
|
|
"rewards/accuracies": 0.859375,
|
|
"rewards/chosen": 0.4323474168777466,
|
|
"rewards/margins": 4.400308609008789,
|
|
"rewards/rejected": -3.967961311340332,
|
|
"step": 308
|
|
},
|
|
{
|
|
"epoch": 1.2719579433260675,
|
|
"grad_norm": 13.202795454945317,
|
|
"learning_rate": 1.7479974400425123e-07,
|
|
"logits/chosen": -2.046172857284546,
|
|
"logits/rejected": -2.0446221828460693,
|
|
"logps/chosen": -23.60657501220703,
|
|
"logps/rejected": -39.958396911621094,
|
|
"loss": 0.1706,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 0.3769833445549011,
|
|
"rewards/margins": 4.0399041175842285,
|
|
"rewards/rejected": -3.66292142868042,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 1.2719579433260675,
|
|
"eval_logits/chosen": -2.043919086456299,
|
|
"eval_logits/rejected": -2.04194974899292,
|
|
"eval_logps/chosen": -24.46892547607422,
|
|
"eval_logps/rejected": -38.424713134765625,
|
|
"eval_loss": 0.21722769737243652,
|
|
"eval_rewards/accuracies": 0.8375576138496399,
|
|
"eval_rewards/chosen": 0.08574579656124115,
|
|
"eval_rewards/margins": 3.7290680408477783,
|
|
"eval_rewards/rejected": -3.643322467803955,
|
|
"eval_runtime": 383.1854,
|
|
"eval_samples_per_second": 4.525,
|
|
"eval_steps_per_second": 1.133,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 1.2801641236055905,
|
|
"grad_norm": 10.439998982466557,
|
|
"learning_rate": 1.7137960767475263e-07,
|
|
"logits/chosen": -2.062668561935425,
|
|
"logits/rejected": -2.058307647705078,
|
|
"logps/chosen": -20.401418685913086,
|
|
"logps/rejected": -39.55351257324219,
|
|
"loss": 0.179,
|
|
"rewards/accuracies": 0.84375,
|
|
"rewards/chosen": 0.12827420234680176,
|
|
"rewards/margins": 4.090062141418457,
|
|
"rewards/rejected": -3.9617881774902344,
|
|
"step": 312
|
|
},
|
|
{
|
|
"epoch": 1.2883703038851135,
|
|
"grad_norm": 7.172405069379902,
|
|
"learning_rate": 1.6797572401040244e-07,
|
|
"logits/chosen": -2.0692923069000244,
|
|
"logits/rejected": -2.067417860031128,
|
|
"logps/chosen": -30.879615783691406,
|
|
"logps/rejected": -40.37434387207031,
|
|
"loss": 0.1199,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": 0.12180179357528687,
|
|
"rewards/margins": 4.198694229125977,
|
|
"rewards/rejected": -4.076892375946045,
|
|
"step": 314
|
|
},
|
|
{
|
|
"epoch": 1.2965764841646366,
|
|
"grad_norm": 24.11773202321173,
|
|
"learning_rate": 1.6458879667318687e-07,
|
|
"logits/chosen": -2.064960479736328,
|
|
"logits/rejected": -2.0638976097106934,
|
|
"logps/chosen": -29.61989974975586,
|
|
"logps/rejected": -38.073875427246094,
|
|
"loss": 0.1542,
|
|
"rewards/accuracies": 0.90625,
|
|
"rewards/chosen": -0.3140927255153656,
|
|
"rewards/margins": 3.569211721420288,
|
|
"rewards/rejected": -3.8833043575286865,
|
|
"step": 316
|
|
},
|
|
{
|
|
"epoch": 1.3047826644441596,
|
|
"grad_norm": 15.353619034619824,
|
|
"learning_rate": 1.612195258198243e-07,
|
|
"logits/chosen": -2.1528408527374268,
|
|
"logits/rejected": -2.1589419841766357,
|
|
"logps/chosen": -25.104019165039062,
|
|
"logps/rejected": -56.826202392578125,
|
|
"loss": 0.1502,
|
|
"rewards/accuracies": 0.859375,
|
|
"rewards/chosen": 0.11969700455665588,
|
|
"rewards/margins": 5.174037933349609,
|
|
"rewards/rejected": -5.054340362548828,
|
|
"step": 318
|
|
},
|
|
{
|
|
"epoch": 1.3129888447236826,
|
|
"grad_norm": 6.821222616952283,
|
|
"learning_rate": 1.57868607957027e-07,
|
|
"logits/chosen": -2.0985031127929688,
|
|
"logits/rejected": -2.0935959815979004,
|
|
"logps/chosen": -30.2029972076416,
|
|
"logps/rejected": -43.66718292236328,
|
|
"loss": 0.1404,
|
|
"rewards/accuracies": 0.921875,
|
|
"rewards/chosen": -0.6867274641990662,
|
|
"rewards/margins": 4.196517467498779,
|
|
"rewards/rejected": -4.883245468139648,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 1.3129888447236826,
|
|
"eval_logits/chosen": -2.0446114540100098,
|
|
"eval_logits/rejected": -2.042722702026367,
|
|
"eval_logps/chosen": -25.383127212524414,
|
|
"eval_logps/rejected": -39.56816101074219,
|
|
"eval_loss": 0.21271604299545288,
|
|
"eval_rewards/accuracies": 0.8375576138496399,
|
|
"eval_rewards/chosen": -0.37135595083236694,
|
|
"eval_rewards/margins": 3.84369158744812,
|
|
"eval_rewards/rejected": -4.215047359466553,
|
|
"eval_runtime": 383.232,
|
|
"eval_samples_per_second": 4.525,
|
|
"eval_steps_per_second": 1.132,
|
|
"step": 320
|
|
}
|
|
],
|
|
"logging_steps": 2,
|
|
"max_steps": 486,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 2,
|
|
"save_steps": 10,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|