2924 lines
102 KiB
JSON
2924 lines
102 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 0.8599068434252956,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 900,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"epoch": 0.004777260241251642,
|
||
|
|
"grad_norm": 0.396484375,
|
||
|
|
"learning_rate": 2.222222222222222e-08,
|
||
|
|
"logits/chosen": -1.7133970260620117,
|
||
|
|
"logits/rejected": -1.6876779794692993,
|
||
|
|
"logps/chosen": -0.2796992361545563,
|
||
|
|
"logps/rejected": -0.28187569975852966,
|
||
|
|
"loss": 0.6895627379417419,
|
||
|
|
"loss/core": 0.6895627379417419,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.4170753955841064,
|
||
|
|
"rewards/margins": 1.4557363986968994,
|
||
|
|
"rewards/rejected": 0.9613393545150757,
|
||
|
|
"step": 5
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.009554520482503284,
|
||
|
|
"grad_norm": 2.28125,
|
||
|
|
"learning_rate": 5e-08,
|
||
|
|
"logits/chosen": -1.9638487100601196,
|
||
|
|
"logits/rejected": -2.050144910812378,
|
||
|
|
"logps/chosen": -0.28905805945396423,
|
||
|
|
"logps/rejected": -0.2884846329689026,
|
||
|
|
"loss": 0.6929782032966614,
|
||
|
|
"loss/core": 0.6929782032966614,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.6930923461914062,
|
||
|
|
"rewards/margins": 0.08007440716028214,
|
||
|
|
"rewards/rejected": 1.6130180358886719,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.014331780723754926,
|
||
|
|
"grad_norm": 0.44140625,
|
||
|
|
"learning_rate": 7.777777777777778e-08,
|
||
|
|
"logits/chosen": -1.8020557165145874,
|
||
|
|
"logits/rejected": -1.7314844131469727,
|
||
|
|
"logps/chosen": -0.27712005376815796,
|
||
|
|
"logps/rejected": -0.2847710847854614,
|
||
|
|
"loss": 0.6906865239143372,
|
||
|
|
"loss/core": 0.6906865239143372,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.1371042728424072,
|
||
|
|
"rewards/margins": 1.0041325092315674,
|
||
|
|
"rewards/rejected": 1.1329715251922607,
|
||
|
|
"step": 15
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.01910904096500657,
|
||
|
|
"grad_norm": 0.3046875,
|
||
|
|
"learning_rate": 1.0555555555555555e-07,
|
||
|
|
"logits/chosen": -1.9990392923355103,
|
||
|
|
"logits/rejected": -1.998739242553711,
|
||
|
|
"logps/chosen": -0.3013564348220825,
|
||
|
|
"logps/rejected": -0.28647691011428833,
|
||
|
|
"loss": 0.6906139254570007,
|
||
|
|
"loss/core": 0.6906139254570007,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.9157869815826416,
|
||
|
|
"rewards/margins": 1.0352976322174072,
|
||
|
|
"rewards/rejected": 0.8804894685745239,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.02388630120625821,
|
||
|
|
"grad_norm": 0.2392578125,
|
||
|
|
"learning_rate": 1.3333333333333334e-07,
|
||
|
|
"logits/chosen": -1.9096256494522095,
|
||
|
|
"logits/rejected": -1.9282000064849854,
|
||
|
|
"logps/chosen": -0.2734399437904358,
|
||
|
|
"logps/rejected": -0.2599286735057831,
|
||
|
|
"loss": 0.6880686283111572,
|
||
|
|
"loss/core": 0.6880686283111572,
|
||
|
|
"rewards/accuracies": 0.824999988079071,
|
||
|
|
"rewards/chosen": 3.8207664489746094,
|
||
|
|
"rewards/margins": 2.1955325603485107,
|
||
|
|
"rewards/rejected": 1.6252334117889404,
|
||
|
|
"step": 25
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.028663561447509853,
|
||
|
|
"grad_norm": 0.59765625,
|
||
|
|
"learning_rate": 1.611111111111111e-07,
|
||
|
|
"logits/chosen": -2.0870721340179443,
|
||
|
|
"logits/rejected": -2.0627994537353516,
|
||
|
|
"logps/chosen": -0.30142563581466675,
|
||
|
|
"logps/rejected": -0.32440415024757385,
|
||
|
|
"loss": 0.6901697516441345,
|
||
|
|
"loss/core": 0.6901697516441345,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.26503849029541,
|
||
|
|
"rewards/margins": 1.239709496498108,
|
||
|
|
"rewards/rejected": 1.0253286361694336,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.033440821688761495,
|
||
|
|
"grad_norm": 0.12890625,
|
||
|
|
"learning_rate": 1.8888888888888888e-07,
|
||
|
|
"logits/chosen": -1.8673341274261475,
|
||
|
|
"logits/rejected": -1.8807346820831299,
|
||
|
|
"logps/chosen": -0.2671809196472168,
|
||
|
|
"logps/rejected": -0.26550590991973877,
|
||
|
|
"loss": 0.6895102262496948,
|
||
|
|
"loss/core": 0.6895102262496948,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.6639485359191895,
|
||
|
|
"rewards/margins": 1.506712555885315,
|
||
|
|
"rewards/rejected": 1.157235860824585,
|
||
|
|
"step": 35
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.03821808193001314,
|
||
|
|
"grad_norm": 0.421875,
|
||
|
|
"learning_rate": 2.1666666666666667e-07,
|
||
|
|
"logits/chosen": -1.6795200109481812,
|
||
|
|
"logits/rejected": -1.7307487726211548,
|
||
|
|
"logps/chosen": -0.2936388850212097,
|
||
|
|
"logps/rejected": -0.2830830514431,
|
||
|
|
"loss": 0.6864294409751892,
|
||
|
|
"loss/core": 0.6864294409751892,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 4.467957019805908,
|
||
|
|
"rewards/margins": 2.8773446083068848,
|
||
|
|
"rewards/rejected": 1.5906131267547607,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.04299534217126478,
|
||
|
|
"grad_norm": 0.1943359375,
|
||
|
|
"learning_rate": 2.4444444444444445e-07,
|
||
|
|
"logits/chosen": -1.867241621017456,
|
||
|
|
"logits/rejected": -1.9943357706069946,
|
||
|
|
"logps/chosen": -0.28906434774398804,
|
||
|
|
"logps/rejected": -0.28838273882865906,
|
||
|
|
"loss": 0.690056562423706,
|
||
|
|
"loss/core": 0.690056562423706,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 2.092913866043091,
|
||
|
|
"rewards/margins": 1.250601053237915,
|
||
|
|
"rewards/rejected": 0.8423126935958862,
|
||
|
|
"step": 45
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.04777260241251642,
|
||
|
|
"grad_norm": 0.94140625,
|
||
|
|
"learning_rate": 2.7222222222222216e-07,
|
||
|
|
"logits/chosen": -1.9381431341171265,
|
||
|
|
"logits/rejected": -1.944374442100525,
|
||
|
|
"logps/chosen": -0.2815219759941101,
|
||
|
|
"logps/rejected": -0.28127673268318176,
|
||
|
|
"loss": 0.6898090839385986,
|
||
|
|
"loss/core": 0.6898090839385986,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.8812808990478516,
|
||
|
|
"rewards/margins": 1.3943486213684082,
|
||
|
|
"rewards/rejected": 1.4869322776794434,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.05254986265376806,
|
||
|
|
"grad_norm": 0.07958984375,
|
||
|
|
"learning_rate": 3e-07,
|
||
|
|
"logits/chosen": -1.9054334163665771,
|
||
|
|
"logits/rejected": -1.8701159954071045,
|
||
|
|
"logps/chosen": -0.28710708022117615,
|
||
|
|
"logps/rejected": -0.28994759917259216,
|
||
|
|
"loss": 0.6927647590637207,
|
||
|
|
"loss/core": 0.6927647590637207,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.277327060699463,
|
||
|
|
"rewards/margins": 0.2503672242164612,
|
||
|
|
"rewards/rejected": 2.0269598960876465,
|
||
|
|
"step": 55
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.057327122895019705,
|
||
|
|
"grad_norm": 3.265625,
|
||
|
|
"learning_rate": 3.2777777777777776e-07,
|
||
|
|
"logits/chosen": -1.8265936374664307,
|
||
|
|
"logits/rejected": -1.817793846130371,
|
||
|
|
"logps/chosen": -0.27443718910217285,
|
||
|
|
"logps/rejected": -0.2840142548084259,
|
||
|
|
"loss": 0.6916871666908264,
|
||
|
|
"loss/core": 0.6916871666908264,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.1564693450927734,
|
||
|
|
"rewards/margins": 0.6028232574462891,
|
||
|
|
"rewards/rejected": 1.553646206855774,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.06210438313627135,
|
||
|
|
"grad_norm": 0.1669921875,
|
||
|
|
"learning_rate": 3.5555555555555553e-07,
|
||
|
|
"logits/chosen": -1.6590009927749634,
|
||
|
|
"logits/rejected": -1.720873236656189,
|
||
|
|
"logps/chosen": -0.3111419975757599,
|
||
|
|
"logps/rejected": -0.304778516292572,
|
||
|
|
"loss": 0.688835859298706,
|
||
|
|
"loss/core": 0.688835859298706,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.9504354000091553,
|
||
|
|
"rewards/margins": 1.7633116245269775,
|
||
|
|
"rewards/rejected": 1.1871236562728882,
|
||
|
|
"step": 65
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.06688164337752299,
|
||
|
|
"grad_norm": 0.462890625,
|
||
|
|
"learning_rate": 3.8333333333333335e-07,
|
||
|
|
"logits/chosen": -1.652000069618225,
|
||
|
|
"logits/rejected": -1.681039571762085,
|
||
|
|
"logps/chosen": -0.37990865111351013,
|
||
|
|
"logps/rejected": -0.29802781343460083,
|
||
|
|
"loss": 0.6898948550224304,
|
||
|
|
"loss/core": 0.6898948550224304,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 3.6380057334899902,
|
||
|
|
"rewards/margins": 1.4650633335113525,
|
||
|
|
"rewards/rejected": 2.1729423999786377,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07165890361877464,
|
||
|
|
"grad_norm": 0.12890625,
|
||
|
|
"learning_rate": 4.1111111111111107e-07,
|
||
|
|
"logits/chosen": -1.710818886756897,
|
||
|
|
"logits/rejected": -1.736729383468628,
|
||
|
|
"logps/chosen": -0.31785252690315247,
|
||
|
|
"logps/rejected": -0.2882820963859558,
|
||
|
|
"loss": 0.6908040642738342,
|
||
|
|
"loss/core": 0.6908040642738342,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 2.5783510208129883,
|
||
|
|
"rewards/margins": 0.9765921831130981,
|
||
|
|
"rewards/rejected": 1.6017587184906006,
|
||
|
|
"step": 75
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07643616386002627,
|
||
|
|
"grad_norm": 0.1689453125,
|
||
|
|
"learning_rate": 4.3888888888888884e-07,
|
||
|
|
"logits/chosen": -1.7501447200775146,
|
||
|
|
"logits/rejected": -1.8643512725830078,
|
||
|
|
"logps/chosen": -0.2600446045398712,
|
||
|
|
"logps/rejected": -0.27591589093208313,
|
||
|
|
"loss": 0.6883189082145691,
|
||
|
|
"loss/core": 0.6883189082145691,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.2465500831604004,
|
||
|
|
"rewards/margins": 2.013552188873291,
|
||
|
|
"rewards/rejected": 1.2329976558685303,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.08121342410127792,
|
||
|
|
"grad_norm": 0.2001953125,
|
||
|
|
"learning_rate": 4.6666666666666666e-07,
|
||
|
|
"logits/chosen": -1.8498824834823608,
|
||
|
|
"logits/rejected": -1.8502506017684937,
|
||
|
|
"logps/chosen": -0.30674856901168823,
|
||
|
|
"logps/rejected": -0.27847328782081604,
|
||
|
|
"loss": 0.6879407167434692,
|
||
|
|
"loss/core": 0.6879407167434692,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.505821943283081,
|
||
|
|
"rewards/margins": 2.1826915740966797,
|
||
|
|
"rewards/rejected": 1.323130488395691,
|
||
|
|
"step": 85
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.08599068434252956,
|
||
|
|
"grad_norm": 0.0810546875,
|
||
|
|
"learning_rate": 4.944444444444445e-07,
|
||
|
|
"logits/chosen": -2.03493595123291,
|
||
|
|
"logits/rejected": -2.0456414222717285,
|
||
|
|
"logps/chosen": -0.28171059489250183,
|
||
|
|
"logps/rejected": -0.29523080587387085,
|
||
|
|
"loss": 0.6909321546554565,
|
||
|
|
"loss/core": 0.6909321546554565,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.7502819299697876,
|
||
|
|
"rewards/margins": 0.8973544836044312,
|
||
|
|
"rewards/rejected": 0.8529273867607117,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.09076794458378121,
|
||
|
|
"grad_norm": 0.37890625,
|
||
|
|
"learning_rate": 4.999699149323369e-07,
|
||
|
|
"logits/chosen": -1.6194324493408203,
|
||
|
|
"logits/rejected": -1.6468042135238647,
|
||
|
|
"logps/chosen": -0.2668497860431671,
|
||
|
|
"logps/rejected": -0.27971798181533813,
|
||
|
|
"loss": 0.6904693841934204,
|
||
|
|
"loss/core": 0.6904693841934204,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 3.2152295112609863,
|
||
|
|
"rewards/margins": 1.1187059879302979,
|
||
|
|
"rewards/rejected": 2.0965230464935303,
|
||
|
|
"step": 95
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.09554520482503284,
|
||
|
|
"grad_norm": 0.19921875,
|
||
|
|
"learning_rate": 4.998477067547739e-07,
|
||
|
|
"logits/chosen": -1.8947890996932983,
|
||
|
|
"logits/rejected": -1.9110848903656006,
|
||
|
|
"logps/chosen": -0.2584729790687561,
|
||
|
|
"logps/rejected": -0.24682433903217316,
|
||
|
|
"loss": 0.6902803182601929,
|
||
|
|
"loss/core": 0.6902803182601929,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.4498555660247803,
|
||
|
|
"rewards/margins": 1.1651194095611572,
|
||
|
|
"rewards/rejected": 1.2847360372543335,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10032246506628449,
|
||
|
|
"grad_norm": 0.67578125,
|
||
|
|
"learning_rate": 4.996315410727229e-07,
|
||
|
|
"logits/chosen": -1.93181574344635,
|
||
|
|
"logits/rejected": -1.9506447315216064,
|
||
|
|
"logps/chosen": -0.283364474773407,
|
||
|
|
"logps/rejected": -0.27992352843284607,
|
||
|
|
"loss": 0.6902110576629639,
|
||
|
|
"loss/core": 0.6902110576629639,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.005598306655884,
|
||
|
|
"rewards/margins": 1.2088587284088135,
|
||
|
|
"rewards/rejected": 0.7967394590377808,
|
||
|
|
"step": 105
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10509972530753613,
|
||
|
|
"grad_norm": 0.1845703125,
|
||
|
|
"learning_rate": 4.993214991772562e-07,
|
||
|
|
"logits/chosen": -1.7346656322479248,
|
||
|
|
"logits/rejected": -1.6285288333892822,
|
||
|
|
"logps/chosen": -0.33683377504348755,
|
||
|
|
"logps/rejected": -0.29718539118766785,
|
||
|
|
"loss": 0.6869697570800781,
|
||
|
|
"loss/core": 0.6869697570800781,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 4.486939907073975,
|
||
|
|
"rewards/margins": 2.5928866863250732,
|
||
|
|
"rewards/rejected": 1.8940532207489014,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10987698554878778,
|
||
|
|
"grad_norm": 0.859375,
|
||
|
|
"learning_rate": 4.989176976624511e-07,
|
||
|
|
"logits/chosen": -1.63823664188385,
|
||
|
|
"logits/rejected": -1.5655968189239502,
|
||
|
|
"logps/chosen": -0.30025607347488403,
|
||
|
|
"logps/rejected": -0.3001025915145874,
|
||
|
|
"loss": 0.6923608183860779,
|
||
|
|
"loss/core": 0.6923608183860779,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.7768100500106812,
|
||
|
|
"rewards/margins": 0.34278303384780884,
|
||
|
|
"rewards/rejected": 1.434027075767517,
|
||
|
|
"step": 115
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.11465424579003941,
|
||
|
|
"grad_norm": 1.65625,
|
||
|
|
"learning_rate": 4.984202883815428e-07,
|
||
|
|
"logits/chosen": -1.682744026184082,
|
||
|
|
"logits/rejected": -1.6865253448486328,
|
||
|
|
"logps/chosen": -0.30211514234542847,
|
||
|
|
"logps/rejected": -0.29606205224990845,
|
||
|
|
"loss": 0.6914452910423279,
|
||
|
|
"loss/core": 0.6914452910423279,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.8103902339935303,
|
||
|
|
"rewards/margins": 0.6875762343406677,
|
||
|
|
"rewards/rejected": 1.1228139400482178,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.11943150603129106,
|
||
|
|
"grad_norm": 0.130859375,
|
||
|
|
"learning_rate": 4.978294583898195e-07,
|
||
|
|
"logits/chosen": -1.8059618473052979,
|
||
|
|
"logits/rejected": -1.8209705352783203,
|
||
|
|
"logps/chosen": -0.2989288568496704,
|
||
|
|
"logps/rejected": -0.3068915009498596,
|
||
|
|
"loss": 0.6905549168586731,
|
||
|
|
"loss/core": 0.6905549168586731,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.0661938190460205,
|
||
|
|
"rewards/margins": 1.0517460107803345,
|
||
|
|
"rewards/rejected": 1.0144474506378174,
|
||
|
|
"step": 125
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1242087662725427,
|
||
|
|
"grad_norm": 0.27734375,
|
||
|
|
"learning_rate": 4.971454298742779e-07,
|
||
|
|
"logits/chosen": -1.6160223484039307,
|
||
|
|
"logits/rejected": -1.7299797534942627,
|
||
|
|
"logps/chosen": -0.27980923652648926,
|
||
|
|
"logps/rejected": -0.2786286771297455,
|
||
|
|
"loss": 0.6908574104309082,
|
||
|
|
"loss/core": 0.6908574104309082,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 1.8091075420379639,
|
||
|
|
"rewards/margins": 0.9205910563468933,
|
||
|
|
"rewards/rejected": 0.8885166049003601,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.12898602651379434,
|
||
|
|
"grad_norm": 0.408203125,
|
||
|
|
"learning_rate": 4.963684600700678e-07,
|
||
|
|
"logits/chosen": -1.771058440208435,
|
||
|
|
"logits/rejected": -1.842339277267456,
|
||
|
|
"logps/chosen": -0.3017735481262207,
|
||
|
|
"logps/rejected": -0.29897257685661316,
|
||
|
|
"loss": 0.6880813837051392,
|
||
|
|
"loss/core": 0.6880813837051392,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 3.3773961067199707,
|
||
|
|
"rewards/margins": 2.0708842277526855,
|
||
|
|
"rewards/rejected": 1.3065119981765747,
|
||
|
|
"step": 135
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.13376328675504598,
|
||
|
|
"grad_norm": 0.1669921875,
|
||
|
|
"learning_rate": 4.954988411637571e-07,
|
||
|
|
"logits/chosen": -2.0918688774108887,
|
||
|
|
"logits/rejected": -2.117227077484131,
|
||
|
|
"logps/chosen": -0.28260543942451477,
|
||
|
|
"logps/rejected": -0.28928446769714355,
|
||
|
|
"loss": 0.6908842325210571,
|
||
|
|
"loss/core": 0.6908842325210571,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.6272077560424805,
|
||
|
|
"rewards/margins": 0.9105762243270874,
|
||
|
|
"rewards/rejected": 0.7166314721107483,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.13854054699629761,
|
||
|
|
"grad_norm": 0.1630859375,
|
||
|
|
"learning_rate": 4.945369001834514e-07,
|
||
|
|
"logits/chosen": -1.5617491006851196,
|
||
|
|
"logits/rejected": -1.5613640546798706,
|
||
|
|
"logps/chosen": -0.3025224804878235,
|
||
|
|
"logps/rejected": -0.3009711802005768,
|
||
|
|
"loss": 0.6887967586517334,
|
||
|
|
"loss/core": 0.6887967586517334,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.592343807220459,
|
||
|
|
"rewards/margins": 1.7993513345718384,
|
||
|
|
"rewards/rejected": 1.7929922342300415,
|
||
|
|
"step": 145
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.14331780723754928,
|
||
|
|
"grad_norm": 0.404296875,
|
||
|
|
"learning_rate": 4.93482998875813e-07,
|
||
|
|
"logits/chosen": -1.913665771484375,
|
||
|
|
"logits/rejected": -1.9741178750991821,
|
||
|
|
"logps/chosen": -0.3164519667625427,
|
||
|
|
"logps/rejected": -0.30851152539253235,
|
||
|
|
"loss": 0.6878332495689392,
|
||
|
|
"loss/core": 0.6878332495689392,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.6481528282165527,
|
||
|
|
"rewards/margins": 2.1936802864074707,
|
||
|
|
"rewards/rejected": 1.4544728994369507,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1480950674788009,
|
||
|
|
"grad_norm": 0.2041015625,
|
||
|
|
"learning_rate": 4.923375335700223e-07,
|
||
|
|
"logits/chosen": -1.8092594146728516,
|
||
|
|
"logits/rejected": -1.8692359924316406,
|
||
|
|
"logps/chosen": -0.284790575504303,
|
||
|
|
"logps/rejected": -0.2954999506473541,
|
||
|
|
"loss": 0.6881645917892456,
|
||
|
|
"loss/core": 0.6881645917892456,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.8475961685180664,
|
||
|
|
"rewards/margins": 2.052863597869873,
|
||
|
|
"rewards/rejected": 0.7947325110435486,
|
||
|
|
"step": 155
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.15287232772005255,
|
||
|
|
"grad_norm": 0.91015625,
|
||
|
|
"learning_rate": 4.911009350287347e-07,
|
||
|
|
"logits/chosen": -1.7557824850082397,
|
||
|
|
"logits/rejected": -1.7192802429199219,
|
||
|
|
"logps/chosen": -0.3111857771873474,
|
||
|
|
"logps/rejected": -0.3161669075489044,
|
||
|
|
"loss": 0.6890115737915039,
|
||
|
|
"loss/core": 0.6890115737915039,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 3.1435818672180176,
|
||
|
|
"rewards/margins": 1.7508246898651123,
|
||
|
|
"rewards/rejected": 1.3927572965621948,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.15764958796130418,
|
||
|
|
"grad_norm": 0.2294921875,
|
||
|
|
"learning_rate": 4.897736682860885e-07,
|
||
|
|
"logits/chosen": -1.7782700061798096,
|
||
|
|
"logits/rejected": -1.7287534475326538,
|
||
|
|
"logps/chosen": -0.31163567304611206,
|
||
|
|
"logps/rejected": -0.3064088225364685,
|
||
|
|
"loss": 0.69112628698349,
|
||
|
|
"loss/core": 0.69112628698349,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.5969486236572266,
|
||
|
|
"rewards/margins": 0.8150261044502258,
|
||
|
|
"rewards/rejected": 0.7819225788116455,
|
||
|
|
"step": 165
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.16242684820255585,
|
||
|
|
"grad_norm": 0.8828125,
|
||
|
|
"learning_rate": 4.883562324728241e-07,
|
||
|
|
"logits/chosen": -1.8792940378189087,
|
||
|
|
"logits/rejected": -1.8148422241210938,
|
||
|
|
"logps/chosen": -0.27398645877838135,
|
||
|
|
"logps/rejected": -0.2596638798713684,
|
||
|
|
"loss": 0.6938985586166382,
|
||
|
|
"loss/core": 0.6938985586166382,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.6043877601623535,
|
||
|
|
"rewards/margins": -0.16653324663639069,
|
||
|
|
"rewards/rejected": 2.770921230316162,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.16720410844380748,
|
||
|
|
"grad_norm": 1.359375,
|
||
|
|
"learning_rate": 4.868491606285823e-07,
|
||
|
|
"logits/chosen": -1.7575905323028564,
|
||
|
|
"logits/rejected": -1.772924780845642,
|
||
|
|
"logps/chosen": -0.28738316893577576,
|
||
|
|
"logps/rejected": -0.3002680242061615,
|
||
|
|
"loss": 0.6899343729019165,
|
||
|
|
"loss/core": 0.6899343729019165,
|
||
|
|
"rewards/accuracies": 0.987500011920929,
|
||
|
|
"rewards/chosen": 2.5785281658172607,
|
||
|
|
"rewards/margins": 1.3044723272323608,
|
||
|
|
"rewards/rejected": 1.274056077003479,
|
||
|
|
"step": 175
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.17198136868505912,
|
||
|
|
"grad_norm": 1.453125,
|
||
|
|
"learning_rate": 4.852530195014489e-07,
|
||
|
|
"logits/chosen": -1.8603105545043945,
|
||
|
|
"logits/rejected": -1.8089433908462524,
|
||
|
|
"logps/chosen": -0.2715868353843689,
|
||
|
|
"logps/rejected": -0.26199498772621155,
|
||
|
|
"loss": 0.6889883279800415,
|
||
|
|
"loss/core": 0.6889883279800415,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 3.23846697807312,
|
||
|
|
"rewards/margins": 1.7489553689956665,
|
||
|
|
"rewards/rejected": 1.4895117282867432,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.17675862892631075,
|
||
|
|
"grad_norm": 0.16015625,
|
||
|
|
"learning_rate": 4.835684093348244e-07,
|
||
|
|
"logits/chosen": -1.9552524089813232,
|
||
|
|
"logits/rejected": -1.9065921306610107,
|
||
|
|
"logps/chosen": -0.2885257303714752,
|
||
|
|
"logps/rejected": -0.2921094298362732,
|
||
|
|
"loss": 0.6921087503433228,
|
||
|
|
"loss/core": 0.6921087503433228,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.0444104671478271,
|
||
|
|
"rewards/margins": 0.41770505905151367,
|
||
|
|
"rewards/rejected": 0.6267052888870239,
|
||
|
|
"step": 185
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18153588916756241,
|
||
|
|
"grad_norm": 0.341796875,
|
||
|
|
"learning_rate": 4.817959636416969e-07,
|
||
|
|
"logits/chosen": -1.9302583932876587,
|
||
|
|
"logits/rejected": -1.9251024723052979,
|
||
|
|
"logps/chosen": -0.2820015251636505,
|
||
|
|
"logps/rejected": -0.2878292202949524,
|
||
|
|
"loss": 0.6891991496086121,
|
||
|
|
"loss/core": 0.6891991496086121,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.8541626930236816,
|
||
|
|
"rewards/margins": 1.6507861614227295,
|
||
|
|
"rewards/rejected": 1.2033761739730835,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18631314940881405,
|
||
|
|
"grad_norm": 0.2275390625,
|
||
|
|
"learning_rate": 4.799363489664039e-07,
|
||
|
|
"logits/chosen": -1.654236078262329,
|
||
|
|
"logits/rejected": -1.6461032629013062,
|
||
|
|
"logps/chosen": -0.2908519208431244,
|
||
|
|
"logps/rejected": -0.2898189425468445,
|
||
|
|
"loss": 0.6891365051269531,
|
||
|
|
"loss/core": 0.6891365051269531,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.66994047164917,
|
||
|
|
"rewards/margins": 1.7045999765396118,
|
||
|
|
"rewards/rejected": 0.9653403162956238,
|
||
|
|
"step": 195
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.19109040965006568,
|
||
|
|
"grad_norm": 0.205078125,
|
||
|
|
"learning_rate": 4.779902646339721e-07,
|
||
|
|
"logits/chosen": -1.722882628440857,
|
||
|
|
"logits/rejected": -1.7433340549468994,
|
||
|
|
"logps/chosen": -0.3230392038822174,
|
||
|
|
"logps/rejected": -0.31719517707824707,
|
||
|
|
"loss": 0.689689040184021,
|
||
|
|
"loss/core": 0.689689040184021,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.6197898387908936,
|
||
|
|
"rewards/margins": 1.4049842357635498,
|
||
|
|
"rewards/rejected": 1.2148058414459229,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.19586766989131732,
|
||
|
|
"grad_norm": 0.138671875,
|
||
|
|
"learning_rate": 4.759584424871301e-07,
|
||
|
|
"logits/chosen": -1.8010181188583374,
|
||
|
|
"logits/rejected": -1.880504846572876,
|
||
|
|
"logps/chosen": -0.25066837668418884,
|
||
|
|
"logps/rejected": -0.2679479718208313,
|
||
|
|
"loss": 0.6904597282409668,
|
||
|
|
"loss/core": 0.6904597282409668,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 3.283839702606201,
|
||
|
|
"rewards/margins": 1.1656420230865479,
|
||
|
|
"rewards/rejected": 2.1181979179382324,
|
||
|
|
"step": 205
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20064493013256898,
|
||
|
|
"grad_norm": 0.353515625,
|
||
|
|
"learning_rate": 4.738416466110917e-07,
|
||
|
|
"logits/chosen": -1.8253402709960938,
|
||
|
|
"logits/rejected": -1.8085130453109741,
|
||
|
|
"logps/chosen": -0.3120880722999573,
|
||
|
|
"logps/rejected": -0.3225416839122772,
|
||
|
|
"loss": 0.6898230314254761,
|
||
|
|
"loss/core": 0.6898230314254761,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.835217237472534,
|
||
|
|
"rewards/margins": 1.358270525932312,
|
||
|
|
"rewards/rejected": 1.4769468307495117,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20542219037382062,
|
||
|
|
"grad_norm": 0.345703125,
|
||
|
|
"learning_rate": 4.716406730462153e-07,
|
||
|
|
"logits/chosen": -1.854882836341858,
|
||
|
|
"logits/rejected": -1.812995195388794,
|
||
|
|
"logps/chosen": -0.2745797038078308,
|
||
|
|
"logps/rejected": -0.2867583930492401,
|
||
|
|
"loss": 0.689670205116272,
|
||
|
|
"loss/core": 0.689670205116272,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 3.2003917694091797,
|
||
|
|
"rewards/margins": 1.5229412317276,
|
||
|
|
"rewards/rejected": 1.6774505376815796,
|
||
|
|
"step": 215
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.21019945061507225,
|
||
|
|
"grad_norm": 0.353515625,
|
||
|
|
"learning_rate": 4.693563494886454e-07,
|
||
|
|
"logits/chosen": -1.7740695476531982,
|
||
|
|
"logits/rejected": -1.7250359058380127,
|
||
|
|
"logps/chosen": -0.30255165696144104,
|
||
|
|
"logps/rejected": -0.30088987946510315,
|
||
|
|
"loss": 0.6911439895629883,
|
||
|
|
"loss/core": 0.6911439895629883,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.885136365890503,
|
||
|
|
"rewards/margins": 0.8066938519477844,
|
||
|
|
"rewards/rejected": 1.0784424543380737,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2149767108563239,
|
||
|
|
"grad_norm": 0.193359375,
|
||
|
|
"learning_rate": 4.6698953497905016e-07,
|
||
|
|
"logits/chosen": -1.8212635517120361,
|
||
|
|
"logits/rejected": -1.884863257408142,
|
||
|
|
"logps/chosen": -0.2905574440956116,
|
||
|
|
"logps/rejected": -0.2746399939060211,
|
||
|
|
"loss": 0.6877695918083191,
|
||
|
|
"loss/core": 0.6877695918083191,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 3.6224703788757324,
|
||
|
|
"rewards/margins": 2.204864501953125,
|
||
|
|
"rewards/rejected": 1.417605996131897,
|
||
|
|
"step": 225
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.21975397109757555,
|
||
|
|
"grad_norm": 1.875,
|
||
|
|
"learning_rate": 4.645411195795709e-07,
|
||
|
|
"logits/chosen": -1.955877661705017,
|
||
|
|
"logits/rejected": -1.9922053813934326,
|
||
|
|
"logps/chosen": -0.28303641080856323,
|
||
|
|
"logps/rejected": -0.305550754070282,
|
||
|
|
"loss": 0.6905184984207153,
|
||
|
|
"loss/core": 0.6905184984207153,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.8412774801254272,
|
||
|
|
"rewards/margins": 1.0649278163909912,
|
||
|
|
"rewards/rejected": 0.776349663734436,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2245312313388272,
|
||
|
|
"grad_norm": 0.2451171875,
|
||
|
|
"learning_rate": 4.6201202403910643e-07,
|
||
|
|
"logits/chosen": -1.695408821105957,
|
||
|
|
"logits/rejected": -1.7123172283172607,
|
||
|
|
"logps/chosen": -0.2921196520328522,
|
||
|
|
"logps/rejected": -0.2863583266735077,
|
||
|
|
"loss": 0.691733717918396,
|
||
|
|
"loss/core": 0.691733717918396,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.9099562168121338,
|
||
|
|
"rewards/margins": 0.5792465806007385,
|
||
|
|
"rewards/rejected": 1.33070969581604,
|
||
|
|
"step": 235
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.22930849158007882,
|
||
|
|
"grad_norm": 0.2353515625,
|
||
|
|
"learning_rate": 4.594031994470573e-07,
|
||
|
|
"logits/chosen": -1.7617086172103882,
|
||
|
|
"logits/rejected": -1.7821518182754517,
|
||
|
|
"logps/chosen": -0.2942032217979431,
|
||
|
|
"logps/rejected": -0.2943900525569916,
|
||
|
|
"loss": 0.6913124918937683,
|
||
|
|
"loss/core": 0.6913124918937683,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.7612661123275757,
|
||
|
|
"rewards/margins": 0.7373276948928833,
|
||
|
|
"rewards/rejected": 1.023938536643982,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.23408575182133046,
|
||
|
|
"grad_norm": 1.8515625,
|
||
|
|
"learning_rate": 4.567156268756593e-07,
|
||
|
|
"logits/chosen": -1.919356346130371,
|
||
|
|
"logits/rejected": -1.896200180053711,
|
||
|
|
"logps/chosen": -0.2767832577228546,
|
||
|
|
"logps/rejected": -0.29151099920272827,
|
||
|
|
"loss": 0.6873155832290649,
|
||
|
|
"loss/core": 0.6873155832290649,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 3.6117758750915527,
|
||
|
|
"rewards/margins": 2.4639251232147217,
|
||
|
|
"rewards/rejected": 1.1478503942489624,
|
||
|
|
"step": 245
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.23886301206258212,
|
||
|
|
"grad_norm": 0.29296875,
|
||
|
|
"learning_rate": 4.5395031701104303e-07,
|
||
|
|
"logits/chosen": -1.6390726566314697,
|
||
|
|
"logits/rejected": -1.6990959644317627,
|
||
|
|
"logps/chosen": -0.3004065454006195,
|
||
|
|
"logps/rejected": -0.2949826121330261,
|
||
|
|
"loss": 0.6900212168693542,
|
||
|
|
"loss/core": 0.6900212168693542,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.3332924842834473,
|
||
|
|
"rewards/margins": 1.2689025402069092,
|
||
|
|
"rewards/rejected": 1.0643898248672485,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.24364027230383375,
|
||
|
|
"grad_norm": 0.1708984375,
|
||
|
|
"learning_rate": 4.511083097731555e-07,
|
||
|
|
"logits/chosen": -1.8358352184295654,
|
||
|
|
"logits/rejected": -1.8455946445465088,
|
||
|
|
"logps/chosen": -0.28558123111724854,
|
||
|
|
"logps/rejected": -0.28805023431777954,
|
||
|
|
"loss": 0.6910611391067505,
|
||
|
|
"loss/core": 0.6910611391067505,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.444948673248291,
|
||
|
|
"rewards/margins": 0.8477497100830078,
|
||
|
|
"rewards/rejected": 1.597198724746704,
|
||
|
|
"step": 255
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2484175325450854,
|
||
|
|
"grad_norm": 0.08935546875,
|
||
|
|
"learning_rate": 4.481906739246894e-07,
|
||
|
|
"logits/chosen": -1.9092012643814087,
|
||
|
|
"logits/rejected": -1.9937626123428345,
|
||
|
|
"logps/chosen": -0.28719645738601685,
|
||
|
|
"logps/rejected": -0.2705669701099396,
|
||
|
|
"loss": 0.691189169883728,
|
||
|
|
"loss/core": 0.691189169883728,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.4050984382629395,
|
||
|
|
"rewards/margins": 0.8234094381332397,
|
||
|
|
"rewards/rejected": 1.5816890001296997,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.25319479278633705,
|
||
|
|
"grad_norm": 0.08447265625,
|
||
|
|
"learning_rate": 4.451985066691648e-07,
|
||
|
|
"logits/chosen": -1.7978445291519165,
|
||
|
|
"logits/rejected": -1.8153514862060547,
|
||
|
|
"logps/chosen": -0.29918935894966125,
|
||
|
|
"logps/rejected": -0.30210354924201965,
|
||
|
|
"loss": 0.6931018233299255,
|
||
|
|
"loss/core": 0.6931018233299255,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.2333295345306396,
|
||
|
|
"rewards/margins": 0.06469568610191345,
|
||
|
|
"rewards/rejected": 1.1686336994171143,
|
||
|
|
"step": 265
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2579720530275887,
|
||
|
|
"grad_norm": 0.16015625,
|
||
|
|
"learning_rate": 4.421329332383158e-07,
|
||
|
|
"logits/chosen": -1.6397037506103516,
|
||
|
|
"logits/rejected": -1.7484862804412842,
|
||
|
|
"logps/chosen": -0.2880600392818451,
|
||
|
|
"logps/rejected": -0.2899312376976013,
|
||
|
|
"loss": 0.6904688477516174,
|
||
|
|
"loss/core": 0.6904688477516174,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.2856483459472656,
|
||
|
|
"rewards/margins": 1.1514227390289307,
|
||
|
|
"rewards/rejected": 2.134225606918335,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2627493132688403,
|
||
|
|
"grad_norm": 1.3125,
|
||
|
|
"learning_rate": 4.3899510646893696e-07,
|
||
|
|
"logits/chosen": -1.868186593055725,
|
||
|
|
"logits/rejected": -1.8120629787445068,
|
||
|
|
"logps/chosen": -0.29679086804389954,
|
||
|
|
"logps/rejected": -0.29938483238220215,
|
||
|
|
"loss": 0.6903032064437866,
|
||
|
|
"loss/core": 0.6903032064437866,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 2.321898937225342,
|
||
|
|
"rewards/margins": 1.151911735534668,
|
||
|
|
"rewards/rejected": 1.1699872016906738,
|
||
|
|
"step": 275
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.26752657351009196,
|
||
|
|
"grad_norm": 0.1435546875,
|
||
|
|
"learning_rate": 4.357862063693485e-07,
|
||
|
|
"logits/chosen": -1.7666244506835938,
|
||
|
|
"logits/rejected": -1.7176767587661743,
|
||
|
|
"logps/chosen": -0.28811368346214294,
|
||
|
|
"logps/rejected": -0.3051670491695404,
|
||
|
|
"loss": 0.6900285482406616,
|
||
|
|
"loss/core": 0.6900285482406616,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.0461502075195312,
|
||
|
|
"rewards/margins": 1.2626762390136719,
|
||
|
|
"rewards/rejected": 0.7834742069244385,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2723038337513436,
|
||
|
|
"grad_norm": 0.8359375,
|
||
|
|
"learning_rate": 4.3250743967564364e-07,
|
||
|
|
"logits/chosen": -1.8882230520248413,
|
||
|
|
"logits/rejected": -1.8435192108154297,
|
||
|
|
"logps/chosen": -0.2955285310745239,
|
||
|
|
"logps/rejected": -0.2927626669406891,
|
||
|
|
"loss": 0.6907385587692261,
|
||
|
|
"loss/core": 0.6907385587692261,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.62160325050354,
|
||
|
|
"rewards/margins": 0.9927070736885071,
|
||
|
|
"rewards/rejected": 1.6288961172103882,
|
||
|
|
"step": 285
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.27708109399259523,
|
||
|
|
"grad_norm": 1.984375,
|
||
|
|
"learning_rate": 4.29160039397884e-07,
|
||
|
|
"logits/chosen": -1.8738224506378174,
|
||
|
|
"logits/rejected": -1.9035943746566772,
|
||
|
|
"logps/chosen": -0.2865127623081207,
|
||
|
|
"logps/rejected": -0.28757035732269287,
|
||
|
|
"loss": 0.6931725740432739,
|
||
|
|
"loss/core": 0.6931725740432739,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.4819949865341187,
|
||
|
|
"rewards/margins": 0.02711881324648857,
|
||
|
|
"rewards/rejected": 1.454876184463501,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.28185835423384686,
|
||
|
|
"grad_norm": 0.171875,
|
||
|
|
"learning_rate": 4.2574526435641546e-07,
|
||
|
|
"logits/chosen": -2.1733994483947754,
|
||
|
|
"logits/rejected": -2.1729981899261475,
|
||
|
|
"logps/chosen": -0.24496681988239288,
|
||
|
|
"logps/rejected": -0.25612831115722656,
|
||
|
|
"loss": 0.6915844082832336,
|
||
|
|
"loss/core": 0.6915844082832336,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 1.3056122064590454,
|
||
|
|
"rewards/margins": 0.6268185973167419,
|
||
|
|
"rewards/rejected": 0.6787935495376587,
|
||
|
|
"step": 295
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.28663561447509855,
|
||
|
|
"grad_norm": 2.359375,
|
||
|
|
"learning_rate": 4.22264398708477e-07,
|
||
|
|
"logits/chosen": -1.5950920581817627,
|
||
|
|
"logits/rejected": -1.6194194555282593,
|
||
|
|
"logps/chosen": -0.31406429409980774,
|
||
|
|
"logps/rejected": -0.3146823048591614,
|
||
|
|
"loss": 0.6867777109146118,
|
||
|
|
"loss/core": 0.6867777109146118,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 3.7072041034698486,
|
||
|
|
"rewards/margins": 2.6247377395629883,
|
||
|
|
"rewards/rejected": 1.08246648311615,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2914128747163502,
|
||
|
|
"grad_norm": 0.2119140625,
|
||
|
|
"learning_rate": 4.187187514652819e-07,
|
||
|
|
"logits/chosen": -1.7839601039886475,
|
||
|
|
"logits/rejected": -1.8283946514129639,
|
||
|
|
"logps/chosen": -0.27006229758262634,
|
||
|
|
"logps/rejected": -0.25947099924087524,
|
||
|
|
"loss": 0.6900721788406372,
|
||
|
|
"loss/core": 0.6900721788406372,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.1529765129089355,
|
||
|
|
"rewards/margins": 1.242806077003479,
|
||
|
|
"rewards/rejected": 0.9101703763008118,
|
||
|
|
"step": 305
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2961901349576018,
|
||
|
|
"grad_norm": 2.5,
|
||
|
|
"learning_rate": 4.151096559997519e-07,
|
||
|
|
"logits/chosen": -1.8296064138412476,
|
||
|
|
"logits/rejected": -1.7307389974594116,
|
||
|
|
"logps/chosen": -0.2725442945957184,
|
||
|
|
"logps/rejected": -0.27926790714263916,
|
||
|
|
"loss": 0.6918596625328064,
|
||
|
|
"loss/core": 0.6918596625328064,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.6172053813934326,
|
||
|
|
"rewards/margins": 0.5880039930343628,
|
||
|
|
"rewards/rejected": 2.029201030731201,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.30096739519885346,
|
||
|
|
"grad_norm": 0.158203125,
|
||
|
|
"learning_rate": 4.114384695450905e-07,
|
||
|
|
"logits/chosen": -1.722001314163208,
|
||
|
|
"logits/rejected": -1.8182132244110107,
|
||
|
|
"logps/chosen": -0.26871687173843384,
|
||
|
|
"logps/rejected": -0.2755733132362366,
|
||
|
|
"loss": 0.6892502903938293,
|
||
|
|
"loss/core": 0.6892502903938293,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.917477607727051,
|
||
|
|
"rewards/margins": 1.570624828338623,
|
||
|
|
"rewards/rejected": 1.3468530178070068,
|
||
|
|
"step": 315
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3057446554401051,
|
||
|
|
"grad_norm": 0.134765625,
|
||
|
|
"learning_rate": 4.077065726843828e-07,
|
||
|
|
"logits/chosen": -1.7197027206420898,
|
||
|
|
"logits/rejected": -1.6908174753189087,
|
||
|
|
"logps/chosen": -0.2902878522872925,
|
||
|
|
"logps/rejected": -0.29251793026924133,
|
||
|
|
"loss": 0.6922987103462219,
|
||
|
|
"loss/core": 0.6922987103462219,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 1.5793731212615967,
|
||
|
|
"rewards/margins": 0.34973055124282837,
|
||
|
|
"rewards/rejected": 1.229642629623413,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.31052191568135673,
|
||
|
|
"grad_norm": 1.2578125,
|
||
|
|
"learning_rate": 4.039153688314145e-07,
|
||
|
|
"logits/chosen": -1.9648669958114624,
|
||
|
|
"logits/rejected": -1.9164518117904663,
|
||
|
|
"logps/chosen": -0.2875637114048004,
|
||
|
|
"logps/rejected": -0.28430742025375366,
|
||
|
|
"loss": 0.688163697719574,
|
||
|
|
"loss/core": 0.688163697719574,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 3.6862456798553467,
|
||
|
|
"rewards/margins": 2.1581714153289795,
|
||
|
|
"rewards/rejected": 1.5280741453170776,
|
||
|
|
"step": 325
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.31529917592260837,
|
||
|
|
"grad_norm": 0.103515625,
|
||
|
|
"learning_rate": 4.0006628370290613e-07,
|
||
|
|
"logits/chosen": -1.8999824523925781,
|
||
|
|
"logits/rejected": -1.8628829717636108,
|
||
|
|
"logps/chosen": -0.2762024998664856,
|
||
|
|
"logps/rejected": -0.2705720365047455,
|
||
|
|
"loss": 0.6908760070800781,
|
||
|
|
"loss/core": 0.6908760070800781,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 2.054731845855713,
|
||
|
|
"rewards/margins": 0.9240147471427917,
|
||
|
|
"rewards/rejected": 1.130717158317566,
|
||
|
|
"step": 330
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.32007643616386,
|
||
|
|
"grad_norm": 3.0,
|
||
|
|
"learning_rate": 3.9616076478235826e-07,
|
||
|
|
"logits/chosen": -1.8090893030166626,
|
||
|
|
"logits/rejected": -1.8052091598510742,
|
||
|
|
"logps/chosen": -0.30249348282814026,
|
||
|
|
"logps/rejected": -0.31619948148727417,
|
||
|
|
"loss": 0.6897532343864441,
|
||
|
|
"loss/core": 0.6897532343864441,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.117969036102295,
|
||
|
|
"rewards/margins": 1.4059538841247559,
|
||
|
|
"rewards/rejected": 0.7120150327682495,
|
||
|
|
"step": 335
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3248536964051117,
|
||
|
|
"grad_norm": 1.0390625,
|
||
|
|
"learning_rate": 3.922002807757129e-07,
|
||
|
|
"logits/chosen": -1.8413950204849243,
|
||
|
|
"logits/rejected": -1.9030017852783203,
|
||
|
|
"logps/chosen": -0.2811221480369568,
|
||
|
|
"logps/rejected": -0.28223514556884766,
|
||
|
|
"loss": 0.6902806162834167,
|
||
|
|
"loss/core": 0.6902806162834167,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.029637575149536,
|
||
|
|
"rewards/margins": 1.1650125980377197,
|
||
|
|
"rewards/rejected": 0.8646249771118164,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3296309566463633,
|
||
|
|
"grad_norm": 0.259765625,
|
||
|
|
"learning_rate": 3.8818632105903315e-07,
|
||
|
|
"logits/chosen": -1.8142598867416382,
|
||
|
|
"logits/rejected": -1.8903926610946655,
|
||
|
|
"logps/chosen": -0.28178513050079346,
|
||
|
|
"logps/rejected": -0.29352661967277527,
|
||
|
|
"loss": 0.6911460757255554,
|
||
|
|
"loss/core": 0.6911460757255554,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 1.6476151943206787,
|
||
|
|
"rewards/margins": 0.8132193684577942,
|
||
|
|
"rewards/rejected": 0.8343957662582397,
|
||
|
|
"step": 345
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.33440821688761496,
|
||
|
|
"grad_norm": 0.2060546875,
|
||
|
|
"learning_rate": 3.841203951184094e-07,
|
||
|
|
"logits/chosen": -1.8314239978790283,
|
||
|
|
"logits/rejected": -1.8243396282196045,
|
||
|
|
"logps/chosen": -0.2603296935558319,
|
||
|
|
"logps/rejected": -0.2569552958011627,
|
||
|
|
"loss": 0.6905307769775391,
|
||
|
|
"loss/core": 0.6905307769775391,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.9986594915390015,
|
||
|
|
"rewards/margins": 1.056462287902832,
|
||
|
|
"rewards/rejected": 0.9421975016593933,
|
||
|
|
"step": 350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3391854771288666,
|
||
|
|
"grad_norm": 0.201171875,
|
||
|
|
"learning_rate": 3.800040319823038e-07,
|
||
|
|
"logits/chosen": -1.7914011478424072,
|
||
|
|
"logits/rejected": -1.8717437982559204,
|
||
|
|
"logps/chosen": -0.2806769907474518,
|
||
|
|
"logps/rejected": -0.2930828928947449,
|
||
|
|
"loss": 0.691979169845581,
|
||
|
|
"loss/core": 0.691979169845581,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.851120948791504,
|
||
|
|
"rewards/margins": 0.5048354864120483,
|
||
|
|
"rewards/rejected": 1.346285343170166,
|
||
|
|
"step": 355
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.34396273737011823,
|
||
|
|
"grad_norm": 1.40625,
|
||
|
|
"learning_rate": 3.75838779646545e-07,
|
||
|
|
"logits/chosen": -1.856149673461914,
|
||
|
|
"logits/rejected": -1.932403326034546,
|
||
|
|
"logps/chosen": -0.2797439694404602,
|
||
|
|
"logps/rejected": -0.2799036204814911,
|
||
|
|
"loss": 0.6898868680000305,
|
||
|
|
"loss/core": 0.6898868680000305,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.6374688148498535,
|
||
|
|
"rewards/margins": 1.3426095247268677,
|
||
|
|
"rewards/rejected": 1.2948594093322754,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.34873999761136987,
|
||
|
|
"grad_norm": 1.0390625,
|
||
|
|
"learning_rate": 3.7162620449218993e-07,
|
||
|
|
"logits/chosen": -1.8249025344848633,
|
||
|
|
"logits/rejected": -1.8712995052337646,
|
||
|
|
"logps/chosen": -0.28412720561027527,
|
||
|
|
"logps/rejected": -0.28386378288269043,
|
||
|
|
"loss": 0.6882467865943909,
|
||
|
|
"loss/core": 0.6882467865943909,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.882859945297241,
|
||
|
|
"rewards/margins": 2.0732054710388184,
|
||
|
|
"rewards/rejected": 0.8096548318862915,
|
||
|
|
"step": 365
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3535172578526215,
|
||
|
|
"grad_norm": 0.384765625,
|
||
|
|
"learning_rate": 3.673678906964727e-07,
|
||
|
|
"logits/chosen": -1.9151166677474976,
|
||
|
|
"logits/rejected": -2.025606155395508,
|
||
|
|
"logps/chosen": -0.28493157029151917,
|
||
|
|
"logps/rejected": -0.28482019901275635,
|
||
|
|
"loss": 0.6917159557342529,
|
||
|
|
"loss/core": 0.6917159557342529,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.6433870792388916,
|
||
|
|
"rewards/margins": 0.576850175857544,
|
||
|
|
"rewards/rejected": 1.0665369033813477,
|
||
|
|
"step": 370
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.35829451809387314,
|
||
|
|
"grad_norm": 0.1875,
|
||
|
|
"learning_rate": 3.6306543963705936e-07,
|
||
|
|
"logits/chosen": -1.7109639644622803,
|
||
|
|
"logits/rejected": -1.7306935787200928,
|
||
|
|
"logps/chosen": -0.25769031047821045,
|
||
|
|
"logps/rejected": -0.25766560435295105,
|
||
|
|
"loss": 0.6912951469421387,
|
||
|
|
"loss/core": 0.6912951469421387,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 3.024385929107666,
|
||
|
|
"rewards/margins": 0.7996337413787842,
|
||
|
|
"rewards/rejected": 2.22475266456604,
|
||
|
|
"step": 375
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.36307177833512483,
|
||
|
|
"grad_norm": 0.26171875,
|
||
|
|
"learning_rate": 3.5872046928983623e-07,
|
||
|
|
"logits/chosen": -1.8375533819198608,
|
||
|
|
"logits/rejected": -1.8777393102645874,
|
||
|
|
"logps/chosen": -0.2792344093322754,
|
||
|
|
"logps/rejected": -0.2905234694480896,
|
||
|
|
"loss": 0.6921393871307373,
|
||
|
|
"loss/core": 0.6921393871307373,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.058870315551758,
|
||
|
|
"rewards/margins": 0.4689110219478607,
|
||
|
|
"rewards/rejected": 1.5899592638015747,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.36784903857637646,
|
||
|
|
"grad_norm": 0.361328125,
|
||
|
|
"learning_rate": 3.5433461362045447e-07,
|
||
|
|
"logits/chosen": -1.8220224380493164,
|
||
|
|
"logits/rejected": -1.8235307931900024,
|
||
|
|
"logps/chosen": -0.2701142430305481,
|
||
|
|
"logps/rejected": -0.27504315972328186,
|
||
|
|
"loss": 0.6915521621704102,
|
||
|
|
"loss/core": 0.6915521621704102,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 2.1791160106658936,
|
||
|
|
"rewards/margins": 0.6585676670074463,
|
||
|
|
"rewards/rejected": 1.5205484628677368,
|
||
|
|
"step": 385
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3726262988176281,
|
||
|
|
"grad_norm": 0.1103515625,
|
||
|
|
"learning_rate": 3.4990952196986305e-07,
|
||
|
|
"logits/chosen": -1.6957165002822876,
|
||
|
|
"logits/rejected": -1.7675812244415283,
|
||
|
|
"logps/chosen": -0.30696994066238403,
|
||
|
|
"logps/rejected": -0.3029830753803253,
|
||
|
|
"loss": 0.6909955739974976,
|
||
|
|
"loss/core": 0.6909955739974976,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.7640354633331299,
|
||
|
|
"rewards/margins": 0.8730796575546265,
|
||
|
|
"rewards/rejected": 0.8909558057785034,
|
||
|
|
"step": 390
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.37740355905887973,
|
||
|
|
"grad_norm": 0.3359375,
|
||
|
|
"learning_rate": 3.4544685843405875e-07,
|
||
|
|
"logits/chosen": -1.924168348312378,
|
||
|
|
"logits/rejected": -1.979353666305542,
|
||
|
|
"logps/chosen": -0.2945513129234314,
|
||
|
|
"logps/rejected": -0.2900546193122864,
|
||
|
|
"loss": 0.6909527778625488,
|
||
|
|
"loss/core": 0.6909527778625488,
|
||
|
|
"rewards/accuracies": 0.9750000238418579,
|
||
|
|
"rewards/chosen": 1.686589241027832,
|
||
|
|
"rewards/margins": 0.8840621113777161,
|
||
|
|
"rewards/rejected": 0.8025272488594055,
|
||
|
|
"step": 395
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.38218081930013137,
|
||
|
|
"grad_norm": 0.2373046875,
|
||
|
|
"learning_rate": 3.4094830123828786e-07,
|
||
|
|
"logits/chosen": -1.6329810619354248,
|
||
|
|
"logits/rejected": -1.6912469863891602,
|
||
|
|
"logps/chosen": -0.28971177339553833,
|
||
|
|
"logps/rejected": -0.2938389778137207,
|
||
|
|
"loss": 0.6880810260772705,
|
||
|
|
"loss/core": 0.6880810260772705,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.956228256225586,
|
||
|
|
"rewards/margins": 2.1219091415405273,
|
||
|
|
"rewards/rejected": 0.8343192934989929,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.386958079541383,
|
||
|
|
"grad_norm": 0.1748046875,
|
||
|
|
"learning_rate": 3.3641554210593414e-07,
|
||
|
|
"logits/chosen": -1.8328759670257568,
|
||
|
|
"logits/rejected": -1.867147445678711,
|
||
|
|
"logps/chosen": -0.31374454498291016,
|
||
|
|
"logps/rejected": -0.3037855923175812,
|
||
|
|
"loss": 0.6904991269111633,
|
||
|
|
"loss/core": 0.6904991269111633,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.327601432800293,
|
||
|
|
"rewards/margins": 1.0923205614089966,
|
||
|
|
"rewards/rejected": 1.2352811098098755,
|
||
|
|
"step": 405
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.39173533978263464,
|
||
|
|
"grad_norm": 0.2216796875,
|
||
|
|
"learning_rate": 3.3185028562233107e-07,
|
||
|
|
"logits/chosen": -2.08441424369812,
|
||
|
|
"logits/rejected": -2.0821681022644043,
|
||
|
|
"logps/chosen": -0.27686241269111633,
|
||
|
|
"logps/rejected": -0.288194864988327,
|
||
|
|
"loss": 0.6909042596817017,
|
||
|
|
"loss/core": 0.6909042596817017,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.0894763469696045,
|
||
|
|
"rewards/margins": 0.9023572206497192,
|
||
|
|
"rewards/rejected": 1.1871192455291748,
|
||
|
|
"step": 410
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3965126000238863,
|
||
|
|
"grad_norm": 0.22265625,
|
||
|
|
"learning_rate": 3.272542485937368e-07,
|
||
|
|
"logits/chosen": -1.667150855064392,
|
||
|
|
"logits/rejected": -1.7496572732925415,
|
||
|
|
"logps/chosen": -0.32961294054985046,
|
||
|
|
"logps/rejected": -0.32087013125419617,
|
||
|
|
"loss": 0.6915945410728455,
|
||
|
|
"loss/core": 0.6915945410728455,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.0019575357437134,
|
||
|
|
"rewards/margins": 0.6251779794692993,
|
||
|
|
"rewards/rejected": 0.37677961587905884,
|
||
|
|
"step": 415
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.40128986026513797,
|
||
|
|
"grad_norm": 0.609375,
|
||
|
|
"learning_rate": 3.226291594017137e-07,
|
||
|
|
"logits/chosen": -2.015800952911377,
|
||
|
|
"logits/rejected": -2.012826919555664,
|
||
|
|
"logps/chosen": -0.28343337774276733,
|
||
|
|
"logps/rejected": -0.29794415831565857,
|
||
|
|
"loss": 0.6899334192276001,
|
||
|
|
"loss/core": 0.6899334192276001,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 2.0445852279663086,
|
||
|
|
"rewards/margins": 1.314929723739624,
|
||
|
|
"rewards/rejected": 0.7296555638313293,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4060671205063896,
|
||
|
|
"grad_norm": 0.8984375,
|
||
|
|
"learning_rate": 3.1797675735315454e-07,
|
||
|
|
"logits/chosen": -1.7338758707046509,
|
||
|
|
"logits/rejected": -1.8056646585464478,
|
||
|
|
"logps/chosen": -0.291433185338974,
|
||
|
|
"logps/rejected": -0.2666085660457611,
|
||
|
|
"loss": 0.691834032535553,
|
||
|
|
"loss/core": 0.691834032535553,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.2369046211242676,
|
||
|
|
"rewards/margins": 0.6058995127677917,
|
||
|
|
"rewards/rejected": 1.6310052871704102,
|
||
|
|
"step": 425
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.41084438074764124,
|
||
|
|
"grad_norm": 1.0,
|
||
|
|
"learning_rate": 3.1329879202620047e-07,
|
||
|
|
"logits/chosen": -1.8133704662322998,
|
||
|
|
"logits/rejected": -1.8503717184066772,
|
||
|
|
"logps/chosen": -0.2787948548793793,
|
||
|
|
"logps/rejected": -0.2662429213523865,
|
||
|
|
"loss": 0.6860483884811401,
|
||
|
|
"loss/core": 0.6860483884811401,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 4.410374641418457,
|
||
|
|
"rewards/margins": 2.9887146949768066,
|
||
|
|
"rewards/rejected": 1.4216595888137817,
|
||
|
|
"step": 430
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.41562164098889287,
|
||
|
|
"grad_norm": 0.2431640625,
|
||
|
|
"learning_rate": 3.0859702261229613e-07,
|
||
|
|
"logits/chosen": -2.042301893234253,
|
||
|
|
"logits/rejected": -1.9976842403411865,
|
||
|
|
"logps/chosen": -0.27745509147644043,
|
||
|
|
"logps/rejected": -0.29362621903419495,
|
||
|
|
"loss": 0.6915836334228516,
|
||
|
|
"loss/core": 0.6915836334228516,
|
||
|
|
"rewards/accuracies": 0.824999988079071,
|
||
|
|
"rewards/chosen": 1.6335694789886475,
|
||
|
|
"rewards/margins": 0.6322575807571411,
|
||
|
|
"rewards/rejected": 1.0013117790222168,
|
||
|
|
"step": 435
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4203989012301445,
|
||
|
|
"grad_norm": 0.97265625,
|
||
|
|
"learning_rate": 3.0387321725463e-07,
|
||
|
|
"logits/chosen": -1.752429723739624,
|
||
|
|
"logits/rejected": -1.7554237842559814,
|
||
|
|
"logps/chosen": -0.2960103452205658,
|
||
|
|
"logps/rejected": -0.3004111647605896,
|
||
|
|
"loss": 0.6891411542892456,
|
||
|
|
"loss/core": 0.6891411542892456,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 3.0792078971862793,
|
||
|
|
"rewards/margins": 1.6967817544937134,
|
||
|
|
"rewards/rejected": 1.3824257850646973,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.42517616147139614,
|
||
|
|
"grad_norm": 0.369140625,
|
||
|
|
"learning_rate": 2.991291523832075e-07,
|
||
|
|
"logits/chosen": -1.6757290363311768,
|
||
|
|
"logits/rejected": -1.6205558776855469,
|
||
|
|
"logps/chosen": -0.2944514751434326,
|
||
|
|
"logps/rejected": -0.29790419340133667,
|
||
|
|
"loss": 0.689667284488678,
|
||
|
|
"loss/core": 0.689667284488678,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.9721293449401855,
|
||
|
|
"rewards/margins": 1.4168652296066284,
|
||
|
|
"rewards/rejected": 1.5552639961242676,
|
||
|
|
"step": 445
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4299534217126478,
|
||
|
|
"grad_norm": 1.6953125,
|
||
|
|
"learning_rate": 2.943666120468088e-07,
|
||
|
|
"logits/chosen": -1.6464523077011108,
|
||
|
|
"logits/rejected": -1.6663997173309326,
|
||
|
|
"logps/chosen": -0.29781681299209595,
|
||
|
|
"logps/rejected": -0.29050326347351074,
|
||
|
|
"loss": 0.6886329054832458,
|
||
|
|
"loss/core": 0.6886329054832458,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 3.4628093242645264,
|
||
|
|
"rewards/margins": 1.8670060634613037,
|
||
|
|
"rewards/rejected": 1.5958033800125122,
|
||
|
|
"step": 450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4347306819538994,
|
||
|
|
"grad_norm": 0.58203125,
|
||
|
|
"learning_rate": 2.8958738724208073e-07,
|
||
|
|
"logits/chosen": -1.641239881515503,
|
||
|
|
"logits/rejected": -1.832718849182129,
|
||
|
|
"logps/chosen": -0.324314147233963,
|
||
|
|
"logps/rejected": -0.31268811225891113,
|
||
|
|
"loss": 0.688445508480072,
|
||
|
|
"loss/core": 0.688445508480072,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.4496636390686035,
|
||
|
|
"rewards/margins": 1.958392858505249,
|
||
|
|
"rewards/rejected": 1.4912705421447754,
|
||
|
|
"step": 455
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4395079421951511,
|
||
|
|
"grad_norm": 0.419921875,
|
||
|
|
"learning_rate": 2.8479327524001633e-07,
|
||
|
|
"logits/chosen": -1.6839364767074585,
|
||
|
|
"logits/rejected": -1.7105252742767334,
|
||
|
|
"logps/chosen": -0.2922411262989044,
|
||
|
|
"logps/rejected": -0.3082396388053894,
|
||
|
|
"loss": 0.688636839389801,
|
||
|
|
"loss/core": 0.688636839389801,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 3.7023568153381348,
|
||
|
|
"rewards/margins": 1.8497339487075806,
|
||
|
|
"rewards/rejected": 1.852622389793396,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.44428520243640274,
|
||
|
|
"grad_norm": 0.166015625,
|
||
|
|
"learning_rate": 2.7998607891007493e-07,
|
||
|
|
"logits/chosen": -1.5535634756088257,
|
||
|
|
"logits/rejected": -1.6348403692245483,
|
||
|
|
"logps/chosen": -0.29626718163490295,
|
||
|
|
"logps/rejected": -0.29167088866233826,
|
||
|
|
"loss": 0.6897278428077698,
|
||
|
|
"loss/core": 0.6897278428077698,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 2.309946298599243,
|
||
|
|
"rewards/margins": 1.3803914785385132,
|
||
|
|
"rewards/rejected": 0.9295549392700195,
|
||
|
|
"step": 465
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4490624626776544,
|
||
|
|
"grad_norm": 1.859375,
|
||
|
|
"learning_rate": 2.7516760604219616e-07,
|
||
|
|
"logits/chosen": -1.8788540363311768,
|
||
|
|
"logits/rejected": -1.9592180252075195,
|
||
|
|
"logps/chosen": -0.27540072798728943,
|
||
|
|
"logps/rejected": -0.2596161961555481,
|
||
|
|
"loss": 0.6886305809020996,
|
||
|
|
"loss/core": 0.6886305809020996,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 3.0465798377990723,
|
||
|
|
"rewards/margins": 1.8731689453125,
|
||
|
|
"rewards/rejected": 1.1734110116958618,
|
||
|
|
"step": 470
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.453839722918906,
|
||
|
|
"grad_norm": 0.6640625,
|
||
|
|
"learning_rate": 2.703396686669646e-07,
|
||
|
|
"logits/chosen": -1.8047220706939697,
|
||
|
|
"logits/rejected": -1.782589316368103,
|
||
|
|
"logps/chosen": -0.2949548363685608,
|
||
|
|
"logps/rejected": -0.31417742371559143,
|
||
|
|
"loss": 0.6925175786018372,
|
||
|
|
"loss/core": 0.6925175786018372,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.8359830379486084,
|
||
|
|
"rewards/margins": 0.3028287887573242,
|
||
|
|
"rewards/rejected": 1.5331543684005737,
|
||
|
|
"step": 475
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.45861698316015764,
|
||
|
|
"grad_norm": 0.19140625,
|
||
|
|
"learning_rate": 2.6550408237417884e-07,
|
||
|
|
"logits/chosen": -1.9774296283721924,
|
||
|
|
"logits/rejected": -2.0623116493225098,
|
||
|
|
"logps/chosen": -0.27460455894470215,
|
||
|
|
"logps/rejected": -0.27350351214408875,
|
||
|
|
"loss": 0.6898782849311829,
|
||
|
|
"loss/core": 0.6898782849311829,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.366447687149048,
|
||
|
|
"rewards/margins": 1.3521076440811157,
|
||
|
|
"rewards/rejected": 1.0143399238586426,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4633942434014093,
|
||
|
|
"grad_norm": 0.173828125,
|
||
|
|
"learning_rate": 2.6066266563008265e-07,
|
||
|
|
"logits/chosen": -2.0387158393859863,
|
||
|
|
"logits/rejected": -2.014871120452881,
|
||
|
|
"logps/chosen": -0.2912241816520691,
|
||
|
|
"logps/rejected": -0.2957185208797455,
|
||
|
|
"loss": 0.6910130381584167,
|
||
|
|
"loss/core": 0.6910130381584167,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.7464618682861328,
|
||
|
|
"rewards/margins": 0.8589704632759094,
|
||
|
|
"rewards/rejected": 0.8874912261962891,
|
||
|
|
"step": 485
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4681715036426609,
|
||
|
|
"grad_norm": 0.33203125,
|
||
|
|
"learning_rate": 2.5581723909351404e-07,
|
||
|
|
"logits/chosen": -1.8752286434173584,
|
||
|
|
"logits/rejected": -1.8981329202651978,
|
||
|
|
"logps/chosen": -0.3062916398048401,
|
||
|
|
"logps/rejected": -0.2947053015232086,
|
||
|
|
"loss": 0.6905814409255981,
|
||
|
|
"loss/core": 0.6905814409255981,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.0861191749572754,
|
||
|
|
"rewards/margins": 1.0360615253448486,
|
||
|
|
"rewards/rejected": 1.0500577688217163,
|
||
|
|
"step": 490
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.47294876388391255,
|
||
|
|
"grad_norm": 0.353515625,
|
||
|
|
"learning_rate": 2.509696249312301e-07,
|
||
|
|
"logits/chosen": -1.8822190761566162,
|
||
|
|
"logits/rejected": -1.9906330108642578,
|
||
|
|
"logps/chosen": -0.29457131028175354,
|
||
|
|
"logps/rejected": -0.2900959551334381,
|
||
|
|
"loss": 0.6889396905899048,
|
||
|
|
"loss/core": 0.6889396905899048,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.6438632011413574,
|
||
|
|
"rewards/margins": 1.7446101903915405,
|
||
|
|
"rewards/rejected": 0.8992528915405273,
|
||
|
|
"step": 495
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.47772602412516424,
|
||
|
|
"grad_norm": 0.447265625,
|
||
|
|
"learning_rate": 2.461216461326642e-07,
|
||
|
|
"logits/chosen": -1.7823041677474976,
|
||
|
|
"logits/rejected": -1.7042522430419922,
|
||
|
|
"logps/chosen": -0.276335746049881,
|
||
|
|
"logps/rejected": -0.30160245299339294,
|
||
|
|
"loss": 0.6881250143051147,
|
||
|
|
"loss/core": 0.6881250143051147,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 4.211169242858887,
|
||
|
|
"rewards/margins": 2.174818277359009,
|
||
|
|
"rewards/rejected": 2.036350727081299,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4825032843664159,
|
||
|
|
"grad_norm": 1.28125,
|
||
|
|
"learning_rate": 2.412751258243748e-07,
|
||
|
|
"logits/chosen": -1.5979036092758179,
|
||
|
|
"logits/rejected": -1.695265531539917,
|
||
|
|
"logps/chosen": -0.28008055686950684,
|
||
|
|
"logps/rejected": -0.27932852506637573,
|
||
|
|
"loss": 0.6890323162078857,
|
||
|
|
"loss/core": 0.6890323162078857,
|
||
|
|
"rewards/accuracies": 0.8125,
|
||
|
|
"rewards/chosen": 3.665738344192505,
|
||
|
|
"rewards/margins": 1.8280740976333618,
|
||
|
|
"rewards/rejected": 1.8376646041870117,
|
||
|
|
"step": 505
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4872805446076675,
|
||
|
|
"grad_norm": 0.060302734375,
|
||
|
|
"learning_rate": 2.3643188658444158e-07,
|
||
|
|
"logits/chosen": -1.8131134510040283,
|
||
|
|
"logits/rejected": -1.9090598821640015,
|
||
|
|
"logps/chosen": -0.31393712759017944,
|
||
|
|
"logps/rejected": -0.29055261611938477,
|
||
|
|
"loss": 0.6901880502700806,
|
||
|
|
"loss/core": 0.6901880502700806,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.030540704727173,
|
||
|
|
"rewards/margins": 1.2225555181503296,
|
||
|
|
"rewards/rejected": 0.8079849481582642,
|
||
|
|
"step": 510
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.49205780484891914,
|
||
|
|
"grad_norm": 0.369140625,
|
||
|
|
"learning_rate": 2.315937497570688e-07,
|
||
|
|
"logits/chosen": -1.6081165075302124,
|
||
|
|
"logits/rejected": -1.6191253662109375,
|
||
|
|
"logps/chosen": -0.28171616792678833,
|
||
|
|
"logps/rejected": -0.30241820216178894,
|
||
|
|
"loss": 0.6910374164581299,
|
||
|
|
"loss/core": 0.6910374164581299,
|
||
|
|
"rewards/accuracies": 0.824999988079071,
|
||
|
|
"rewards/chosen": 2.5543110370635986,
|
||
|
|
"rewards/margins": 0.8602564930915833,
|
||
|
|
"rewards/rejected": 1.6940546035766602,
|
||
|
|
"step": 515
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4968350650901708,
|
||
|
|
"grad_norm": 0.50390625,
|
||
|
|
"learning_rate": 2.2676253476765194e-07,
|
||
|
|
"logits/chosen": -1.8473793268203735,
|
||
|
|
"logits/rejected": -1.940637230873108,
|
||
|
|
"logps/chosen": -0.3185504972934723,
|
||
|
|
"logps/rejected": -0.3340589106082916,
|
||
|
|
"loss": 0.6879608035087585,
|
||
|
|
"loss/core": 0.6879608035087585,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 3.108973503112793,
|
||
|
|
"rewards/margins": 2.145254373550415,
|
||
|
|
"rewards/rejected": 0.9637192487716675,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5016123253314224,
|
||
|
|
"grad_norm": 1.5703125,
|
||
|
|
"learning_rate": 2.2194005843856633e-07,
|
||
|
|
"logits/chosen": -1.9106519222259521,
|
||
|
|
"logits/rejected": -1.854853630065918,
|
||
|
|
"logps/chosen": -0.2881598472595215,
|
||
|
|
"logps/rejected": -0.29906392097473145,
|
||
|
|
"loss": 0.6862886548042297,
|
||
|
|
"loss/core": 0.6862886548042297,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 4.429782390594482,
|
||
|
|
"rewards/margins": 2.9549190998077393,
|
||
|
|
"rewards/rejected": 1.4748632907867432,
|
||
|
|
"step": 525
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5063895855726741,
|
||
|
|
"grad_norm": 0.068359375,
|
||
|
|
"learning_rate": 2.1712813430593434e-07,
|
||
|
|
"logits/chosen": -1.698067307472229,
|
||
|
|
"logits/rejected": -1.693956732749939,
|
||
|
|
"logps/chosen": -0.28551727533340454,
|
||
|
|
"logps/rejected": -0.2868136763572693,
|
||
|
|
"loss": 0.6893957853317261,
|
||
|
|
"loss/core": 0.6893957853317261,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.0555648803710938,
|
||
|
|
"rewards/margins": 1.5484126806259155,
|
||
|
|
"rewards/rejected": 1.5071521997451782,
|
||
|
|
"step": 530
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5111668458139257,
|
||
|
|
"grad_norm": 0.62890625,
|
||
|
|
"learning_rate": 2.123285719376292e-07,
|
||
|
|
"logits/chosen": -1.7900841236114502,
|
||
|
|
"logits/rejected": -1.7464206218719482,
|
||
|
|
"logps/chosen": -0.28272828459739685,
|
||
|
|
"logps/rejected": -0.27548089623451233,
|
||
|
|
"loss": 0.6883460283279419,
|
||
|
|
"loss/core": 0.6883460283279419,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 3.122218370437622,
|
||
|
|
"rewards/margins": 2.007307529449463,
|
||
|
|
"rewards/rejected": 1.1149110794067383,
|
||
|
|
"step": 535
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5159441060551774,
|
||
|
|
"grad_norm": 0.10400390625,
|
||
|
|
"learning_rate": 2.0754317625276982e-07,
|
||
|
|
"logits/chosen": -1.7317934036254883,
|
||
|
|
"logits/rejected": -1.6521661281585693,
|
||
|
|
"logps/chosen": -0.27843254804611206,
|
||
|
|
"logps/rejected": -0.2827979624271393,
|
||
|
|
"loss": 0.693123996257782,
|
||
|
|
"loss/core": 0.693123996257782,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.5914747714996338,
|
||
|
|
"rewards/margins": 0.03556249290704727,
|
||
|
|
"rewards/rejected": 1.5559122562408447,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.520721366296429,
|
||
|
|
"grad_norm": 1.578125,
|
||
|
|
"learning_rate": 2.0277374684296498e-07,
|
||
|
|
"logits/chosen": -1.9993709325790405,
|
||
|
|
"logits/rejected": -1.906482458114624,
|
||
|
|
"logps/chosen": -0.2804117798805237,
|
||
|
|
"logps/rejected": -0.29083675146102905,
|
||
|
|
"loss": 0.6889429092407227,
|
||
|
|
"loss/core": 0.6889429092407227,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.6694138050079346,
|
||
|
|
"rewards/margins": 1.7691431045532227,
|
||
|
|
"rewards/rejected": 0.9002708196640015,
|
||
|
|
"step": 545
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5254986265376806,
|
||
|
|
"grad_norm": 0.265625,
|
||
|
|
"learning_rate": 1.980220772955602e-07,
|
||
|
|
"logits/chosen": -1.5809681415557861,
|
||
|
|
"logits/rejected": -1.5856965780258179,
|
||
|
|
"logps/chosen": -0.2750489413738251,
|
||
|
|
"logps/rejected": -0.2972986698150635,
|
||
|
|
"loss": 0.6903039216995239,
|
||
|
|
"loss/core": 0.6903039216995239,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.7791810035705566,
|
||
|
|
"rewards/margins": 1.1509668827056885,
|
||
|
|
"rewards/rejected": 1.628213882446289,
|
||
|
|
"step": 550
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5302758867789323,
|
||
|
|
"grad_norm": 0.1396484375,
|
||
|
|
"learning_rate": 1.932899545191433e-07,
|
||
|
|
"logits/chosen": -1.8478825092315674,
|
||
|
|
"logits/rejected": -1.8661056756973267,
|
||
|
|
"logps/chosen": -0.31153231859207153,
|
||
|
|
"logps/rejected": -0.2762632966041565,
|
||
|
|
"loss": 0.6875492334365845,
|
||
|
|
"loss/core": 0.6875492334365845,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.2964320182800293,
|
||
|
|
"rewards/margins": 2.3140857219696045,
|
||
|
|
"rewards/rejected": 0.9823463559150696,
|
||
|
|
"step": 555
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5350531470201839,
|
||
|
|
"grad_norm": 0.408203125,
|
||
|
|
"learning_rate": 1.885791580715609e-07,
|
||
|
|
"logits/chosen": -1.7739731073379517,
|
||
|
|
"logits/rejected": -1.8065807819366455,
|
||
|
|
"logps/chosen": -0.2748362421989441,
|
||
|
|
"logps/rejected": -0.2793380916118622,
|
||
|
|
"loss": 0.6899937987327576,
|
||
|
|
"loss/core": 0.6899937987327576,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.8241641521453857,
|
||
|
|
"rewards/margins": 1.279166340827942,
|
||
|
|
"rewards/rejected": 1.544998049736023,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5398304072614356,
|
||
|
|
"grad_norm": 0.376953125,
|
||
|
|
"learning_rate": 1.8389145949069951e-07,
|
||
|
|
"logits/chosen": -1.8410762548446655,
|
||
|
|
"logits/rejected": -1.8144041299819946,
|
||
|
|
"logps/chosen": -0.2863827347755432,
|
||
|
|
"logps/rejected": -0.30311352014541626,
|
||
|
|
"loss": 0.6904335618019104,
|
||
|
|
"loss/core": 0.6904335618019104,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.7921481132507324,
|
||
|
|
"rewards/margins": 1.1107183694839478,
|
||
|
|
"rewards/rejected": 1.6814296245574951,
|
||
|
|
"step": 565
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5446076675026872,
|
||
|
|
"grad_norm": 1.046875,
|
||
|
|
"learning_rate": 1.792286216282824e-07,
|
||
|
|
"logits/chosen": -1.79177725315094,
|
||
|
|
"logits/rejected": -1.8119325637817383,
|
||
|
|
"logps/chosen": -0.2888721525669098,
|
||
|
|
"logps/rejected": -0.2847359776496887,
|
||
|
|
"loss": 0.689185619354248,
|
||
|
|
"loss/core": 0.689185619354248,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.3419628143310547,
|
||
|
|
"rewards/margins": 1.616077184677124,
|
||
|
|
"rewards/rejected": 1.7258857488632202,
|
||
|
|
"step": 570
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5493849277439389,
|
||
|
|
"grad_norm": 0.111328125,
|
||
|
|
"learning_rate": 1.745923979869336e-07,
|
||
|
|
"logits/chosen": -1.88888418674469,
|
||
|
|
"logits/rejected": -1.9161269664764404,
|
||
|
|
"logps/chosen": -0.29976314306259155,
|
||
|
|
"logps/rejected": -0.3220902383327484,
|
||
|
|
"loss": 0.6900423169136047,
|
||
|
|
"loss/core": 0.6900423169136047,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.8517236709594727,
|
||
|
|
"rewards/margins": 1.2799955606460571,
|
||
|
|
"rewards/rejected": 0.5717281103134155,
|
||
|
|
"step": 575
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5541621879851905,
|
||
|
|
"grad_norm": 0.07568359375,
|
||
|
|
"learning_rate": 1.6998453206075708e-07,
|
||
|
|
"logits/chosen": -1.9753665924072266,
|
||
|
|
"logits/rejected": -2.079908847808838,
|
||
|
|
"logps/chosen": -0.26681944727897644,
|
||
|
|
"logps/rejected": -0.2805357575416565,
|
||
|
|
"loss": 0.6920329332351685,
|
||
|
|
"loss/core": 0.6920329332351685,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.4170547723770142,
|
||
|
|
"rewards/margins": 0.45395904779434204,
|
||
|
|
"rewards/rejected": 0.9630956649780273,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5589394482264421,
|
||
|
|
"grad_norm": 0.2265625,
|
||
|
|
"learning_rate": 1.6540675667967973e-07,
|
||
|
|
"logits/chosen": -1.9732328653335571,
|
||
|
|
"logits/rejected": -1.867099404335022,
|
||
|
|
"logps/chosen": -0.2693113684654236,
|
||
|
|
"logps/rejected": -0.28243768215179443,
|
||
|
|
"loss": 0.6910297274589539,
|
||
|
|
"loss/core": 0.6910297274589539,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.231765031814575,
|
||
|
|
"rewards/margins": 0.8599586486816406,
|
||
|
|
"rewards/rejected": 1.3718063831329346,
|
||
|
|
"step": 585
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5637167084676937,
|
||
|
|
"grad_norm": 0.1318359375,
|
||
|
|
"learning_rate": 1.60860793357805e-07,
|
||
|
|
"logits/chosen": -1.827453851699829,
|
||
|
|
"logits/rejected": -1.8766067028045654,
|
||
|
|
"logps/chosen": -0.2868167757987976,
|
||
|
|
"logps/rejected": -0.2861761450767517,
|
||
|
|
"loss": 0.6869019269943237,
|
||
|
|
"loss/core": 0.6869019269943237,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 3.6281845569610596,
|
||
|
|
"rewards/margins": 2.5879273414611816,
|
||
|
|
"rewards/rejected": 1.0402569770812988,
|
||
|
|
"step": 590
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5684939687089454,
|
||
|
|
"grad_norm": 0.1767578125,
|
||
|
|
"learning_rate": 1.5634835164602196e-07,
|
||
|
|
"logits/chosen": -1.771225929260254,
|
||
|
|
"logits/rejected": -1.781171202659607,
|
||
|
|
"logps/chosen": -0.27533966302871704,
|
||
|
|
"logps/rejected": -0.2719757854938507,
|
||
|
|
"loss": 0.6911031603813171,
|
||
|
|
"loss/core": 0.6911031603813171,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.7094968557357788,
|
||
|
|
"rewards/margins": 0.8211535215377808,
|
||
|
|
"rewards/rejected": 0.8883434534072876,
|
||
|
|
"step": 595
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5732712289501971,
|
||
|
|
"grad_norm": 0.314453125,
|
||
|
|
"learning_rate": 1.518711284891132e-07,
|
||
|
|
"logits/chosen": -1.8096263408660889,
|
||
|
|
"logits/rejected": -1.7713695764541626,
|
||
|
|
"logps/chosen": -0.26596006751060486,
|
||
|
|
"logps/rejected": -0.28354257345199585,
|
||
|
|
"loss": 0.691169023513794,
|
||
|
|
"loss/core": 0.691169023513794,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.876749038696289,
|
||
|
|
"rewards/margins": 0.8061097860336304,
|
||
|
|
"rewards/rejected": 1.0706393718719482,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5780484891914487,
|
||
|
|
"grad_norm": 0.6953125,
|
||
|
|
"learning_rate": 1.47430807587603e-07,
|
||
|
|
"logits/chosen": -1.9364635944366455,
|
||
|
|
"logits/rejected": -2.0032143592834473,
|
||
|
|
"logps/chosen": -0.2760660648345947,
|
||
|
|
"logps/rejected": -0.2748982906341553,
|
||
|
|
"loss": 0.6860328316688538,
|
||
|
|
"loss/core": 0.6860328316688538,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 4.224491596221924,
|
||
|
|
"rewards/margins": 3.010608196258545,
|
||
|
|
"rewards/rejected": 1.2138831615447998,
|
||
|
|
"step": 605
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5828257494327004,
|
||
|
|
"grad_norm": 0.5078125,
|
||
|
|
"learning_rate": 1.430290587645865e-07,
|
||
|
|
"logits/chosen": -1.6363016366958618,
|
||
|
|
"logits/rejected": -1.6160857677459717,
|
||
|
|
"logps/chosen": -0.27192258834838867,
|
||
|
|
"logps/rejected": -0.2753311097621918,
|
||
|
|
"loss": 0.6874152421951294,
|
||
|
|
"loss/core": 0.6874152421951294,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 4.3153300285339355,
|
||
|
|
"rewards/margins": 2.379615068435669,
|
||
|
|
"rewards/rejected": 1.9357149600982666,
|
||
|
|
"step": 610
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.587603009673952,
|
||
|
|
"grad_norm": 0.19921875,
|
||
|
|
"learning_rate": 1.3866753733777765e-07,
|
||
|
|
"logits/chosen": -1.4840619564056396,
|
||
|
|
"logits/rejected": -1.5672607421875,
|
||
|
|
"logps/chosen": -0.3140578866004944,
|
||
|
|
"logps/rejected": -0.30204764008522034,
|
||
|
|
"loss": 0.6901291012763977,
|
||
|
|
"loss/core": 0.6901291012763977,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.7394516468048096,
|
||
|
|
"rewards/margins": 1.2340558767318726,
|
||
|
|
"rewards/rejected": 1.505395770072937,
|
||
|
|
"step": 615
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5923802699152036,
|
||
|
|
"grad_norm": 0.1572265625,
|
||
|
|
"learning_rate": 1.343478834970121e-07,
|
||
|
|
"logits/chosen": -1.865546464920044,
|
||
|
|
"logits/rejected": -1.8963212966918945,
|
||
|
|
"logps/chosen": -0.28501859307289124,
|
||
|
|
"logps/rejected": -0.2918963134288788,
|
||
|
|
"loss": 0.692618191242218,
|
||
|
|
"loss/core": 0.692618191242218,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.7005846500396729,
|
||
|
|
"rewards/margins": 0.2428729087114334,
|
||
|
|
"rewards/rejected": 1.4577115774154663,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5971575301564552,
|
||
|
|
"grad_norm": 3.3125,
|
||
|
|
"learning_rate": 1.3007172168743852e-07,
|
||
|
|
"logits/chosen": -1.7032978534698486,
|
||
|
|
"logits/rejected": -1.6907075643539429,
|
||
|
|
"logps/chosen": -0.30372241139411926,
|
||
|
|
"logps/rejected": -0.33234214782714844,
|
||
|
|
"loss": 0.6876958012580872,
|
||
|
|
"loss/core": 0.6876958012580872,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 3.8974437713623047,
|
||
|
|
"rewards/margins": 2.3479323387145996,
|
||
|
|
"rewards/rejected": 1.5495115518569946,
|
||
|
|
"step": 625
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6019347903977069,
|
||
|
|
"grad_norm": 0.2255859375,
|
||
|
|
"learning_rate": 1.25840659998631e-07,
|
||
|
|
"logits/chosen": -1.7279109954833984,
|
||
|
|
"logits/rejected": -1.8030815124511719,
|
||
|
|
"logps/chosen": -0.2717967629432678,
|
||
|
|
"logps/rejected": -0.2828608751296997,
|
||
|
|
"loss": 0.6890043020248413,
|
||
|
|
"loss/core": 0.6890043020248413,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.9497199058532715,
|
||
|
|
"rewards/margins": 1.700548529624939,
|
||
|
|
"rewards/rejected": 1.249171257019043,
|
||
|
|
"step": 630
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6067120506389586,
|
||
|
|
"grad_norm": 1.0,
|
||
|
|
"learning_rate": 1.2165628955985313e-07,
|
||
|
|
"logits/chosen": -1.7578712701797485,
|
||
|
|
"logits/rejected": -1.8161617517471313,
|
||
|
|
"logps/chosen": -0.303678959608078,
|
||
|
|
"logps/rejected": -0.2981964647769928,
|
||
|
|
"loss": 0.6900124549865723,
|
||
|
|
"loss/core": 0.6900124549865723,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.3181376457214355,
|
||
|
|
"rewards/margins": 1.2847710847854614,
|
||
|
|
"rewards/rejected": 1.0333666801452637,
|
||
|
|
"step": 635
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6114893108802102,
|
||
|
|
"grad_norm": 0.125,
|
||
|
|
"learning_rate": 1.175201839416988e-07,
|
||
|
|
"logits/chosen": -1.9067970514297485,
|
||
|
|
"logits/rejected": -1.9922056198120117,
|
||
|
|
"logps/chosen": -0.29978805780410767,
|
||
|
|
"logps/rejected": -0.2979918420314789,
|
||
|
|
"loss": 0.6902927160263062,
|
||
|
|
"loss/core": 0.6902927160263062,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 1.974582314491272,
|
||
|
|
"rewards/margins": 1.1694762706756592,
|
||
|
|
"rewards/rejected": 0.8051063418388367,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6162665711214619,
|
||
|
|
"grad_norm": 0.28515625,
|
||
|
|
"learning_rate": 1.1343389856433658e-07,
|
||
|
|
"logits/chosen": -1.998485803604126,
|
||
|
|
"logits/rejected": -2.0181636810302734,
|
||
|
|
"logps/chosen": -0.28951841592788696,
|
||
|
|
"logps/rejected": -0.2937147319316864,
|
||
|
|
"loss": 0.6920903921127319,
|
||
|
|
"loss/core": 0.6920903921127319,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.685060739517212,
|
||
|
|
"rewards/margins": 0.43871307373046875,
|
||
|
|
"rewards/rejected": 1.2463476657867432,
|
||
|
|
"step": 645
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6210438313627135,
|
||
|
|
"grad_norm": 0.1767578125,
|
||
|
|
"learning_rate": 1.0939897011258e-07,
|
||
|
|
"logits/chosen": -1.9956214427947998,
|
||
|
|
"logits/rejected": -1.9716342687606812,
|
||
|
|
"logps/chosen": -0.2891247570514679,
|
||
|
|
"logps/rejected": -0.2986387312412262,
|
||
|
|
"loss": 0.691946804523468,
|
||
|
|
"loss/core": 0.691946804523468,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.3371341228485107,
|
||
|
|
"rewards/margins": 0.49254298210144043,
|
||
|
|
"rewards/rejected": 0.8445911407470703,
|
||
|
|
"step": 650
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6258210916039652,
|
||
|
|
"grad_norm": 0.8359375,
|
||
|
|
"learning_rate": 1.0541691595800336e-07,
|
||
|
|
"logits/chosen": -1.7673661708831787,
|
||
|
|
"logits/rejected": -1.8068790435791016,
|
||
|
|
"logps/chosen": -0.282105028629303,
|
||
|
|
"logps/rejected": -0.2889975309371948,
|
||
|
|
"loss": 0.690642774105072,
|
||
|
|
"loss/core": 0.690642774105072,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.0060832500457764,
|
||
|
|
"rewards/margins": 1.010461688041687,
|
||
|
|
"rewards/rejected": 0.9956215023994446,
|
||
|
|
"step": 655
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6305983518452167,
|
||
|
|
"grad_norm": 0.11474609375,
|
||
|
|
"learning_rate": 1.0148923358832021e-07,
|
||
|
|
"logits/chosen": -1.9046757221221924,
|
||
|
|
"logits/rejected": -1.9204623699188232,
|
||
|
|
"logps/chosen": -0.29088854789733887,
|
||
|
|
"logps/rejected": -0.30732461810112,
|
||
|
|
"loss": 0.6896063685417175,
|
||
|
|
"loss/core": 0.6896063685417175,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.8072502613067627,
|
||
|
|
"rewards/margins": 1.5126233100891113,
|
||
|
|
"rewards/rejected": 1.2946269512176514,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6353756120864684,
|
||
|
|
"grad_norm": 0.5390625,
|
||
|
|
"learning_rate": 9.761740004423926e-08,
|
||
|
|
"logits/chosen": -1.9125967025756836,
|
||
|
|
"logits/rejected": -1.8837387561798096,
|
||
|
|
"logps/chosen": -0.2830875813961029,
|
||
|
|
"logps/rejected": -0.27817443013191223,
|
||
|
|
"loss": 0.6911165118217468,
|
||
|
|
"loss/core": 0.6911165118217468,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.3600566387176514,
|
||
|
|
"rewards/margins": 0.8475581407546997,
|
||
|
|
"rewards/rejected": 1.5124986171722412,
|
||
|
|
"step": 665
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.64015287232772,
|
||
|
|
"grad_norm": 0.11083984375,
|
||
|
|
"learning_rate": 9.380287136400999e-08,
|
||
|
|
"logits/chosen": -1.7506167888641357,
|
||
|
|
"logits/rejected": -1.8097076416015625,
|
||
|
|
"logps/chosen": -0.2727493345737457,
|
||
|
|
"logps/rejected": -0.3100685477256775,
|
||
|
|
"loss": 0.6875694394111633,
|
||
|
|
"loss/core": 0.6875694394111633,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 3.660663604736328,
|
||
|
|
"rewards/margins": 2.3024368286132812,
|
||
|
|
"rewards/rejected": 1.3582265377044678,
|
||
|
|
"step": 670
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6449301325689717,
|
||
|
|
"grad_norm": 0.19921875,
|
||
|
|
"learning_rate": 9.004708203586628e-08,
|
||
|
|
"logits/chosen": -1.661525011062622,
|
||
|
|
"logits/rejected": -1.6602637767791748,
|
||
|
|
"logps/chosen": -0.2878963053226471,
|
||
|
|
"logps/rejected": -0.28721392154693604,
|
||
|
|
"loss": 0.6913235187530518,
|
||
|
|
"loss/core": 0.6913235187530518,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.6275174617767334,
|
||
|
|
"rewards/margins": 0.7366446256637573,
|
||
|
|
"rewards/rejected": 0.8908728361129761,
|
||
|
|
"step": 675
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6497073928102234,
|
||
|
|
"grad_norm": 0.10791015625,
|
||
|
|
"learning_rate": 8.635144445857407e-08,
|
||
|
|
"logits/chosen": -1.8359218835830688,
|
||
|
|
"logits/rejected": -1.7717132568359375,
|
||
|
|
"logps/chosen": -0.2822825312614441,
|
||
|
|
"logps/rejected": -0.2898341715335846,
|
||
|
|
"loss": 0.6913445591926575,
|
||
|
|
"loss/core": 0.6913445591926575,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.1436684131622314,
|
||
|
|
"rewards/margins": 0.7406994700431824,
|
||
|
|
"rewards/rejected": 1.4029688835144043,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.654484653051475,
|
||
|
|
"grad_norm": 0.1455078125,
|
||
|
|
"learning_rate": 8.271734841028552e-08,
|
||
|
|
"logits/chosen": -1.6390689611434937,
|
||
|
|
"logits/rejected": -1.651624321937561,
|
||
|
|
"logps/chosen": -0.2526906132698059,
|
||
|
|
"logps/rejected": -0.27529576420783997,
|
||
|
|
"loss": 0.6899958848953247,
|
||
|
|
"loss/core": 0.6899958848953247,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.895305633544922,
|
||
|
|
"rewards/margins": 1.3431675434112549,
|
||
|
|
"rewards/rejected": 1.552138090133667,
|
||
|
|
"step": 685
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6592619132927267,
|
||
|
|
"grad_norm": 0.1337890625,
|
||
|
|
"learning_rate": 7.91461605259007e-08,
|
||
|
|
"logits/chosen": -1.5240414142608643,
|
||
|
|
"logits/rejected": -1.5546308755874634,
|
||
|
|
"logps/chosen": -0.3081050515174866,
|
||
|
|
"logps/rejected": -0.32126057147979736,
|
||
|
|
"loss": 0.6916640996932983,
|
||
|
|
"loss/core": 0.6916640996932983,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.7689863443374634,
|
||
|
|
"rewards/margins": 0.6094117164611816,
|
||
|
|
"rewards/rejected": 1.1595746278762817,
|
||
|
|
"step": 690
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6640391735339782,
|
||
|
|
"grad_norm": 0.1640625,
|
||
|
|
"learning_rate": 7.563922378313217e-08,
|
||
|
|
"logits/chosen": -1.8862450122833252,
|
||
|
|
"logits/rejected": -1.9309171438217163,
|
||
|
|
"logps/chosen": -0.30440554022789,
|
||
|
|
"logps/rejected": -0.2918163239955902,
|
||
|
|
"loss": 0.6900017857551575,
|
||
|
|
"loss/core": 0.6900017857551575,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.9893100261688232,
|
||
|
|
"rewards/margins": 1.2960065603256226,
|
||
|
|
"rewards/rejected": 0.6933035850524902,
|
||
|
|
"step": 695
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6688164337752299,
|
||
|
|
"grad_norm": 2.703125,
|
||
|
|
"learning_rate": 7.219785699746572e-08,
|
||
|
|
"logits/chosen": -1.7974674701690674,
|
||
|
|
"logits/rejected": -1.8728835582733154,
|
||
|
|
"logps/chosen": -0.2968365252017975,
|
||
|
|
"logps/rejected": -0.3178980350494385,
|
||
|
|
"loss": 0.6916329264640808,
|
||
|
|
"loss/core": 0.6916329264640808,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.230217933654785,
|
||
|
|
"rewards/margins": 0.6277396082878113,
|
||
|
|
"rewards/rejected": 1.60247802734375,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6735936940164815,
|
||
|
|
"grad_norm": 0.52734375,
|
||
|
|
"learning_rate": 6.882335432620779e-08,
|
||
|
|
"logits/chosen": -1.846928596496582,
|
||
|
|
"logits/rejected": -1.8282711505889893,
|
||
|
|
"logps/chosen": -0.28012657165527344,
|
||
|
|
"logps/rejected": -0.26722002029418945,
|
||
|
|
"loss": 0.6885994672775269,
|
||
|
|
"loss/core": 0.6885994672775269,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 3.4068984985351562,
|
||
|
|
"rewards/margins": 1.8730993270874023,
|
||
|
|
"rewards/rejected": 1.533799171447754,
|
||
|
|
"step": 705
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6783709542577332,
|
||
|
|
"grad_norm": 1.6953125,
|
||
|
|
"learning_rate": 6.551698478180589e-08,
|
||
|
|
"logits/chosen": -1.6279361248016357,
|
||
|
|
"logits/rejected": -1.647020936012268,
|
||
|
|
"logps/chosen": -0.2771731913089752,
|
||
|
|
"logps/rejected": -0.29674896597862244,
|
||
|
|
"loss": 0.6890315413475037,
|
||
|
|
"loss/core": 0.6890315413475037,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 3.5971343517303467,
|
||
|
|
"rewards/margins": 1.6893699169158936,
|
||
|
|
"rewards/rejected": 1.9077644348144531,
|
||
|
|
"step": 710
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6831482144989849,
|
||
|
|
"grad_norm": 0.1572265625,
|
||
|
|
"learning_rate": 6.22799917546252e-08,
|
||
|
|
"logits/chosen": -1.8586616516113281,
|
||
|
|
"logits/rejected": -1.8125330209732056,
|
||
|
|
"logps/chosen": -0.2845900058746338,
|
||
|
|
"logps/rejected": -0.32791072130203247,
|
||
|
|
"loss": 0.6905220746994019,
|
||
|
|
"loss/core": 0.6905220746994019,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.8608309030532837,
|
||
|
|
"rewards/margins": 1.073857307434082,
|
||
|
|
"rewards/rejected": 0.7869734168052673,
|
||
|
|
"step": 715
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6879254747402365,
|
||
|
|
"grad_norm": 0.423828125,
|
||
|
|
"learning_rate": 5.9113592545359944e-08,
|
||
|
|
"logits/chosen": -1.7632392644882202,
|
||
|
|
"logits/rejected": -1.7893062829971313,
|
||
|
|
"logps/chosen": -0.2961721122264862,
|
||
|
|
"logps/rejected": -0.300367534160614,
|
||
|
|
"loss": 0.6917133331298828,
|
||
|
|
"loss/core": 0.6917133331298828,
|
||
|
|
"rewards/accuracies": 0.800000011920929,
|
||
|
|
"rewards/chosen": 2.103755474090576,
|
||
|
|
"rewards/margins": 0.6050199866294861,
|
||
|
|
"rewards/rejected": 1.4987354278564453,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6927027349814882,
|
||
|
|
"grad_norm": 0.1708984375,
|
||
|
|
"learning_rate": 5.601897790725643e-08,
|
||
|
|
"logits/chosen": -1.9446632862091064,
|
||
|
|
"logits/rejected": -1.969635248184204,
|
||
|
|
"logps/chosen": -0.2848110795021057,
|
||
|
|
"logps/rejected": -0.27734309434890747,
|
||
|
|
"loss": 0.6910138130187988,
|
||
|
|
"loss/core": 0.6910138130187988,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.5936205387115479,
|
||
|
|
"rewards/margins": 0.8628653287887573,
|
||
|
|
"rewards/rejected": 0.7307552099227905,
|
||
|
|
"step": 725
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6974799952227397,
|
||
|
|
"grad_norm": 1.4375,
|
||
|
|
"learning_rate": 5.299731159831952e-08,
|
||
|
|
"logits/chosen": -1.901888132095337,
|
||
|
|
"logits/rejected": -1.9483245611190796,
|
||
|
|
"logps/chosen": -0.2863468527793884,
|
||
|
|
"logps/rejected": -0.2913190722465515,
|
||
|
|
"loss": 0.6928960680961609,
|
||
|
|
"loss/core": 0.6928960680961609,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.0203444957733154,
|
||
|
|
"rewards/margins": 0.1184670701622963,
|
||
|
|
"rewards/rejected": 0.9018775224685669,
|
||
|
|
"step": 730
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7022572554639914,
|
||
|
|
"grad_norm": 0.408203125,
|
||
|
|
"learning_rate": 5.0049729943671013e-08,
|
||
|
|
"logits/chosen": -1.8452632427215576,
|
||
|
|
"logits/rejected": -1.8946354389190674,
|
||
|
|
"logps/chosen": -0.319896399974823,
|
||
|
|
"logps/rejected": -0.3175990581512451,
|
||
|
|
"loss": 0.6912761926651001,
|
||
|
|
"loss/core": 0.6912761926651001,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.4188251495361328,
|
||
|
|
"rewards/margins": 0.7536794543266296,
|
||
|
|
"rewards/rejected": 0.6651455760002136,
|
||
|
|
"step": 735
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.707034515705243,
|
||
|
|
"grad_norm": 0.1826171875,
|
||
|
|
"learning_rate": 4.7177341408223994e-08,
|
||
|
|
"logits/chosen": -1.7158161401748657,
|
||
|
|
"logits/rejected": -1.7437493801116943,
|
||
|
|
"logps/chosen": -0.31194645166397095,
|
||
|
|
"logps/rejected": -0.3125692307949066,
|
||
|
|
"loss": 0.6917774081230164,
|
||
|
|
"loss/core": 0.6917774081230164,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.575186252593994,
|
||
|
|
"rewards/margins": 0.7101010084152222,
|
||
|
|
"rewards/rejected": 1.865085244178772,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7118117759464947,
|
||
|
|
"grad_norm": 1.0546875,
|
||
|
|
"learning_rate": 4.438122617983442e-08,
|
||
|
|
"logits/chosen": -1.7589060068130493,
|
||
|
|
"logits/rejected": -1.833776831626892,
|
||
|
|
"logps/chosen": -0.28943413496017456,
|
||
|
|
"logps/rejected": -0.28058308362960815,
|
||
|
|
"loss": 0.6892428994178772,
|
||
|
|
"loss/core": 0.6892428994178772,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.131422758102417,
|
||
|
|
"rewards/margins": 1.6019914150238037,
|
||
|
|
"rewards/rejected": 1.5294312238693237,
|
||
|
|
"step": 745
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7165890361877463,
|
||
|
|
"grad_norm": 0.451171875,
|
||
|
|
"learning_rate": 4.1662435763087114e-08,
|
||
|
|
"logits/chosen": -1.912740707397461,
|
||
|
|
"logits/rejected": -1.9157087802886963,
|
||
|
|
"logps/chosen": -0.31203845143318176,
|
||
|
|
"logps/rejected": -0.3028412461280823,
|
||
|
|
"loss": 0.6934337615966797,
|
||
|
|
"loss/core": 0.6934337615966797,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.2026363611221313,
|
||
|
|
"rewards/margins": -0.061604440212249756,
|
||
|
|
"rewards/rejected": 1.2642407417297363,
|
||
|
|
"step": 750
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.721366296428998,
|
||
|
|
"grad_norm": 0.373046875,
|
||
|
|
"learning_rate": 3.902199258386732e-08,
|
||
|
|
"logits/chosen": -1.7527711391448975,
|
||
|
|
"logits/rejected": -1.647605299949646,
|
||
|
|
"logps/chosen": -0.3076718747615814,
|
||
|
|
"logps/rejected": -0.2929893732070923,
|
||
|
|
"loss": 0.6915236115455627,
|
||
|
|
"loss/core": 0.6915236115455627,
|
||
|
|
"rewards/accuracies": 0.824999988079071,
|
||
|
|
"rewards/chosen": 2.39526629447937,
|
||
|
|
"rewards/margins": 0.6723926663398743,
|
||
|
|
"rewards/rejected": 1.7228736877441406,
|
||
|
|
"step": 755
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7261435566702497,
|
||
|
|
"grad_norm": 0.201171875,
|
||
|
|
"learning_rate": 3.646088960486862e-08,
|
||
|
|
"logits/chosen": -1.7982404232025146,
|
||
|
|
"logits/rejected": -1.840656042098999,
|
||
|
|
"logps/chosen": -0.3029170632362366,
|
||
|
|
"logps/rejected": -0.29293784499168396,
|
||
|
|
"loss": 0.6903898119926453,
|
||
|
|
"loss/core": 0.6903898119926453,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.9886500835418701,
|
||
|
|
"rewards/margins": 1.1183176040649414,
|
||
|
|
"rewards/rejected": 0.8703324198722839,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7309208169115012,
|
||
|
|
"grad_norm": 0.328125,
|
||
|
|
"learning_rate": 3.398008995217988e-08,
|
||
|
|
"logits/chosen": -1.7162113189697266,
|
||
|
|
"logits/rejected": -1.7504310607910156,
|
||
|
|
"logps/chosen": -0.2765401601791382,
|
||
|
|
"logps/rejected": -0.2717045247554779,
|
||
|
|
"loss": 0.691356360912323,
|
||
|
|
"loss/core": 0.691356360912323,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.7894961833953857,
|
||
|
|
"rewards/margins": 0.7241676449775696,
|
||
|
|
"rewards/rejected": 1.0653284788131714,
|
||
|
|
"step": 765
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7356980771527529,
|
||
|
|
"grad_norm": 0.53125,
|
||
|
|
"learning_rate": 3.1580526553093315e-08,
|
||
|
|
"logits/chosen": -1.6956571340560913,
|
||
|
|
"logits/rejected": -1.7221298217773438,
|
||
|
|
"logps/chosen": -0.3087131679058075,
|
||
|
|
"logps/rejected": -0.28246551752090454,
|
||
|
|
"loss": 0.6899880170822144,
|
||
|
|
"loss/core": 0.6899880170822144,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 3.7439963817596436,
|
||
|
|
"rewards/margins": 1.3693008422851562,
|
||
|
|
"rewards/rejected": 2.3746957778930664,
|
||
|
|
"step": 770
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7404753373940045,
|
||
|
|
"grad_norm": 0.07861328125,
|
||
|
|
"learning_rate": 2.9263101785268252e-08,
|
||
|
|
"logits/chosen": -1.8691062927246094,
|
||
|
|
"logits/rejected": -1.896593451499939,
|
||
|
|
"logps/chosen": -0.306645005941391,
|
||
|
|
"logps/rejected": -0.3000980317592621,
|
||
|
|
"loss": 0.6917179822921753,
|
||
|
|
"loss/core": 0.6917179822921753,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.5224562883377075,
|
||
|
|
"rewards/margins": 0.5784978866577148,
|
||
|
|
"rewards/rejected": 0.9439584016799927,
|
||
|
|
"step": 775
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7452525976352562,
|
||
|
|
"grad_norm": 0.287109375,
|
||
|
|
"learning_rate": 2.7028687137384264e-08,
|
||
|
|
"logits/chosen": -1.7383358478546143,
|
||
|
|
"logits/rejected": -1.7722053527832031,
|
||
|
|
"logps/chosen": -0.301580548286438,
|
||
|
|
"logps/rejected": -0.3000149130821228,
|
||
|
|
"loss": 0.6916896104812622,
|
||
|
|
"loss/core": 0.6916896104812622,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.896596908569336,
|
||
|
|
"rewards/margins": 0.5912862420082092,
|
||
|
|
"rewards/rejected": 1.305310606956482,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7500298578765078,
|
||
|
|
"grad_norm": 0.107421875,
|
||
|
|
"learning_rate": 2.4878122881409447e-08,
|
||
|
|
"logits/chosen": -1.5884864330291748,
|
||
|
|
"logits/rejected": -1.562572717666626,
|
||
|
|
"logps/chosen": -0.30816543102264404,
|
||
|
|
"logps/rejected": -0.2903262674808502,
|
||
|
|
"loss": 0.6882777810096741,
|
||
|
|
"loss/core": 0.6882777810096741,
|
||
|
|
"rewards/accuracies": 0.8125,
|
||
|
|
"rewards/chosen": 3.4592926502227783,
|
||
|
|
"rewards/margins": 2.0797040462493896,
|
||
|
|
"rewards/rejected": 1.3795884847640991,
|
||
|
|
"step": 785
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7548071181177595,
|
||
|
|
"grad_norm": 3.609375,
|
||
|
|
"learning_rate": 2.2812217756608937e-08,
|
||
|
|
"logits/chosen": -1.8189325332641602,
|
||
|
|
"logits/rejected": -1.710569143295288,
|
||
|
|
"logps/chosen": -0.28520792722702026,
|
||
|
|
"logps/rejected": -0.317575603723526,
|
||
|
|
"loss": 0.69153892993927,
|
||
|
|
"loss/core": 0.69153892993927,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 2.6281354427337646,
|
||
|
|
"rewards/margins": 0.6913524866104126,
|
||
|
|
"rewards/rejected": 1.9367828369140625,
|
||
|
|
"step": 790
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7595843783590112,
|
||
|
|
"grad_norm": 0.400390625,
|
||
|
|
"learning_rate": 2.0831748665410763e-08,
|
||
|
|
"logits/chosen": -1.7783273458480835,
|
||
|
|
"logits/rejected": -1.7968485355377197,
|
||
|
|
"logps/chosen": -0.30635857582092285,
|
||
|
|
"logps/rejected": -0.29732808470726013,
|
||
|
|
"loss": 0.6901718378067017,
|
||
|
|
"loss/core": 0.6901718378067017,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.3910913467407227,
|
||
|
|
"rewards/margins": 1.2171629667282104,
|
||
|
|
"rewards/rejected": 1.173928141593933,
|
||
|
|
"step": 795
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7643616386002627,
|
||
|
|
"grad_norm": 0.154296875,
|
||
|
|
"learning_rate": 1.8937460381244967e-08,
|
||
|
|
"logits/chosen": -1.8456178903579712,
|
||
|
|
"logits/rejected": -1.8204724788665771,
|
||
|
|
"logps/chosen": -0.2890564501285553,
|
||
|
|
"logps/rejected": -0.28678005933761597,
|
||
|
|
"loss": 0.6904189586639404,
|
||
|
|
"loss/core": 0.6904189586639404,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.2987735271453857,
|
||
|
|
"rewards/margins": 1.1124370098114014,
|
||
|
|
"rewards/rejected": 1.186336636543274,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7691388988415144,
|
||
|
|
"grad_norm": 0.25,
|
||
|
|
"learning_rate": 1.713006526846439e-08,
|
||
|
|
"logits/chosen": -1.7271690368652344,
|
||
|
|
"logits/rejected": -1.7207380533218384,
|
||
|
|
"logps/chosen": -0.2747926414012909,
|
||
|
|
"logps/rejected": -0.26928240060806274,
|
||
|
|
"loss": 0.69256192445755,
|
||
|
|
"loss/core": 0.69256192445755,
|
||
|
|
"rewards/accuracies": 0.800000011920929,
|
||
|
|
"rewards/chosen": 2.2577762603759766,
|
||
|
|
"rewards/margins": 0.2586982250213623,
|
||
|
|
"rewards/rejected": 1.9990781545639038,
|
||
|
|
"step": 805
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.773916159082766,
|
||
|
|
"grad_norm": 0.15234375,
|
||
|
|
"learning_rate": 1.541024301445404e-08,
|
||
|
|
"logits/chosen": -1.8081337213516235,
|
||
|
|
"logits/rejected": -1.7127681970596313,
|
||
|
|
"logps/chosen": -0.3074837327003479,
|
||
|
|
"logps/rejected": -0.316093385219574,
|
||
|
|
"loss": 0.6930532455444336,
|
||
|
|
"loss/core": 0.6930532455444336,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.5384314060211182,
|
||
|
|
"rewards/margins": 0.054566673934459686,
|
||
|
|
"rewards/rejected": 1.4838647842407227,
|
||
|
|
"step": 810
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7786934193240177,
|
||
|
|
"grad_norm": 0.353515625,
|
||
|
|
"learning_rate": 1.3778640374027983e-08,
|
||
|
|
"logits/chosen": -1.8720842599868774,
|
||
|
|
"logits/rejected": -1.9219611883163452,
|
||
|
|
"logps/chosen": -0.28327468037605286,
|
||
|
|
"logps/rejected": -0.28367432951927185,
|
||
|
|
"loss": 0.6905892491340637,
|
||
|
|
"loss/core": 0.6905892491340637,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.1285481452941895,
|
||
|
|
"rewards/margins": 1.0284489393234253,
|
||
|
|
"rewards/rejected": 1.1000992059707642,
|
||
|
|
"step": 815
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7834706795652693,
|
||
|
|
"grad_norm": 0.115234375,
|
||
|
|
"learning_rate": 1.2235870926211616e-08,
|
||
|
|
"logits/chosen": -1.7334009408950806,
|
||
|
|
"logits/rejected": -1.737236738204956,
|
||
|
|
"logps/chosen": -0.28783631324768066,
|
||
|
|
"logps/rejected": -0.3172575533390045,
|
||
|
|
"loss": 0.6899994611740112,
|
||
|
|
"loss/core": 0.6899994611740112,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 1.9646492004394531,
|
||
|
|
"rewards/margins": 1.2791537046432495,
|
||
|
|
"rewards/rejected": 0.6854956150054932,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.788247939806521,
|
||
|
|
"grad_norm": 0.546875,
|
||
|
|
"learning_rate": 1.0782514843499652e-08,
|
||
|
|
"logits/chosen": -1.7240092754364014,
|
||
|
|
"logits/rejected": -1.8827893733978271,
|
||
|
|
"logps/chosen": -0.2997475564479828,
|
||
|
|
"logps/rejected": -0.283795028924942,
|
||
|
|
"loss": 0.690019965171814,
|
||
|
|
"loss/core": 0.690019965171814,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 2.4867701530456543,
|
||
|
|
"rewards/margins": 1.2625161409378052,
|
||
|
|
"rewards/rejected": 1.22425377368927,
|
||
|
|
"step": 825
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7930252000477725,
|
||
|
|
"grad_norm": 0.2412109375,
|
||
|
|
"learning_rate": 9.419118673676924e-09,
|
||
|
|
"logits/chosen": -1.9089800119400024,
|
||
|
|
"logits/rejected": -1.8700017929077148,
|
||
|
|
"logps/chosen": -0.27884355187416077,
|
||
|
|
"logps/rejected": -0.30505234003067017,
|
||
|
|
"loss": 0.6930972933769226,
|
||
|
|
"loss/core": 0.6930972933769226,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.7232449054718018,
|
||
|
|
"rewards/margins": 0.04113560914993286,
|
||
|
|
"rewards/rejected": 1.6821091175079346,
|
||
|
|
"step": 830
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7978024602890242,
|
||
|
|
"grad_norm": 1.296875,
|
||
|
|
"learning_rate": 8.14619513428405e-09,
|
||
|
|
"logits/chosen": -1.7318191528320312,
|
||
|
|
"logits/rejected": -1.74942946434021,
|
||
|
|
"logps/chosen": -0.25466471910476685,
|
||
|
|
"logps/rejected": -0.30034616589546204,
|
||
|
|
"loss": 0.6893445253372192,
|
||
|
|
"loss/core": 0.6893445253372192,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.404738426208496,
|
||
|
|
"rewards/margins": 1.5665807723999023,
|
||
|
|
"rewards/rejected": 0.8381577730178833,
|
||
|
|
"step": 835
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8025797205302759,
|
||
|
|
"grad_norm": 4.0,
|
||
|
|
"learning_rate": 6.96422291980539e-09,
|
||
|
|
"logits/chosen": -1.8222589492797852,
|
||
|
|
"logits/rejected": -1.8721163272857666,
|
||
|
|
"logps/chosen": -0.32260316610336304,
|
||
|
|
"logps/rejected": -0.30097338557243347,
|
||
|
|
"loss": 0.6889389157295227,
|
||
|
|
"loss/core": 0.6889389157295227,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.8018436431884766,
|
||
|
|
"rewards/margins": 1.7603601217269897,
|
||
|
|
"rewards/rejected": 1.0414836406707764,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8073569807715275,
|
||
|
|
"grad_norm": 0.1318359375,
|
||
|
|
"learning_rate": 5.8736465216517594e-09,
|
||
|
|
"logits/chosen": -1.7940136194229126,
|
||
|
|
"logits/rejected": -1.8399118185043335,
|
||
|
|
"logps/chosen": -0.2931648790836334,
|
||
|
|
"logps/rejected": -0.3112708628177643,
|
||
|
|
"loss": 0.6907740235328674,
|
||
|
|
"loss/core": 0.6907740235328674,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 1.4224978685379028,
|
||
|
|
"rewards/margins": 0.967786431312561,
|
||
|
|
"rewards/rejected": 0.45471152663230896,
|
||
|
|
"step": 845
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8121342410127792,
|
||
|
|
"grad_norm": 0.703125,
|
||
|
|
"learning_rate": 4.874876061005173e-09,
|
||
|
|
"logits/chosen": -1.9608793258666992,
|
||
|
|
"logits/rejected": -2.056640625,
|
||
|
|
"logps/chosen": -0.25170543789863586,
|
||
|
|
"logps/rejected": -0.24661597609519958,
|
||
|
|
"loss": 0.6915278434753418,
|
||
|
|
"loss/core": 0.6915278434753418,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.5610125064849854,
|
||
|
|
"rewards/margins": 0.6514654755592346,
|
||
|
|
"rewards/rejected": 0.9095470309257507,
|
||
|
|
"step": 850
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8169115012540308,
|
||
|
|
"grad_norm": 0.44140625,
|
||
|
|
"learning_rate": 3.968287134589188e-09,
|
||
|
|
"logits/chosen": -1.767511010169983,
|
||
|
|
"logits/rejected": -1.7454007863998413,
|
||
|
|
"logps/chosen": -0.30029794573783875,
|
||
|
|
"logps/rejected": -0.29418742656707764,
|
||
|
|
"loss": 0.6896105408668518,
|
||
|
|
"loss/core": 0.6896105408668518,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.694100856781006,
|
||
|
|
"rewards/margins": 1.4545179605484009,
|
||
|
|
"rewards/rejected": 1.2395830154418945,
|
||
|
|
"step": 855
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8216887614952825,
|
||
|
|
"grad_norm": 0.095703125,
|
||
|
|
"learning_rate": 3.154220673422192e-09,
|
||
|
|
"logits/chosen": -1.6382343769073486,
|
||
|
|
"logits/rejected": -1.68215811252594,
|
||
|
|
"logps/chosen": -0.29962679743766785,
|
||
|
|
"logps/rejected": -0.2946147322654724,
|
||
|
|
"loss": 0.6880518198013306,
|
||
|
|
"loss/core": 0.6880518198013306,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 4.085516452789307,
|
||
|
|
"rewards/margins": 2.11319637298584,
|
||
|
|
"rewards/rejected": 1.9723198413848877,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.826466021736534,
|
||
|
|
"grad_norm": 0.275390625,
|
||
|
|
"learning_rate": 2.4329828146074096e-09,
|
||
|
|
"logits/chosen": -1.656494379043579,
|
||
|
|
"logits/rejected": -1.6885122060775757,
|
||
|
|
"logps/chosen": -0.2906996011734009,
|
||
|
|
"logps/rejected": -0.2946203649044037,
|
||
|
|
"loss": 0.6910411715507507,
|
||
|
|
"loss/core": 0.6910411715507507,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 2.4540257453918457,
|
||
|
|
"rewards/margins": 0.8756850957870483,
|
||
|
|
"rewards/rejected": 1.5783402919769287,
|
||
|
|
"step": 865
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8312432819777857,
|
||
|
|
"grad_norm": 0.236328125,
|
||
|
|
"learning_rate": 1.8048447862070714e-09,
|
||
|
|
"logits/chosen": -1.8312631845474243,
|
||
|
|
"logits/rejected": -1.8569021224975586,
|
||
|
|
"logps/chosen": -0.3106675148010254,
|
||
|
|
"logps/rejected": -0.29439225792884827,
|
||
|
|
"loss": 0.6878628134727478,
|
||
|
|
"loss/core": 0.6878628134727478,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.189491033554077,
|
||
|
|
"rewards/margins": 2.2258591651916504,
|
||
|
|
"rewards/rejected": 0.9636317491531372,
|
||
|
|
"step": 870
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8360205422190374,
|
||
|
|
"grad_norm": 0.1787109375,
|
||
|
|
"learning_rate": 1.2700428052447033e-09,
|
||
|
|
"logits/chosen": -1.7763378620147705,
|
||
|
|
"logits/rejected": -1.8419290781021118,
|
||
|
|
"logps/chosen": -0.3076077103614807,
|
||
|
|
"logps/rejected": -0.3310580551624298,
|
||
|
|
"loss": 0.6911420822143555,
|
||
|
|
"loss/core": 0.6911420822143555,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 1.865380883216858,
|
||
|
|
"rewards/margins": 0.844092071056366,
|
||
|
|
"rewards/rejected": 1.0212888717651367,
|
||
|
|
"step": 875
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.840797802460289,
|
||
|
|
"grad_norm": 1.4375,
|
||
|
|
"learning_rate": 8.287779888734858e-10,
|
||
|
|
"logits/chosen": -1.8257224559783936,
|
||
|
|
"logits/rejected": -1.7873103618621826,
|
||
|
|
"logps/chosen": -0.29086047410964966,
|
||
|
|
"logps/rejected": -0.327653706073761,
|
||
|
|
"loss": 0.6917628049850464,
|
||
|
|
"loss/core": 0.6917628049850464,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.455867290496826,
|
||
|
|
"rewards/margins": 0.6519712209701538,
|
||
|
|
"rewards/rejected": 1.8038959503173828,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8455750627015407,
|
||
|
|
"grad_norm": 0.291015625,
|
||
|
|
"learning_rate": 4.812162787445062e-10,
|
||
|
|
"logits/chosen": -1.7672336101531982,
|
||
|
|
"logits/rejected": -1.8428665399551392,
|
||
|
|
"logps/chosen": -0.28123897314071655,
|
||
|
|
"logps/rejected": -0.27698567509651184,
|
||
|
|
"loss": 0.689162015914917,
|
||
|
|
"loss/core": 0.689162015914917,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 2.7381834983825684,
|
||
|
|
"rewards/margins": 1.6145788431167603,
|
||
|
|
"rewards/rejected": 1.123604655265808,
|
||
|
|
"step": 885
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8503523229427923,
|
||
|
|
"grad_norm": 0.97265625,
|
||
|
|
"learning_rate": 2.2748837860270265e-10,
|
||
|
|
"logits/chosen": -1.7094967365264893,
|
||
|
|
"logits/rejected": -1.819848656654358,
|
||
|
|
"logps/chosen": -0.29713571071624756,
|
||
|
|
"logps/rejected": -0.2740597426891327,
|
||
|
|
"loss": 0.6906223297119141,
|
||
|
|
"loss/core": 0.6906223297119141,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.4846818447113037,
|
||
|
|
"rewards/margins": 1.043480396270752,
|
||
|
|
"rewards/rejected": 1.4412018060684204,
|
||
|
|
"step": 890
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.855129583184044,
|
||
|
|
"grad_norm": 0.451171875,
|
||
|
|
"learning_rate": 6.768970513457151e-11,
|
||
|
|
"logits/chosen": -1.7548071146011353,
|
||
|
|
"logits/rejected": -1.7341846227645874,
|
||
|
|
"logps/chosen": -0.2722460627555847,
|
||
|
|
"logps/rejected": -0.2857206165790558,
|
||
|
|
"loss": 0.6913274526596069,
|
||
|
|
"loss/core": 0.6913274526596069,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 2.302839517593384,
|
||
|
|
"rewards/margins": 0.7413143515586853,
|
||
|
|
"rewards/rejected": 1.5615251064300537,
|
||
|
|
"step": 895
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8599068434252956,
|
||
|
|
"grad_norm": 0.40625,
|
||
|
|
"learning_rate": 1.8803520859811406e-12,
|
||
|
|
"logits/chosen": -1.736576795578003,
|
||
|
|
"logits/rejected": -1.8592958450317383,
|
||
|
|
"logps/chosen": -0.26908183097839355,
|
||
|
|
"logps/rejected": -0.2669498324394226,
|
||
|
|
"loss": 0.6877947449684143,
|
||
|
|
"loss/core": 0.6877947449684143,
|
||
|
|
"rewards/accuracies": 0.9750000238418579,
|
||
|
|
"rewards/chosen": 3.115830659866333,
|
||
|
|
"rewards/margins": 2.253107786178589,
|
||
|
|
"rewards/rejected": 0.8627226948738098,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8599068434252956,
|
||
|
|
"step": 900,
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_loss": 0.690198503865136,
|
||
|
|
"train_runtime": 8363.1914,
|
||
|
|
"train_samples_per_second": 1.722,
|
||
|
|
"train_steps_per_second": 0.108
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 5,
|
||
|
|
"max_steps": 900,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 1,
|
||
|
|
"save_steps": 900,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": true
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 2,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|