2924 lines
101 KiB
JSON
2924 lines
101 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.8599068434252956,
|
|
"eval_steps": 500,
|
|
"global_step": 900,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.004777260241251642,
|
|
"grad_norm": 24.0,
|
|
"learning_rate": 2.222222222222222e-08,
|
|
"logits/chosen": -1.997597098350525,
|
|
"logits/rejected": -2.0135111808776855,
|
|
"logps/chosen": -0.2795145511627197,
|
|
"logps/rejected": -0.28208082914352417,
|
|
"loss": 0.38181036710739136,
|
|
"loss/core": 0.38181036710739136,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.418922185897827,
|
|
"rewards/margins": 1.4596340656280518,
|
|
"rewards/rejected": 0.9592880010604858,
|
|
"step": 5
|
|
},
|
|
{
|
|
"epoch": 0.009554520482503284,
|
|
"grad_norm": 190.0,
|
|
"learning_rate": 5e-08,
|
|
"logits/chosen": -2.3454456329345703,
|
|
"logits/rejected": -2.392616033554077,
|
|
"logps/chosen": -0.2907744348049164,
|
|
"logps/rejected": -0.28635361790657043,
|
|
"loss": 1.0456370115280151,
|
|
"loss/core": 1.0456370115280151,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.6759283542633057,
|
|
"rewards/margins": 0.041599858552217484,
|
|
"rewards/rejected": 1.6343282461166382,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.014331780723754926,
|
|
"grad_norm": 81.5,
|
|
"learning_rate": 7.777777777777778e-08,
|
|
"logits/chosen": -2.2278778553009033,
|
|
"logits/rejected": -2.1911208629608154,
|
|
"logps/chosen": -0.2769435942173004,
|
|
"logps/rejected": -0.28479596972465515,
|
|
"loss": 0.3574354350566864,
|
|
"loss/core": 0.3574354350566864,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.138868570327759,
|
|
"rewards/margins": 1.00614595413208,
|
|
"rewards/rejected": 1.1327224969863892,
|
|
"step": 15
|
|
},
|
|
{
|
|
"epoch": 0.01910904096500657,
|
|
"grad_norm": 21.75,
|
|
"learning_rate": 1.0555555555555555e-07,
|
|
"logits/chosen": -2.1759560108184814,
|
|
"logits/rejected": -2.218474864959717,
|
|
"logps/chosen": -0.300750195980072,
|
|
"logps/rejected": -0.2876606285572052,
|
|
"loss": 0.4318433701992035,
|
|
"loss/core": 0.4318433701992035,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.921849012374878,
|
|
"rewards/margins": 1.0531971454620361,
|
|
"rewards/rejected": 0.8686518669128418,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.02388630120625821,
|
|
"grad_norm": 25.5,
|
|
"learning_rate": 1.3333333333333334e-07,
|
|
"logits/chosen": -2.0754263401031494,
|
|
"logits/rejected": -2.136157512664795,
|
|
"logps/chosen": -0.27347683906555176,
|
|
"logps/rejected": -0.2596595585346222,
|
|
"loss": 1.702324628829956,
|
|
"loss/core": 1.702324628829956,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.820397138595581,
|
|
"rewards/margins": 2.1924726963043213,
|
|
"rewards/rejected": 1.6279243230819702,
|
|
"step": 25
|
|
},
|
|
{
|
|
"epoch": 0.028663561447509853,
|
|
"grad_norm": 274.0,
|
|
"learning_rate": 1.611111111111111e-07,
|
|
"logits/chosen": -2.3495566844940186,
|
|
"logits/rejected": -2.331704616546631,
|
|
"logps/chosen": -0.30199751257896423,
|
|
"logps/rejected": -0.3243906497955322,
|
|
"loss": 0.6514055132865906,
|
|
"loss/core": 0.6514055132865906,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.259319543838501,
|
|
"rewards/margins": 1.2338563203811646,
|
|
"rewards/rejected": 1.0254631042480469,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.033440821688761495,
|
|
"grad_norm": 2.265625,
|
|
"learning_rate": 1.8888888888888888e-07,
|
|
"logits/chosen": -2.204205274581909,
|
|
"logits/rejected": -2.1856656074523926,
|
|
"logps/chosen": -0.2655085027217865,
|
|
"logps/rejected": -0.26616171002388,
|
|
"loss": 0.814064621925354,
|
|
"loss/core": 0.814064621925354,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.6806728839874268,
|
|
"rewards/margins": 1.5299947261810303,
|
|
"rewards/rejected": 1.1506781578063965,
|
|
"step": 35
|
|
},
|
|
{
|
|
"epoch": 0.03821808193001314,
|
|
"grad_norm": 80.0,
|
|
"learning_rate": 2.1666666666666667e-07,
|
|
"logits/chosen": -2.0009946823120117,
|
|
"logits/rejected": -2.060720682144165,
|
|
"logps/chosen": -0.2918882966041565,
|
|
"logps/rejected": -0.28243204951286316,
|
|
"loss": 1.9676904678344727,
|
|
"loss/core": 1.9676904678344727,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.485463619232178,
|
|
"rewards/margins": 2.8883402347564697,
|
|
"rewards/rejected": 1.5971235036849976,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.04299534217126478,
|
|
"grad_norm": 8.125,
|
|
"learning_rate": 2.4444444444444445e-07,
|
|
"logits/chosen": -2.197145938873291,
|
|
"logits/rejected": -2.359992504119873,
|
|
"logps/chosen": -0.28900036215782166,
|
|
"logps/rejected": -0.28796130418777466,
|
|
"loss": 0.2774832248687744,
|
|
"loss/core": 0.2774832248687744,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.0935540199279785,
|
|
"rewards/margins": 1.2470275163650513,
|
|
"rewards/rejected": 0.8465266227722168,
|
|
"step": 45
|
|
},
|
|
{
|
|
"epoch": 0.04777260241251642,
|
|
"grad_norm": 77.0,
|
|
"learning_rate": 2.7222222222222216e-07,
|
|
"logits/chosen": -2.166928768157959,
|
|
"logits/rejected": -2.1634888648986816,
|
|
"logps/chosen": -0.28184059262275696,
|
|
"logps/rejected": -0.28433579206466675,
|
|
"loss": 0.7104421257972717,
|
|
"loss/core": 0.7104421257972717,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.8780951499938965,
|
|
"rewards/margins": 1.4217535257339478,
|
|
"rewards/rejected": 1.4563415050506592,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.05254986265376806,
|
|
"grad_norm": 0.87109375,
|
|
"learning_rate": 3e-07,
|
|
"logits/chosen": -2.197507858276367,
|
|
"logits/rejected": -2.185612916946411,
|
|
"logps/chosen": -0.2892429828643799,
|
|
"logps/rejected": -0.2890060245990753,
|
|
"loss": 1.496902346611023,
|
|
"loss/core": 1.496902346611023,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.2559685707092285,
|
|
"rewards/margins": 0.21959257125854492,
|
|
"rewards/rejected": 2.0363757610321045,
|
|
"step": 55
|
|
},
|
|
{
|
|
"epoch": 0.057327122895019705,
|
|
"grad_norm": 520.0,
|
|
"learning_rate": 3.2777777777777776e-07,
|
|
"logits/chosen": -2.09147572517395,
|
|
"logits/rejected": -2.1203830242156982,
|
|
"logps/chosen": -0.2743294835090637,
|
|
"logps/rejected": -0.28498584032058716,
|
|
"loss": 0.337458074092865,
|
|
"loss/core": 0.337458074092865,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.1575465202331543,
|
|
"rewards/margins": 0.6136163473129272,
|
|
"rewards/rejected": 1.543930172920227,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.06210438313627135,
|
|
"grad_norm": 7.1875,
|
|
"learning_rate": 3.5555555555555553e-07,
|
|
"logits/chosen": -1.9565308094024658,
|
|
"logits/rejected": -2.059638023376465,
|
|
"logps/chosen": -0.3096534311771393,
|
|
"logps/rejected": -0.30584970116615295,
|
|
"loss": 0.9800235629081726,
|
|
"loss/core": 0.9800235629081726,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.9653208255767822,
|
|
"rewards/margins": 1.7889091968536377,
|
|
"rewards/rejected": 1.176411747932434,
|
|
"step": 65
|
|
},
|
|
{
|
|
"epoch": 0.06688164337752299,
|
|
"grad_norm": 54.25,
|
|
"learning_rate": 3.8333333333333335e-07,
|
|
"logits/chosen": -1.9700387716293335,
|
|
"logits/rejected": -1.955226182937622,
|
|
"logps/chosen": -0.3858351707458496,
|
|
"logps/rejected": -0.2979809641838074,
|
|
"loss": 1.823669195175171,
|
|
"loss/core": 1.823669195175171,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.578740358352661,
|
|
"rewards/margins": 1.4053298234939575,
|
|
"rewards/rejected": 2.1734108924865723,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.07165890361877464,
|
|
"grad_norm": 6.875,
|
|
"learning_rate": 4.1111111111111107e-07,
|
|
"logits/chosen": -2.0864429473876953,
|
|
"logits/rejected": -2.051452398300171,
|
|
"logps/chosen": -0.3145933449268341,
|
|
"logps/rejected": -0.2891804873943329,
|
|
"loss": 0.7372748255729675,
|
|
"loss/core": 0.7372748255729675,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.610942840576172,
|
|
"rewards/margins": 1.0181684494018555,
|
|
"rewards/rejected": 1.5927742719650269,
|
|
"step": 75
|
|
},
|
|
{
|
|
"epoch": 0.07643616386002627,
|
|
"grad_norm": 4.15625,
|
|
"learning_rate": 4.3888888888888884e-07,
|
|
"logits/chosen": -2.066887855529785,
|
|
"logits/rejected": -2.186957836151123,
|
|
"logps/chosen": -0.26004475355148315,
|
|
"logps/rejected": -0.27356016635894775,
|
|
"loss": 0.9191620945930481,
|
|
"loss/core": 0.9191620945930481,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.2465484142303467,
|
|
"rewards/margins": 1.9899933338165283,
|
|
"rewards/rejected": 1.2565548419952393,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.08121342410127792,
|
|
"grad_norm": 5.75,
|
|
"learning_rate": 4.6666666666666666e-07,
|
|
"logits/chosen": -2.1885199546813965,
|
|
"logits/rejected": -2.2368600368499756,
|
|
"logps/chosen": -0.30883267521858215,
|
|
"logps/rejected": -0.2775411307811737,
|
|
"loss": 1.6861064434051514,
|
|
"loss/core": 1.6861064434051514,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.484980821609497,
|
|
"rewards/margins": 2.152528762817383,
|
|
"rewards/rejected": 1.3324521780014038,
|
|
"step": 85
|
|
},
|
|
{
|
|
"epoch": 0.08599068434252956,
|
|
"grad_norm": 0.98828125,
|
|
"learning_rate": 4.944444444444445e-07,
|
|
"logits/chosen": -2.4196181297302246,
|
|
"logits/rejected": -2.4065661430358887,
|
|
"logps/chosen": -0.281704306602478,
|
|
"logps/rejected": -0.2961405813694,
|
|
"loss": 0.30765506625175476,
|
|
"loss/core": 0.30765506625175476,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.7503442764282227,
|
|
"rewards/margins": 0.9065144658088684,
|
|
"rewards/rejected": 0.8438299894332886,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.09076794458378121,
|
|
"grad_norm": 62.5,
|
|
"learning_rate": 4.999699149323369e-07,
|
|
"logits/chosen": -2.0543789863586426,
|
|
"logits/rejected": -2.11065673828125,
|
|
"logps/chosen": -0.26749688386917114,
|
|
"logps/rejected": -0.2789869010448456,
|
|
"loss": 0.906001091003418,
|
|
"loss/core": 0.906001091003418,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.2087581157684326,
|
|
"rewards/margins": 1.1049243211746216,
|
|
"rewards/rejected": 2.1038336753845215,
|
|
"step": 95
|
|
},
|
|
{
|
|
"epoch": 0.09554520482503284,
|
|
"grad_norm": 10.625,
|
|
"learning_rate": 4.998477067547739e-07,
|
|
"logits/chosen": -2.231830596923828,
|
|
"logits/rejected": -2.2774977684020996,
|
|
"logps/chosen": -0.25898346304893494,
|
|
"logps/rejected": -0.24633264541625977,
|
|
"loss": 0.3568304777145386,
|
|
"loss/core": 0.3568304777145386,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.444751024246216,
|
|
"rewards/margins": 1.1550977230072021,
|
|
"rewards/rejected": 1.2896530628204346,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.10032246506628449,
|
|
"grad_norm": 298.0,
|
|
"learning_rate": 4.996315410727229e-07,
|
|
"logits/chosen": -2.3348228931427,
|
|
"logits/rejected": -2.4207358360290527,
|
|
"logps/chosen": -0.2856779098510742,
|
|
"logps/rejected": -0.279996782541275,
|
|
"loss": 0.3823498785495758,
|
|
"loss/core": 0.3823498785495758,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.9824638366699219,
|
|
"rewards/margins": 1.1864570379257202,
|
|
"rewards/rejected": 0.7960068583488464,
|
|
"step": 105
|
|
},
|
|
{
|
|
"epoch": 0.10509972530753613,
|
|
"grad_norm": 11.875,
|
|
"learning_rate": 4.993214991772562e-07,
|
|
"logits/chosen": -1.9041261672973633,
|
|
"logits/rejected": -1.8569024801254272,
|
|
"logps/chosen": -0.3389981985092163,
|
|
"logps/rejected": -0.29700320959091187,
|
|
"loss": 1.7510207891464233,
|
|
"loss/core": 1.7510207891464233,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.465295314788818,
|
|
"rewards/margins": 2.569420576095581,
|
|
"rewards/rejected": 1.8958752155303955,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.10987698554878778,
|
|
"grad_norm": 9.875,
|
|
"learning_rate": 4.989176976624511e-07,
|
|
"logits/chosen": -2.0836009979248047,
|
|
"logits/rejected": -2.027132749557495,
|
|
"logps/chosen": -0.2996644973754883,
|
|
"logps/rejected": -0.30102577805519104,
|
|
"loss": 0.4198575019836426,
|
|
"loss/core": 0.4198575019836426,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.7827256917953491,
|
|
"rewards/margins": 0.3579304814338684,
|
|
"rewards/rejected": 1.424795389175415,
|
|
"step": 115
|
|
},
|
|
{
|
|
"epoch": 0.11465424579003941,
|
|
"grad_norm": 184.0,
|
|
"learning_rate": 4.984202883815428e-07,
|
|
"logits/chosen": -2.0339629650115967,
|
|
"logits/rejected": -2.0718021392822266,
|
|
"logps/chosen": -0.30192404985427856,
|
|
"logps/rejected": -0.29958024621009827,
|
|
"loss": 0.16679374873638153,
|
|
"loss/core": 0.16679374873638153,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.8123009204864502,
|
|
"rewards/margins": 0.7246686220169067,
|
|
"rewards/rejected": 1.0876322984695435,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.11943150603129106,
|
|
"grad_norm": 5.5625,
|
|
"learning_rate": 4.978294583898195e-07,
|
|
"logits/chosen": -2.104386806488037,
|
|
"logits/rejected": -2.1359241008758545,
|
|
"logps/chosen": -0.2988458275794983,
|
|
"logps/rejected": -0.3063589632511139,
|
|
"loss": 0.2692359387874603,
|
|
"loss/core": 0.2692359387874603,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.067023992538452,
|
|
"rewards/margins": 1.047250747680664,
|
|
"rewards/rejected": 1.019773244857788,
|
|
"step": 125
|
|
},
|
|
{
|
|
"epoch": 0.1242087662725427,
|
|
"grad_norm": 10.6875,
|
|
"learning_rate": 4.971454298742779e-07,
|
|
"logits/chosen": -1.994943380355835,
|
|
"logits/rejected": -2.0879642963409424,
|
|
"logps/chosen": -0.2800437808036804,
|
|
"logps/rejected": -0.279203325510025,
|
|
"loss": 0.14723244309425354,
|
|
"loss/core": 0.14723244309425354,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.806762456893921,
|
|
"rewards/margins": 0.9239923357963562,
|
|
"rewards/rejected": 0.8827699422836304,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.12898602651379434,
|
|
"grad_norm": 21.75,
|
|
"learning_rate": 4.963684600700678e-07,
|
|
"logits/chosen": -2.0894951820373535,
|
|
"logits/rejected": -2.16270112991333,
|
|
"logps/chosen": -0.3019745349884033,
|
|
"logps/rejected": -0.30207985639572144,
|
|
"loss": 0.8815671801567078,
|
|
"loss/core": 0.8815671801567078,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.3753859996795654,
|
|
"rewards/margins": 2.099947214126587,
|
|
"rewards/rejected": 1.2754385471343994,
|
|
"step": 135
|
|
},
|
|
{
|
|
"epoch": 0.13376328675504598,
|
|
"grad_norm": 11.1875,
|
|
"learning_rate": 4.954988411637571e-07,
|
|
"logits/chosen": -2.44862699508667,
|
|
"logits/rejected": -2.508138656616211,
|
|
"logps/chosen": -0.28244370222091675,
|
|
"logps/rejected": -0.2900107800960541,
|
|
"loss": 0.1148979440331459,
|
|
"loss/core": 0.1148979440331459,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.6288248300552368,
|
|
"rewards/margins": 0.9194561243057251,
|
|
"rewards/rejected": 0.7093685865402222,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.13854054699629761,
|
|
"grad_norm": 7.9375,
|
|
"learning_rate": 4.945369001834514e-07,
|
|
"logits/chosen": -1.7729772329330444,
|
|
"logits/rejected": -1.803558111190796,
|
|
"logps/chosen": -0.2981434464454651,
|
|
"logps/rejected": -0.30744919180870056,
|
|
"loss": 2.0826029777526855,
|
|
"loss/core": 2.0826029777526855,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.636133909225464,
|
|
"rewards/margins": 1.9079220294952393,
|
|
"rewards/rejected": 1.7282119989395142,
|
|
"step": 145
|
|
},
|
|
{
|
|
"epoch": 0.14331780723754928,
|
|
"grad_norm": 111.0,
|
|
"learning_rate": 4.93482998875813e-07,
|
|
"logits/chosen": -2.2263197898864746,
|
|
"logits/rejected": -2.274482488632202,
|
|
"logps/chosen": -0.31686437129974365,
|
|
"logps/rejected": -0.3086201250553131,
|
|
"loss": 1.6554409265518188,
|
|
"loss/core": 1.6554409265518188,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.644029140472412,
|
|
"rewards/margins": 2.1906416416168213,
|
|
"rewards/rejected": 1.4533871412277222,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.1480950674788009,
|
|
"grad_norm": 4.875,
|
|
"learning_rate": 4.923375335700223e-07,
|
|
"logits/chosen": -2.094820022583008,
|
|
"logits/rejected": -2.17297101020813,
|
|
"logps/chosen": -0.2831822335720062,
|
|
"logps/rejected": -0.2986108958721161,
|
|
"loss": 0.7216386198997498,
|
|
"loss/core": 0.7216386198997498,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.8636791706085205,
|
|
"rewards/margins": 2.1000559329986572,
|
|
"rewards/rejected": 0.7636229395866394,
|
|
"step": 155
|
|
},
|
|
{
|
|
"epoch": 0.15287232772005255,
|
|
"grad_norm": 272.0,
|
|
"learning_rate": 4.911009350287347e-07,
|
|
"logits/chosen": -2.0500402450561523,
|
|
"logits/rejected": -2.0241525173187256,
|
|
"logps/chosen": -0.3109878599643707,
|
|
"logps/rejected": -0.3158418536186218,
|
|
"loss": 1.3895409107208252,
|
|
"loss/core": 1.3895409107208252,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.1455612182617188,
|
|
"rewards/margins": 1.7495533227920532,
|
|
"rewards/rejected": 1.396007776260376,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.15764958796130418,
|
|
"grad_norm": 5.09375,
|
|
"learning_rate": 4.897736682860885e-07,
|
|
"logits/chosen": -2.216140031814575,
|
|
"logits/rejected": -2.146721363067627,
|
|
"logps/chosen": -0.31132304668426514,
|
|
"logps/rejected": -0.30651506781578064,
|
|
"loss": 0.1788981854915619,
|
|
"loss/core": 0.1788981854915619,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.6000747680664062,
|
|
"rewards/margins": 0.8192144632339478,
|
|
"rewards/rejected": 0.7808603048324585,
|
|
"step": 165
|
|
},
|
|
{
|
|
"epoch": 0.16242684820255585,
|
|
"grad_norm": 402.0,
|
|
"learning_rate": 4.883562324728241e-07,
|
|
"logits/chosen": -2.18127179145813,
|
|
"logits/rejected": -2.126049518585205,
|
|
"logps/chosen": -0.2751903235912323,
|
|
"logps/rejected": -0.261197030544281,
|
|
"loss": 1.8691749572753906,
|
|
"loss/core": 1.8691749572753906,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.59234881401062,
|
|
"rewards/margins": -0.16324080526828766,
|
|
"rewards/rejected": 2.755589485168457,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.16720410844380748,
|
|
"grad_norm": 460.0,
|
|
"learning_rate": 4.868491606285823e-07,
|
|
"logits/chosen": -2.1704468727111816,
|
|
"logits/rejected": -2.2109594345092773,
|
|
"logps/chosen": -0.2882474958896637,
|
|
"logps/rejected": -0.3017005920410156,
|
|
"loss": 1.1066371202468872,
|
|
"loss/core": 1.1066371202468872,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.569884777069092,
|
|
"rewards/margins": 1.3101545572280884,
|
|
"rewards/rejected": 1.259730339050293,
|
|
"step": 175
|
|
},
|
|
{
|
|
"epoch": 0.17198136868505912,
|
|
"grad_norm": 640.0,
|
|
"learning_rate": 4.852530195014489e-07,
|
|
"logits/chosen": -2.193326711654663,
|
|
"logits/rejected": -2.104137897491455,
|
|
"logps/chosen": -0.2730237543582916,
|
|
"logps/rejected": -0.26204752922058105,
|
|
"loss": 1.1543080806732178,
|
|
"loss/core": 1.1543080806732178,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.2240982055664062,
|
|
"rewards/margins": 1.7351115942001343,
|
|
"rewards/rejected": 1.4889863729476929,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.17675862892631075,
|
|
"grad_norm": 3.40625,
|
|
"learning_rate": 4.835684093348244e-07,
|
|
"logits/chosen": -2.1960487365722656,
|
|
"logits/rejected": -2.154365062713623,
|
|
"logps/chosen": -0.2888965308666229,
|
|
"logps/rejected": -0.29239413142204285,
|
|
"loss": 0.06165605038404465,
|
|
"loss/core": 0.06165605038404465,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.0407021045684814,
|
|
"rewards/margins": 0.41684430837631226,
|
|
"rewards/rejected": 0.6238579154014587,
|
|
"step": 185
|
|
},
|
|
{
|
|
"epoch": 0.18153588916756241,
|
|
"grad_norm": 28.875,
|
|
"learning_rate": 4.817959636416969e-07,
|
|
"logits/chosen": -2.2871928215026855,
|
|
"logits/rejected": -2.264662981033325,
|
|
"logps/chosen": -0.28313952684402466,
|
|
"logps/rejected": -0.28482240438461304,
|
|
"loss": 1.1405375003814697,
|
|
"loss/core": 1.1405375003814697,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.842782497406006,
|
|
"rewards/margins": 1.609337568283081,
|
|
"rewards/rejected": 1.2334448099136353,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.18631314940881405,
|
|
"grad_norm": 23.5,
|
|
"learning_rate": 4.799363489664039e-07,
|
|
"logits/chosen": -1.9776523113250732,
|
|
"logits/rejected": -1.999280571937561,
|
|
"logps/chosen": -0.289995938539505,
|
|
"logps/rejected": -0.2919548749923706,
|
|
"loss": 1.1198909282684326,
|
|
"loss/core": 1.1198909282684326,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.678499937057495,
|
|
"rewards/margins": 1.7345187664031982,
|
|
"rewards/rejected": 0.9439811706542969,
|
|
"step": 195
|
|
},
|
|
{
|
|
"epoch": 0.19109040965006568,
|
|
"grad_norm": 20.625,
|
|
"learning_rate": 4.779902646339721e-07,
|
|
"logits/chosen": -1.88909113407135,
|
|
"logits/rejected": -1.9297901391983032,
|
|
"logps/chosen": -0.32461681962013245,
|
|
"logps/rejected": -0.3156924545764923,
|
|
"loss": 0.4924043118953705,
|
|
"loss/core": 0.4924043118953705,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.604013681411743,
|
|
"rewards/margins": 1.374180555343628,
|
|
"rewards/rejected": 1.2298332452774048,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.19586766989131732,
|
|
"grad_norm": 9.875,
|
|
"learning_rate": 4.759584424871301e-07,
|
|
"logits/chosen": -2.110816478729248,
|
|
"logits/rejected": -2.1626930236816406,
|
|
"logps/chosen": -0.25076472759246826,
|
|
"logps/rejected": -0.2683349549770355,
|
|
"loss": 1.2883168458938599,
|
|
"loss/core": 1.2883168458938599,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.282876491546631,
|
|
"rewards/margins": 1.168548345565796,
|
|
"rewards/rejected": 2.114327907562256,
|
|
"step": 205
|
|
},
|
|
{
|
|
"epoch": 0.20064493013256898,
|
|
"grad_norm": 27.875,
|
|
"learning_rate": 4.738416466110917e-07,
|
|
"logits/chosen": -2.1067051887512207,
|
|
"logits/rejected": -2.1317334175109863,
|
|
"logps/chosen": -0.3131873607635498,
|
|
"logps/rejected": -0.32463163137435913,
|
|
"loss": 1.1303998231887817,
|
|
"loss/core": 1.1303998231887817,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.8242244720458984,
|
|
"rewards/margins": 1.3681774139404297,
|
|
"rewards/rejected": 1.4560470581054688,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.20542219037382062,
|
|
"grad_norm": 6.875,
|
|
"learning_rate": 4.716406730462153e-07,
|
|
"logits/chosen": -2.079124927520752,
|
|
"logits/rejected": -2.069756031036377,
|
|
"logps/chosen": -0.2739131450653076,
|
|
"logps/rejected": -0.2875369191169739,
|
|
"loss": 1.5732624530792236,
|
|
"loss/core": 1.5732624530792236,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.207057237625122,
|
|
"rewards/margins": 1.5373919010162354,
|
|
"rewards/rejected": 1.6696653366088867,
|
|
"step": 215
|
|
},
|
|
{
|
|
"epoch": 0.21019945061507225,
|
|
"grad_norm": 30.375,
|
|
"learning_rate": 4.693563494886454e-07,
|
|
"logits/chosen": -2.0912394523620605,
|
|
"logits/rejected": -2.0069117546081543,
|
|
"logps/chosen": -0.3027765452861786,
|
|
"logps/rejected": -0.30113840103149414,
|
|
"loss": 0.21921542286872864,
|
|
"loss/core": 0.21921542286872864,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.8828872442245483,
|
|
"rewards/margins": 0.8069303631782532,
|
|
"rewards/rejected": 1.0759570598602295,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.2149767108563239,
|
|
"grad_norm": 8.25,
|
|
"learning_rate": 4.6698953497905016e-07,
|
|
"logits/chosen": -2.1752381324768066,
|
|
"logits/rejected": -2.1640679836273193,
|
|
"logps/chosen": -0.2903914749622345,
|
|
"logps/rejected": -0.2762240171432495,
|
|
"loss": 0.8437272310256958,
|
|
"loss/core": 0.8437272310256958,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.6241302490234375,
|
|
"rewards/margins": 2.222364902496338,
|
|
"rewards/rejected": 1.4017655849456787,
|
|
"step": 225
|
|
},
|
|
{
|
|
"epoch": 0.21975397109757555,
|
|
"grad_norm": 54.0,
|
|
"learning_rate": 4.645411195795709e-07,
|
|
"logits/chosen": -2.266545057296753,
|
|
"logits/rejected": -2.2732253074645996,
|
|
"logps/chosen": -0.2827457785606384,
|
|
"logps/rejected": -0.31045886874198914,
|
|
"loss": 0.1818530261516571,
|
|
"loss/core": 0.1818530261516571,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.8441836833953857,
|
|
"rewards/margins": 1.116914987564087,
|
|
"rewards/rejected": 0.7272688150405884,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.2245312313388272,
|
|
"grad_norm": 21.0,
|
|
"learning_rate": 4.6201202403910643e-07,
|
|
"logits/chosen": -2.0017685890197754,
|
|
"logits/rejected": -2.0373404026031494,
|
|
"logps/chosen": -0.29275330901145935,
|
|
"logps/rejected": -0.2876564562320709,
|
|
"loss": 0.45062655210494995,
|
|
"loss/core": 0.45062655210494995,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.9036191701889038,
|
|
"rewards/margins": 0.5858906507492065,
|
|
"rewards/rejected": 1.3177284002304077,
|
|
"step": 235
|
|
},
|
|
{
|
|
"epoch": 0.22930849158007882,
|
|
"grad_norm": 11.125,
|
|
"learning_rate": 4.594031994470573e-07,
|
|
"logits/chosen": -2.016340732574463,
|
|
"logits/rejected": -2.035101890563965,
|
|
"logps/chosen": -0.29441291093826294,
|
|
"logps/rejected": -0.29516950249671936,
|
|
"loss": 0.14202021062374115,
|
|
"loss/core": 0.14202021062374115,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.7591701745986938,
|
|
"rewards/margins": 0.7430254220962524,
|
|
"rewards/rejected": 1.0161445140838623,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.23408575182133046,
|
|
"grad_norm": 300.0,
|
|
"learning_rate": 4.567156268756593e-07,
|
|
"logits/chosen": -2.0561702251434326,
|
|
"logits/rejected": -2.043266773223877,
|
|
"logps/chosen": -0.2777637839317322,
|
|
"logps/rejected": -0.2947208285331726,
|
|
"loss": 1.193971037864685,
|
|
"loss/core": 1.193971037864685,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.6019701957702637,
|
|
"rewards/margins": 2.486217975616455,
|
|
"rewards/rejected": 1.1157519817352295,
|
|
"step": 245
|
|
},
|
|
{
|
|
"epoch": 0.23886301206258212,
|
|
"grad_norm": 21.5,
|
|
"learning_rate": 4.5395031701104303e-07,
|
|
"logits/chosen": -1.8958946466445923,
|
|
"logits/rejected": -1.9753758907318115,
|
|
"logps/chosen": -0.30067577958106995,
|
|
"logps/rejected": -0.295326292514801,
|
|
"loss": 0.28905177116394043,
|
|
"loss/core": 0.28905177116394043,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.3306000232696533,
|
|
"rewards/margins": 1.2696471214294434,
|
|
"rewards/rejected": 1.0609527826309204,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.24364027230383375,
|
|
"grad_norm": 12.875,
|
|
"learning_rate": 4.511083097731555e-07,
|
|
"logits/chosen": -2.2028048038482666,
|
|
"logits/rejected": -2.1979146003723145,
|
|
"logps/chosen": -0.2854974865913391,
|
|
"logps/rejected": -0.2871183156967163,
|
|
"loss": 0.6217617392539978,
|
|
"loss/core": 0.6217617392539978,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.4457859992980957,
|
|
"rewards/margins": 0.8392683863639832,
|
|
"rewards/rejected": 1.6065175533294678,
|
|
"step": 255
|
|
},
|
|
{
|
|
"epoch": 0.2484175325450854,
|
|
"grad_norm": 0.78515625,
|
|
"learning_rate": 4.481906739246894e-07,
|
|
"logits/chosen": -2.1431796550750732,
|
|
"logits/rejected": -2.214958906173706,
|
|
"logps/chosen": -0.28957638144493103,
|
|
"logps/rejected": -0.27189141511917114,
|
|
"loss": 0.5298699736595154,
|
|
"loss/core": 0.5298699736595154,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.3812990188598633,
|
|
"rewards/margins": 0.8128549456596375,
|
|
"rewards/rejected": 1.568444013595581,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.25319479278633705,
|
|
"grad_norm": 1.1796875,
|
|
"learning_rate": 4.451985066691648e-07,
|
|
"logits/chosen": -2.025351047515869,
|
|
"logits/rejected": -2.0318732261657715,
|
|
"logps/chosen": -0.3007538318634033,
|
|
"logps/rejected": -0.305298388004303,
|
|
"loss": 0.405862033367157,
|
|
"loss/core": 0.405862033367157,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.2176848649978638,
|
|
"rewards/margins": 0.080999456346035,
|
|
"rewards/rejected": 1.1366853713989258,
|
|
"step": 265
|
|
},
|
|
{
|
|
"epoch": 0.2579720530275887,
|
|
"grad_norm": 12.4375,
|
|
"learning_rate": 4.421329332383158e-07,
|
|
"logits/chosen": -1.928206205368042,
|
|
"logits/rejected": -2.016925096511841,
|
|
"logps/chosen": -0.2899303734302521,
|
|
"logps/rejected": -0.29195359349250793,
|
|
"loss": 1.1587004661560059,
|
|
"loss/core": 1.1587004661560059,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.2669448852539062,
|
|
"rewards/margins": 1.1529427766799927,
|
|
"rewards/rejected": 2.114002227783203,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.2627493132688403,
|
|
"grad_norm": 68.0,
|
|
"learning_rate": 4.3899510646893696e-07,
|
|
"logits/chosen": -2.078123092651367,
|
|
"logits/rejected": -2.032747745513916,
|
|
"logps/chosen": -0.2979772388935089,
|
|
"logps/rejected": -0.3040463328361511,
|
|
"loss": 0.45543140172958374,
|
|
"loss/core": 0.45543140172958374,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.310035467147827,
|
|
"rewards/margins": 1.1866633892059326,
|
|
"rewards/rejected": 1.123372197151184,
|
|
"step": 275
|
|
},
|
|
{
|
|
"epoch": 0.26752657351009196,
|
|
"grad_norm": 4.03125,
|
|
"learning_rate": 4.357862063693485e-07,
|
|
"logits/chosen": -2.0485012531280518,
|
|
"logits/rejected": -1.9654176235198975,
|
|
"logps/chosen": -0.28849276900291443,
|
|
"logps/rejected": -0.3105738162994385,
|
|
"loss": 0.21645864844322205,
|
|
"loss/core": 0.21645864844322205,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.0423591136932373,
|
|
"rewards/margins": 1.3129526376724243,
|
|
"rewards/rejected": 0.7294065952301025,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.2723038337513436,
|
|
"grad_norm": 244.0,
|
|
"learning_rate": 4.3250743967564364e-07,
|
|
"logits/chosen": -2.072444438934326,
|
|
"logits/rejected": -2.069185733795166,
|
|
"logps/chosen": -0.29705238342285156,
|
|
"logps/rejected": -0.29413706064224243,
|
|
"loss": 0.5292507410049438,
|
|
"loss/core": 0.5292507410049438,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.6063647270202637,
|
|
"rewards/margins": 0.9912127256393433,
|
|
"rewards/rejected": 1.6151517629623413,
|
|
"step": 285
|
|
},
|
|
{
|
|
"epoch": 0.27708109399259523,
|
|
"grad_norm": 668.0,
|
|
"learning_rate": 4.29160039397884e-07,
|
|
"logits/chosen": -2.148799419403076,
|
|
"logits/rejected": -2.1607887744903564,
|
|
"logps/chosen": -0.2877974808216095,
|
|
"logps/rejected": -0.2921397387981415,
|
|
"loss": 0.3775253891944885,
|
|
"loss/core": 0.3775253891944885,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.4691474437713623,
|
|
"rewards/margins": 0.05996546894311905,
|
|
"rewards/rejected": 1.4091819524765015,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.28185835423384686,
|
|
"grad_norm": 8.125,
|
|
"learning_rate": 4.2574526435641546e-07,
|
|
"logits/chosen": -2.5482234954833984,
|
|
"logits/rejected": -2.5131990909576416,
|
|
"logps/chosen": -0.24539580941200256,
|
|
"logps/rejected": -0.2573974132537842,
|
|
"loss": 0.05928546190261841,
|
|
"loss/core": 0.05928546190261841,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.3013224601745605,
|
|
"rewards/margins": 0.635219931602478,
|
|
"rewards/rejected": 0.6661025285720825,
|
|
"step": 295
|
|
},
|
|
{
|
|
"epoch": 0.28663561447509855,
|
|
"grad_norm": 142.0,
|
|
"learning_rate": 4.22264398708477e-07,
|
|
"logits/chosen": -1.9641075134277344,
|
|
"logits/rejected": -1.9324430227279663,
|
|
"logps/chosen": -0.3132562041282654,
|
|
"logps/rejected": -0.3241512179374695,
|
|
"loss": 0.9757488369941711,
|
|
"loss/core": 0.9757488369941711,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.715285539627075,
|
|
"rewards/margins": 2.727508306503296,
|
|
"rewards/rejected": 0.9877773523330688,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.2914128747163502,
|
|
"grad_norm": 12.0,
|
|
"learning_rate": 4.187187514652819e-07,
|
|
"logits/chosen": -2.102832794189453,
|
|
"logits/rejected": -2.1649816036224365,
|
|
"logps/chosen": -0.269456148147583,
|
|
"logps/rejected": -0.26115578413009644,
|
|
"loss": 0.22697356343269348,
|
|
"loss/core": 0.22697356343269348,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.1590380668640137,
|
|
"rewards/margins": 1.2657159566879272,
|
|
"rewards/rejected": 0.8933222889900208,
|
|
"step": 305
|
|
},
|
|
{
|
|
"epoch": 0.2961901349576018,
|
|
"grad_norm": 132.0,
|
|
"learning_rate": 4.151096559997519e-07,
|
|
"logits/chosen": -2.111849308013916,
|
|
"logits/rejected": -2.0225253105163574,
|
|
"logps/chosen": -0.2739498019218445,
|
|
"logps/rejected": -0.28690317273139954,
|
|
"loss": 0.9037171602249146,
|
|
"loss/core": 0.9037171602249146,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.6031503677368164,
|
|
"rewards/margins": 0.6503016352653503,
|
|
"rewards/rejected": 1.9528486728668213,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.30096739519885346,
|
|
"grad_norm": 5.84375,
|
|
"learning_rate": 4.114384695450905e-07,
|
|
"logits/chosen": -1.9431352615356445,
|
|
"logits/rejected": -2.0480642318725586,
|
|
"logps/chosen": -0.2702833116054535,
|
|
"logps/rejected": -0.27680981159210205,
|
|
"loss": 0.45981135964393616,
|
|
"loss/core": 0.45981135964393616,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.901813268661499,
|
|
"rewards/margins": 1.5673255920410156,
|
|
"rewards/rejected": 1.3344876766204834,
|
|
"step": 315
|
|
},
|
|
{
|
|
"epoch": 0.3057446554401051,
|
|
"grad_norm": 3.578125,
|
|
"learning_rate": 4.077065726843828e-07,
|
|
"logits/chosen": -1.9837052822113037,
|
|
"logits/rejected": -2.000582456588745,
|
|
"logps/chosen": -0.2918964922428131,
|
|
"logps/rejected": -0.29292041063308716,
|
|
"loss": 0.19344636797904968,
|
|
"loss/core": 0.19344636797904968,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.5632867813110352,
|
|
"rewards/margins": 0.3376685380935669,
|
|
"rewards/rejected": 1.2256181240081787,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.31052191568135673,
|
|
"grad_norm": 556.0,
|
|
"learning_rate": 4.039153688314145e-07,
|
|
"logits/chosen": -2.2357137203216553,
|
|
"logits/rejected": -2.1825029850006104,
|
|
"logps/chosen": -0.28656476736068726,
|
|
"logps/rejected": -0.28232595324516296,
|
|
"loss": 1.8139476776123047,
|
|
"loss/core": 1.8139476776123047,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.6962344646453857,
|
|
"rewards/margins": 2.1483452320098877,
|
|
"rewards/rejected": 1.5478893518447876,
|
|
"step": 325
|
|
},
|
|
{
|
|
"epoch": 0.31529917592260837,
|
|
"grad_norm": 2.640625,
|
|
"learning_rate": 4.0006628370290613e-07,
|
|
"logits/chosen": -2.2076706886291504,
|
|
"logits/rejected": -2.196662664413452,
|
|
"logps/chosen": -0.282164603471756,
|
|
"logps/rejected": -0.27122125029563904,
|
|
"loss": 0.2774432897567749,
|
|
"loss/core": 0.2774432897567749,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.995111107826233,
|
|
"rewards/margins": 0.8708856701850891,
|
|
"rewards/rejected": 1.12422513961792,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.32007643616386,
|
|
"grad_norm": 344.0,
|
|
"learning_rate": 3.9616076478235826e-07,
|
|
"logits/chosen": -2.0309836864471436,
|
|
"logits/rejected": -2.014965057373047,
|
|
"logps/chosen": -0.30268004536628723,
|
|
"logps/rejected": -0.32926321029663086,
|
|
"loss": 0.4523457884788513,
|
|
"loss/core": 0.4523457884788513,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.116103410720825,
|
|
"rewards/margins": 1.5347257852554321,
|
|
"rewards/rejected": 0.5813775658607483,
|
|
"step": 335
|
|
},
|
|
{
|
|
"epoch": 0.3248536964051117,
|
|
"grad_norm": 132.0,
|
|
"learning_rate": 3.922002807757129e-07,
|
|
"logits/chosen": -2.0748965740203857,
|
|
"logits/rejected": -2.196530342102051,
|
|
"logps/chosen": -0.28026294708251953,
|
|
"logps/rejected": -0.2833474576473236,
|
|
"loss": 0.31017398834228516,
|
|
"loss/core": 0.31017398834228516,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.0382297039031982,
|
|
"rewards/margins": 1.1847279071807861,
|
|
"rewards/rejected": 0.8535016775131226,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.3296309566463633,
|
|
"grad_norm": 8.625,
|
|
"learning_rate": 3.8818632105903315e-07,
|
|
"logits/chosen": -2.299494981765747,
|
|
"logits/rejected": -2.3579187393188477,
|
|
"logps/chosen": -0.28255495429039,
|
|
"logps/rejected": -0.2946614921092987,
|
|
"loss": 0.1961742490530014,
|
|
"loss/core": 0.1961742490530014,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.639917016029358,
|
|
"rewards/margins": 0.8168700933456421,
|
|
"rewards/rejected": 0.823046863079071,
|
|
"step": 345
|
|
},
|
|
{
|
|
"epoch": 0.33440821688761496,
|
|
"grad_norm": 7.625,
|
|
"learning_rate": 3.841203951184094e-07,
|
|
"logits/chosen": -2.0392203330993652,
|
|
"logits/rejected": -2.0187597274780273,
|
|
"logps/chosen": -0.261064350605011,
|
|
"logps/rejected": -0.25779426097869873,
|
|
"loss": 0.20222067832946777,
|
|
"loss/core": 0.20222067832946777,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.9913139343261719,
|
|
"rewards/margins": 1.0575062036514282,
|
|
"rewards/rejected": 0.9338076710700989,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.3391854771288666,
|
|
"grad_norm": 9.625,
|
|
"learning_rate": 3.800040319823038e-07,
|
|
"logits/chosen": -2.088989019393921,
|
|
"logits/rejected": -2.2161617279052734,
|
|
"logps/chosen": -0.2810487449169159,
|
|
"logps/rejected": -0.2965686023235321,
|
|
"loss": 0.40663355588912964,
|
|
"loss/core": 0.40663355588912964,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.8474032878875732,
|
|
"rewards/margins": 0.5359752774238586,
|
|
"rewards/rejected": 1.3114280700683594,
|
|
"step": 355
|
|
},
|
|
{
|
|
"epoch": 0.34396273737011823,
|
|
"grad_norm": 149.0,
|
|
"learning_rate": 3.75838779646545e-07,
|
|
"logits/chosen": -2.110039234161377,
|
|
"logits/rejected": -2.1716244220733643,
|
|
"logps/chosen": -0.2803703844547272,
|
|
"logps/rejected": -0.2838951647281647,
|
|
"loss": 0.621938169002533,
|
|
"loss/core": 0.621938169002533,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.6312050819396973,
|
|
"rewards/margins": 1.3762609958648682,
|
|
"rewards/rejected": 1.2549439668655396,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.34873999761136987,
|
|
"grad_norm": 474.0,
|
|
"learning_rate": 3.7162620449218993e-07,
|
|
"logits/chosen": -2.048468589782715,
|
|
"logits/rejected": -2.1561145782470703,
|
|
"logps/chosen": -0.287244588136673,
|
|
"logps/rejected": -0.28558677434921265,
|
|
"loss": 0.9522292017936707,
|
|
"loss/core": 0.9522292017936707,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.8516860008239746,
|
|
"rewards/margins": 2.0592613220214844,
|
|
"rewards/rejected": 0.7924247980117798,
|
|
"step": 365
|
|
},
|
|
{
|
|
"epoch": 0.3535172578526215,
|
|
"grad_norm": 21.25,
|
|
"learning_rate": 3.673678906964727e-07,
|
|
"logits/chosen": -2.246171474456787,
|
|
"logits/rejected": -2.291537046432495,
|
|
"logps/chosen": -0.28519970178604126,
|
|
"logps/rejected": -0.2871125638484955,
|
|
"loss": 0.17473874986171722,
|
|
"loss/core": 0.17473874986171722,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.6407058238983154,
|
|
"rewards/margins": 0.5970925688743591,
|
|
"rewards/rejected": 1.0436131954193115,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.35829451809387314,
|
|
"grad_norm": 7.9375,
|
|
"learning_rate": 3.6306543963705936e-07,
|
|
"logits/chosen": -2.027927875518799,
|
|
"logits/rejected": -2.0061910152435303,
|
|
"logps/chosen": -0.2594534456729889,
|
|
"logps/rejected": -0.2606300711631775,
|
|
"loss": 0.8009087443351746,
|
|
"loss/core": 0.8009087443351746,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.0067553520202637,
|
|
"rewards/margins": 0.8116471171379089,
|
|
"rewards/rejected": 2.195107936859131,
|
|
"step": 375
|
|
},
|
|
{
|
|
"epoch": 0.36307177833512483,
|
|
"grad_norm": 13.1875,
|
|
"learning_rate": 3.5872046928983623e-07,
|
|
"logits/chosen": -2.099975109100342,
|
|
"logits/rejected": -2.1087899208068848,
|
|
"logps/chosen": -0.2818046808242798,
|
|
"logps/rejected": -0.29659533500671387,
|
|
"loss": 0.7302302122116089,
|
|
"loss/core": 0.7302302122116089,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.033167839050293,
|
|
"rewards/margins": 0.5039272904396057,
|
|
"rewards/rejected": 1.529240608215332,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.36784903857637646,
|
|
"grad_norm": 29.375,
|
|
"learning_rate": 3.5433461362045447e-07,
|
|
"logits/chosen": -2.0900938510894775,
|
|
"logits/rejected": -2.1108756065368652,
|
|
"logps/chosen": -0.2719820439815521,
|
|
"logps/rejected": -0.2751936912536621,
|
|
"loss": 0.3768286406993866,
|
|
"loss/core": 0.3768286406993866,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.160437822341919,
|
|
"rewards/margins": 0.6413952112197876,
|
|
"rewards/rejected": 1.5190424919128418,
|
|
"step": 385
|
|
},
|
|
{
|
|
"epoch": 0.3726262988176281,
|
|
"grad_norm": 1.953125,
|
|
"learning_rate": 3.4990952196986305e-07,
|
|
"logits/chosen": -1.8510644435882568,
|
|
"logits/rejected": -1.9279950857162476,
|
|
"logps/chosen": -0.3081986606121063,
|
|
"logps/rejected": -0.30626800656318665,
|
|
"loss": 0.1976659595966339,
|
|
"loss/core": 0.1976659595966339,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.7517478466033936,
|
|
"rewards/margins": 0.8936412930488586,
|
|
"rewards/rejected": 0.8581066131591797,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.37740355905887973,
|
|
"grad_norm": 19.75,
|
|
"learning_rate": 3.4544685843405875e-07,
|
|
"logits/chosen": -2.1983537673950195,
|
|
"logits/rejected": -2.3012776374816895,
|
|
"logps/chosen": -0.2957797050476074,
|
|
"logps/rejected": -0.2903483510017395,
|
|
"loss": 0.1597086489200592,
|
|
"loss/core": 0.1597086489200592,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 1.6743053197860718,
|
|
"rewards/margins": 0.8747150301933289,
|
|
"rewards/rejected": 0.7995903491973877,
|
|
"step": 395
|
|
},
|
|
{
|
|
"epoch": 0.38218081930013137,
|
|
"grad_norm": 50.5,
|
|
"learning_rate": 3.4094830123828786e-07,
|
|
"logits/chosen": -2.0785083770751953,
|
|
"logits/rejected": -2.1138789653778076,
|
|
"logps/chosen": -0.2923468351364136,
|
|
"logps/rejected": -0.29322344064712524,
|
|
"loss": 1.0465948581695557,
|
|
"loss/core": 1.0465948581695557,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.9298787117004395,
|
|
"rewards/margins": 2.0894033908843994,
|
|
"rewards/rejected": 0.8404749631881714,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.386958079541383,
|
|
"grad_norm": 5.90625,
|
|
"learning_rate": 3.3641554210593414e-07,
|
|
"logits/chosen": -2.1093640327453613,
|
|
"logits/rejected": -2.1642727851867676,
|
|
"logps/chosen": -0.31529492139816284,
|
|
"logps/rejected": -0.30353349447250366,
|
|
"loss": 0.6650071740150452,
|
|
"loss/core": 0.6650071740150452,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.3120980262756348,
|
|
"rewards/margins": 1.074296236038208,
|
|
"rewards/rejected": 1.2378015518188477,
|
|
"step": 405
|
|
},
|
|
{
|
|
"epoch": 0.39173533978263464,
|
|
"grad_norm": 20.125,
|
|
"learning_rate": 3.3185028562233107e-07,
|
|
"logits/chosen": -2.227759838104248,
|
|
"logits/rejected": -2.2588248252868652,
|
|
"logps/chosen": -0.277637243270874,
|
|
"logps/rejected": -0.2886066436767578,
|
|
"loss": 0.27694252133369446,
|
|
"loss/core": 0.27694252133369446,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.081727981567383,
|
|
"rewards/margins": 0.8987264633178711,
|
|
"rewards/rejected": 1.1830016374588013,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.3965126000238863,
|
|
"grad_norm": 12.625,
|
|
"learning_rate": 3.272542485937368e-07,
|
|
"logits/chosen": -2.0978333950042725,
|
|
"logits/rejected": -2.175910234451294,
|
|
"logps/chosen": -0.3309980034828186,
|
|
"logps/rejected": -0.32166236639022827,
|
|
"loss": 0.05876865237951279,
|
|
"loss/core": 0.05876865237951279,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 0.988106906414032,
|
|
"rewards/margins": 0.6192494630813599,
|
|
"rewards/rejected": 0.36885741353034973,
|
|
"step": 415
|
|
},
|
|
{
|
|
"epoch": 0.40128986026513797,
|
|
"grad_norm": 88.5,
|
|
"learning_rate": 3.226291594017137e-07,
|
|
"logits/chosen": -2.326503038406372,
|
|
"logits/rejected": -2.2930359840393066,
|
|
"logps/chosen": -0.28376084566116333,
|
|
"logps/rejected": -0.3000229001045227,
|
|
"loss": 0.3358566462993622,
|
|
"loss/core": 0.3358566462993622,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.0413100719451904,
|
|
"rewards/margins": 1.3324414491653442,
|
|
"rewards/rejected": 0.7088686227798462,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.4060671205063896,
|
|
"grad_norm": 196.0,
|
|
"learning_rate": 3.1797675735315454e-07,
|
|
"logits/chosen": -1.9907684326171875,
|
|
"logits/rejected": -2.05031156539917,
|
|
"logps/chosen": -0.2916226387023926,
|
|
"logps/rejected": -0.2697221636772156,
|
|
"loss": 0.768527626991272,
|
|
"loss/core": 0.768527626991272,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.2350106239318848,
|
|
"rewards/margins": 0.6351408958435059,
|
|
"rewards/rejected": 1.5998692512512207,
|
|
"step": 425
|
|
},
|
|
{
|
|
"epoch": 0.41084438074764124,
|
|
"grad_norm": 732.0,
|
|
"learning_rate": 3.1329879202620047e-07,
|
|
"logits/chosen": -2.0472471714019775,
|
|
"logits/rejected": -2.0714943408966064,
|
|
"logps/chosen": -0.28120359778404236,
|
|
"logps/rejected": -0.2685726583003998,
|
|
"loss": 1.8000812530517578,
|
|
"loss/core": 1.8000812530517578,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 4.386286735534668,
|
|
"rewards/margins": 2.987924814224243,
|
|
"rewards/rejected": 1.3983619213104248,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.41562164098889287,
|
|
"grad_norm": 8.8125,
|
|
"learning_rate": 3.0859702261229613e-07,
|
|
"logits/chosen": -2.324946165084839,
|
|
"logits/rejected": -2.3079473972320557,
|
|
"logps/chosen": -0.27902793884277344,
|
|
"logps/rejected": -0.29347580671310425,
|
|
"loss": 0.2374797761440277,
|
|
"loss/core": 0.2374797761440277,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.617841362953186,
|
|
"rewards/margins": 0.6150254011154175,
|
|
"rewards/rejected": 1.0028159618377686,
|
|
"step": 435
|
|
},
|
|
{
|
|
"epoch": 0.4203989012301445,
|
|
"grad_norm": 484.0,
|
|
"learning_rate": 3.0387321725463e-07,
|
|
"logits/chosen": -2.0346179008483887,
|
|
"logits/rejected": -2.105949878692627,
|
|
"logps/chosen": -0.3050372004508972,
|
|
"logps/rejected": -0.304879367351532,
|
|
"loss": 0.9780192375183105,
|
|
"loss/core": 0.9780192375183105,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.988939046859741,
|
|
"rewards/margins": 1.6511949300765991,
|
|
"rewards/rejected": 1.3377439975738525,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.42517616147139614,
|
|
"grad_norm": 41.0,
|
|
"learning_rate": 2.991291523832075e-07,
|
|
"logits/chosen": -1.9136848449707031,
|
|
"logits/rejected": -1.8891462087631226,
|
|
"logps/chosen": -0.29649606347084045,
|
|
"logps/rejected": -0.2987465262413025,
|
|
"loss": 0.5171946287155151,
|
|
"loss/core": 0.5171946287155151,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.9516830444335938,
|
|
"rewards/margins": 1.4048423767089844,
|
|
"rewards/rejected": 1.5468409061431885,
|
|
"step": 445
|
|
},
|
|
{
|
|
"epoch": 0.4299534217126478,
|
|
"grad_norm": 418.0,
|
|
"learning_rate": 2.943666120468088e-07,
|
|
"logits/chosen": -1.9518722295761108,
|
|
"logits/rejected": -1.9711889028549194,
|
|
"logps/chosen": -0.3000376522541046,
|
|
"logps/rejected": -0.29597753286361694,
|
|
"loss": 0.8665231466293335,
|
|
"loss/core": 0.8665231466293335,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.440600872039795,
|
|
"rewards/margins": 1.8995399475097656,
|
|
"rewards/rejected": 1.5410608053207397,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.4347306819538994,
|
|
"grad_norm": 183.0,
|
|
"learning_rate": 2.8958738724208073e-07,
|
|
"logits/chosen": -1.9178097248077393,
|
|
"logits/rejected": -2.1076836585998535,
|
|
"logps/chosen": -0.3262283205986023,
|
|
"logps/rejected": -0.3135696053504944,
|
|
"loss": 1.426609754562378,
|
|
"loss/core": 1.426609754562378,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.4305222034454346,
|
|
"rewards/margins": 1.9480667114257812,
|
|
"rewards/rejected": 1.4824554920196533,
|
|
"step": 455
|
|
},
|
|
{
|
|
"epoch": 0.4395079421951511,
|
|
"grad_norm": 30.75,
|
|
"learning_rate": 2.8479327524001633e-07,
|
|
"logits/chosen": -2.036893606185913,
|
|
"logits/rejected": -2.0251917839050293,
|
|
"logps/chosen": -0.2996273636817932,
|
|
"logps/rejected": -0.30763858556747437,
|
|
"loss": 1.1361322402954102,
|
|
"loss/core": 1.1361322402954102,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.628493547439575,
|
|
"rewards/margins": 1.7698605060577393,
|
|
"rewards/rejected": 1.858633041381836,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.44428520243640274,
|
|
"grad_norm": 10.3125,
|
|
"learning_rate": 2.7998607891007493e-07,
|
|
"logits/chosen": -1.8706251382827759,
|
|
"logits/rejected": -1.9697706699371338,
|
|
"logps/chosen": -0.2975318431854248,
|
|
"logps/rejected": -0.2920708954334259,
|
|
"loss": 0.23389527201652527,
|
|
"loss/core": 0.23389527201652527,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.297299861907959,
|
|
"rewards/margins": 1.371745228767395,
|
|
"rewards/rejected": 0.9255548715591431,
|
|
"step": 465
|
|
},
|
|
{
|
|
"epoch": 0.4490624626776544,
|
|
"grad_norm": 1048.0,
|
|
"learning_rate": 2.7516760604219616e-07,
|
|
"logits/chosen": -2.105297803878784,
|
|
"logits/rejected": -2.173494815826416,
|
|
"logps/chosen": -0.28127214312553406,
|
|
"logps/rejected": -0.2611326277256012,
|
|
"loss": 1.1431621313095093,
|
|
"loss/core": 1.1431621313095093,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.987865447998047,
|
|
"rewards/margins": 1.8296188116073608,
|
|
"rewards/rejected": 1.1582467555999756,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.453839722918906,
|
|
"grad_norm": 90.5,
|
|
"learning_rate": 2.703396686669646e-07,
|
|
"logits/chosen": -2.0695085525512695,
|
|
"logits/rejected": -2.0106379985809326,
|
|
"logps/chosen": -0.2961866855621338,
|
|
"logps/rejected": -0.32303735613822937,
|
|
"loss": 0.46794575452804565,
|
|
"loss/core": 0.46794575452804565,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.823664665222168,
|
|
"rewards/margins": 0.37910977005958557,
|
|
"rewards/rejected": 1.4445549249649048,
|
|
"step": 475
|
|
},
|
|
{
|
|
"epoch": 0.45861698316015764,
|
|
"grad_norm": 6.0,
|
|
"learning_rate": 2.6550408237417884e-07,
|
|
"logits/chosen": -2.2378296852111816,
|
|
"logits/rejected": -2.2934558391571045,
|
|
"logps/chosen": -0.2757987380027771,
|
|
"logps/rejected": -0.2735208570957184,
|
|
"loss": 0.5451094508171082,
|
|
"loss/core": 0.5451094508171082,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.354505777359009,
|
|
"rewards/margins": 1.340338945388794,
|
|
"rewards/rejected": 1.0141668319702148,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.4633942434014093,
|
|
"grad_norm": 13.25,
|
|
"learning_rate": 2.6066266563008265e-07,
|
|
"logits/chosen": -2.183929920196533,
|
|
"logits/rejected": -2.191530704498291,
|
|
"logps/chosen": -0.2924697995185852,
|
|
"logps/rejected": -0.29630663990974426,
|
|
"loss": 0.15576469898223877,
|
|
"loss/core": 0.15576469898223877,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.7340059280395508,
|
|
"rewards/margins": 0.8523953557014465,
|
|
"rewards/rejected": 0.8816105127334595,
|
|
"step": 485
|
|
},
|
|
{
|
|
"epoch": 0.4681715036426609,
|
|
"grad_norm": 20.0,
|
|
"learning_rate": 2.5581723909351404e-07,
|
|
"logits/chosen": -2.13179087638855,
|
|
"logits/rejected": -2.1726982593536377,
|
|
"logps/chosen": -0.3071516156196594,
|
|
"logps/rejected": -0.29533272981643677,
|
|
"loss": 0.29640907049179077,
|
|
"loss/core": 0.29640907049179077,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.0775198936462402,
|
|
"rewards/margins": 1.033736228942871,
|
|
"rewards/rejected": 1.0437836647033691,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.47294876388391255,
|
|
"grad_norm": 26.125,
|
|
"learning_rate": 2.509696249312301e-07,
|
|
"logits/chosen": -2.2027294635772705,
|
|
"logits/rejected": -2.2974157333374023,
|
|
"logps/chosen": -0.2999098598957062,
|
|
"logps/rejected": -0.2914625406265259,
|
|
"loss": 0.7030426263809204,
|
|
"loss/core": 0.7030426263809204,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.590477228164673,
|
|
"rewards/margins": 1.704890251159668,
|
|
"rewards/rejected": 0.8855869174003601,
|
|
"step": 495
|
|
},
|
|
{
|
|
"epoch": 0.47772602412516424,
|
|
"grad_norm": 133.0,
|
|
"learning_rate": 2.461216461326642e-07,
|
|
"logits/chosen": -1.9994663000106812,
|
|
"logits/rejected": -1.9483753442764282,
|
|
"logps/chosen": -0.28346696496009827,
|
|
"logps/rejected": -0.30574333667755127,
|
|
"loss": 2.2656641006469727,
|
|
"loss/core": 2.2656641006469727,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.139856815338135,
|
|
"rewards/margins": 2.1449146270751953,
|
|
"rewards/rejected": 1.994942307472229,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.4825032843664159,
|
|
"grad_norm": 348.0,
|
|
"learning_rate": 2.412751258243748e-07,
|
|
"logits/chosen": -2.1276748180389404,
|
|
"logits/rejected": -2.1600797176361084,
|
|
"logps/chosen": -0.2883078157901764,
|
|
"logps/rejected": -0.29086965322494507,
|
|
"loss": 1.4473992586135864,
|
|
"loss/core": 1.4473992586135864,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.5834662914276123,
|
|
"rewards/margins": 1.8612127304077148,
|
|
"rewards/rejected": 1.7222535610198975,
|
|
"step": 505
|
|
},
|
|
{
|
|
"epoch": 0.4872805446076675,
|
|
"grad_norm": 0.2890625,
|
|
"learning_rate": 2.3643188658444158e-07,
|
|
"logits/chosen": -2.127025604248047,
|
|
"logits/rejected": -2.259706735610962,
|
|
"logps/chosen": -0.31721481680870056,
|
|
"logps/rejected": -0.29182881116867065,
|
|
"loss": 0.41495251655578613,
|
|
"loss/core": 0.41495251655578613,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.9977633953094482,
|
|
"rewards/margins": 1.2025401592254639,
|
|
"rewards/rejected": 0.7952233552932739,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.49205780484891914,
|
|
"grad_norm": 35.75,
|
|
"learning_rate": 2.315937497570688e-07,
|
|
"logits/chosen": -2.010084629058838,
|
|
"logits/rejected": -2.0125226974487305,
|
|
"logps/chosen": -0.2837277054786682,
|
|
"logps/rejected": -0.3070703148841858,
|
|
"loss": 0.43133026361465454,
|
|
"loss/core": 0.43133026361465454,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.5341956615448,
|
|
"rewards/margins": 0.8866618275642395,
|
|
"rewards/rejected": 1.6475337743759155,
|
|
"step": 515
|
|
},
|
|
{
|
|
"epoch": 0.4968350650901708,
|
|
"grad_norm": 40.5,
|
|
"learning_rate": 2.2676253476765194e-07,
|
|
"logits/chosen": -2.039220094680786,
|
|
"logits/rejected": -2.2076566219329834,
|
|
"logps/chosen": -0.32395556569099426,
|
|
"logps/rejected": -0.33448874950408936,
|
|
"loss": 0.802078366279602,
|
|
"loss/core": 0.802078366279602,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.054922580718994,
|
|
"rewards/margins": 2.0955018997192383,
|
|
"rewards/rejected": 0.9594209790229797,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.5016123253314224,
|
|
"grad_norm": 1288.0,
|
|
"learning_rate": 2.2194005843856633e-07,
|
|
"logits/chosen": -2.1711580753326416,
|
|
"logits/rejected": -2.1403403282165527,
|
|
"logps/chosen": -0.3041841983795166,
|
|
"logps/rejected": -0.3024327754974365,
|
|
"loss": 1.989890456199646,
|
|
"loss/core": 1.989890456199646,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 4.269538402557373,
|
|
"rewards/margins": 2.8283638954162598,
|
|
"rewards/rejected": 1.441174864768982,
|
|
"step": 525
|
|
},
|
|
{
|
|
"epoch": 0.5063895855726741,
|
|
"grad_norm": 0.408203125,
|
|
"learning_rate": 2.1712813430593434e-07,
|
|
"logits/chosen": -2.016860246658325,
|
|
"logits/rejected": -1.9806495904922485,
|
|
"logps/chosen": -0.2869611382484436,
|
|
"logps/rejected": -0.2894926369190216,
|
|
"loss": 0.7920265793800354,
|
|
"loss/core": 0.7920265793800354,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.041126251220703,
|
|
"rewards/margins": 1.5607634782791138,
|
|
"rewards/rejected": 1.4803630113601685,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.5111668458139257,
|
|
"grad_norm": 644.0,
|
|
"learning_rate": 2.123285719376292e-07,
|
|
"logits/chosen": -2.1647980213165283,
|
|
"logits/rejected": -2.086789846420288,
|
|
"logps/chosen": -0.2868487536907196,
|
|
"logps/rejected": -0.28134408593177795,
|
|
"loss": 0.9000055193901062,
|
|
"loss/core": 0.9000055193901062,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.0810141563415527,
|
|
"rewards/margins": 2.0247347354888916,
|
|
"rewards/rejected": 1.056279182434082,
|
|
"step": 535
|
|
},
|
|
{
|
|
"epoch": 0.5159441060551774,
|
|
"grad_norm": 1.453125,
|
|
"learning_rate": 2.0754317625276982e-07,
|
|
"logits/chosen": -2.085092067718506,
|
|
"logits/rejected": -2.064635992050171,
|
|
"logps/chosen": -0.2793554961681366,
|
|
"logps/rejected": -0.2893218398094177,
|
|
"loss": 0.3135538399219513,
|
|
"loss/core": 0.3135538399219513,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.5822449922561646,
|
|
"rewards/margins": 0.09157203137874603,
|
|
"rewards/rejected": 1.4906729459762573,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.520721366296429,
|
|
"grad_norm": 1128.0,
|
|
"learning_rate": 2.0277374684296498e-07,
|
|
"logits/chosen": -2.306612730026245,
|
|
"logits/rejected": -2.1498613357543945,
|
|
"logps/chosen": -0.2859051823616028,
|
|
"logps/rejected": -0.29374048113822937,
|
|
"loss": 0.9199239611625671,
|
|
"loss/core": 0.9199239611625671,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.6144795417785645,
|
|
"rewards/margins": 1.7432453632354736,
|
|
"rewards/rejected": 0.8712340593338013,
|
|
"step": 545
|
|
},
|
|
{
|
|
"epoch": 0.5254986265376806,
|
|
"grad_norm": 12.8125,
|
|
"learning_rate": 1.980220772955602e-07,
|
|
"logits/chosen": -1.8739219903945923,
|
|
"logits/rejected": -1.8925116062164307,
|
|
"logps/chosen": -0.27485448122024536,
|
|
"logps/rejected": -0.3017066717147827,
|
|
"loss": 0.7916240692138672,
|
|
"loss/core": 0.7916240692138672,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.78112530708313,
|
|
"rewards/margins": 1.196991205215454,
|
|
"rewards/rejected": 1.5841339826583862,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.5302758867789323,
|
|
"grad_norm": 5.34375,
|
|
"learning_rate": 1.932899545191433e-07,
|
|
"logits/chosen": -2.0721099376678467,
|
|
"logits/rejected": -2.1279807090759277,
|
|
"logps/chosen": -0.311161607503891,
|
|
"logps/rejected": -0.27461129426956177,
|
|
"loss": 0.827431857585907,
|
|
"loss/core": 0.827431857585907,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.3001389503479004,
|
|
"rewards/margins": 2.3012726306915283,
|
|
"rewards/rejected": 0.998866081237793,
|
|
"step": 555
|
|
},
|
|
{
|
|
"epoch": 0.5350531470201839,
|
|
"grad_norm": 54.25,
|
|
"learning_rate": 1.885791580715609e-07,
|
|
"logits/chosen": -2.157796859741211,
|
|
"logits/rejected": -2.144073009490967,
|
|
"logps/chosen": -0.27682000398635864,
|
|
"logps/rejected": -0.28132444620132446,
|
|
"loss": 0.6787029504776001,
|
|
"loss/core": 0.6787029504776001,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.8043270111083984,
|
|
"rewards/margins": 1.2791929244995117,
|
|
"rewards/rejected": 1.5251343250274658,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.5398304072614356,
|
|
"grad_norm": 21.0,
|
|
"learning_rate": 1.8389145949069951e-07,
|
|
"logits/chosen": -2.05513596534729,
|
|
"logits/rejected": -2.0813965797424316,
|
|
"logps/chosen": -0.28800255060195923,
|
|
"logps/rejected": -0.3045352101325989,
|
|
"loss": 1.3088706731796265,
|
|
"loss/core": 1.3088706731796265,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.775949478149414,
|
|
"rewards/margins": 1.1087366342544556,
|
|
"rewards/rejected": 1.6672130823135376,
|
|
"step": 565
|
|
},
|
|
{
|
|
"epoch": 0.5446076675026872,
|
|
"grad_norm": 66.5,
|
|
"learning_rate": 1.792286216282824e-07,
|
|
"logits/chosen": -2.0531630516052246,
|
|
"logits/rejected": -2.1032092571258545,
|
|
"logps/chosen": -0.2883487045764923,
|
|
"logps/rejected": -0.29088732600212097,
|
|
"loss": 0.902314305305481,
|
|
"loss/core": 0.902314305305481,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.3471977710723877,
|
|
"rewards/margins": 1.6828256845474243,
|
|
"rewards/rejected": 1.6643718481063843,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.5493849277439389,
|
|
"grad_norm": 8.6875,
|
|
"learning_rate": 1.745923979869336e-07,
|
|
"logits/chosen": -2.1334240436553955,
|
|
"logits/rejected": -2.164499044418335,
|
|
"logps/chosen": -0.3011549413204193,
|
|
"logps/rejected": -0.32307350635528564,
|
|
"loss": 0.3328213393688202,
|
|
"loss/core": 0.3328213393688202,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.8378055095672607,
|
|
"rewards/margins": 1.2759101390838623,
|
|
"rewards/rejected": 0.5618952512741089,
|
|
"step": 575
|
|
},
|
|
{
|
|
"epoch": 0.5541621879851905,
|
|
"grad_norm": 0.490234375,
|
|
"learning_rate": 1.6998453206075708e-07,
|
|
"logits/chosen": -2.4017269611358643,
|
|
"logits/rejected": -2.458293914794922,
|
|
"logps/chosen": -0.26788607239723206,
|
|
"logps/rejected": -0.28343814611434937,
|
|
"loss": 0.1603700816631317,
|
|
"loss/core": 0.1603700816631317,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.406388282775879,
|
|
"rewards/margins": 0.4723166823387146,
|
|
"rewards/rejected": 0.9340717196464539,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.5589394482264421,
|
|
"grad_norm": 32.0,
|
|
"learning_rate": 1.6540675667967973e-07,
|
|
"logits/chosen": -2.1147303581237793,
|
|
"logits/rejected": -2.0781502723693848,
|
|
"logps/chosen": -0.2706436514854431,
|
|
"logps/rejected": -0.2854623794555664,
|
|
"loss": 1.4029546976089478,
|
|
"loss/core": 1.4029546976089478,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.21844220161438,
|
|
"rewards/margins": 0.8768823742866516,
|
|
"rewards/rejected": 1.341559648513794,
|
|
"step": 585
|
|
},
|
|
{
|
|
"epoch": 0.5637167084676937,
|
|
"grad_norm": 4.40625,
|
|
"learning_rate": 1.60860793357805e-07,
|
|
"logits/chosen": -2.001035690307617,
|
|
"logits/rejected": -2.096832275390625,
|
|
"logps/chosen": -0.29010075330734253,
|
|
"logps/rejected": -0.2887147068977356,
|
|
"loss": 0.8843881487846375,
|
|
"loss/core": 0.8843881487846375,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.5953457355499268,
|
|
"rewards/margins": 2.580474376678467,
|
|
"rewards/rejected": 1.01487135887146,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.5684939687089454,
|
|
"grad_norm": 3.734375,
|
|
"learning_rate": 1.5634835164602196e-07,
|
|
"logits/chosen": -2.020965099334717,
|
|
"logits/rejected": -2.015770673751831,
|
|
"logps/chosen": -0.27582883834838867,
|
|
"logps/rejected": -0.2730068266391754,
|
|
"loss": 0.1141238808631897,
|
|
"loss/core": 0.1141238808631897,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.7046051025390625,
|
|
"rewards/margins": 0.826572060585022,
|
|
"rewards/rejected": 0.8780330419540405,
|
|
"step": 595
|
|
},
|
|
{
|
|
"epoch": 0.5732712289501971,
|
|
"grad_norm": 24.375,
|
|
"learning_rate": 1.518711284891132e-07,
|
|
"logits/chosen": -2.250802755355835,
|
|
"logits/rejected": -2.2414631843566895,
|
|
"logps/chosen": -0.26670363545417786,
|
|
"logps/rejected": -0.286176860332489,
|
|
"loss": 0.2968599498271942,
|
|
"loss/core": 0.2968599498271942,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.8693134784698486,
|
|
"rewards/margins": 0.8250168561935425,
|
|
"rewards/rejected": 1.0442965030670166,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.5780484891914487,
|
|
"grad_norm": 129.0,
|
|
"learning_rate": 1.47430807587603e-07,
|
|
"logits/chosen": -2.0972466468811035,
|
|
"logits/rejected": -2.1712050437927246,
|
|
"logps/chosen": -0.2813746929168701,
|
|
"logps/rejected": -0.2767219841480255,
|
|
"loss": 1.7541176080703735,
|
|
"loss/core": 1.7541176080703735,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.171405792236328,
|
|
"rewards/margins": 2.975759267807007,
|
|
"rewards/rejected": 1.1956462860107422,
|
|
"step": 605
|
|
},
|
|
{
|
|
"epoch": 0.5828257494327004,
|
|
"grad_norm": 104.0,
|
|
"learning_rate": 1.430290587645865e-07,
|
|
"logits/chosen": -2.0927460193634033,
|
|
"logits/rejected": -2.1099424362182617,
|
|
"logps/chosen": -0.27945631742477417,
|
|
"logps/rejected": -0.27728086709976196,
|
|
"loss": 1.8355001211166382,
|
|
"loss/core": 1.8355001211166382,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 4.239993095397949,
|
|
"rewards/margins": 2.323775291442871,
|
|
"rewards/rejected": 1.9162174463272095,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.587603009673952,
|
|
"grad_norm": 13.0,
|
|
"learning_rate": 1.3866753733777765e-07,
|
|
"logits/chosen": -1.9469194412231445,
|
|
"logits/rejected": -2.012991189956665,
|
|
"logps/chosen": -0.3159197270870209,
|
|
"logps/rejected": -0.3063480257987976,
|
|
"loss": 0.5646291971206665,
|
|
"loss/core": 0.5646291971206665,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.7208328247070312,
|
|
"rewards/margins": 1.2584408521652222,
|
|
"rewards/rejected": 1.4623918533325195,
|
|
"step": 615
|
|
},
|
|
{
|
|
"epoch": 0.5923802699152036,
|
|
"grad_norm": 6.78125,
|
|
"learning_rate": 1.343478834970121e-07,
|
|
"logits/chosen": -2.1200366020202637,
|
|
"logits/rejected": -2.1472256183624268,
|
|
"logps/chosen": -0.286575585603714,
|
|
"logps/rejected": -0.29443609714508057,
|
|
"loss": 0.37433984875679016,
|
|
"loss/core": 0.37433984875679016,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.6850149631500244,
|
|
"rewards/margins": 0.25270089507102966,
|
|
"rewards/rejected": 1.432314157485962,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.5971575301564552,
|
|
"grad_norm": 1520.0,
|
|
"learning_rate": 1.3007172168743852e-07,
|
|
"logits/chosen": -1.9102928638458252,
|
|
"logits/rejected": -1.938646912574768,
|
|
"logps/chosen": -0.3089125454425812,
|
|
"logps/rejected": -0.3455188274383545,
|
|
"loss": 2.0537664890289307,
|
|
"loss/core": 2.0537664890289307,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.8455417156219482,
|
|
"rewards/margins": 2.4277970790863037,
|
|
"rewards/rejected": 1.4177448749542236,
|
|
"step": 625
|
|
},
|
|
{
|
|
"epoch": 0.6019347903977069,
|
|
"grad_norm": 178.0,
|
|
"learning_rate": 1.25840659998631e-07,
|
|
"logits/chosen": -1.9074939489364624,
|
|
"logits/rejected": -2.022514820098877,
|
|
"logps/chosen": -0.27223581075668335,
|
|
"logps/rejected": -0.28289422392845154,
|
|
"loss": 0.8955941200256348,
|
|
"loss/core": 0.8955941200256348,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.9453296661376953,
|
|
"rewards/margins": 1.6964915990829468,
|
|
"rewards/rejected": 1.2488377094268799,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.6067120506389586,
|
|
"grad_norm": 286.0,
|
|
"learning_rate": 1.2165628955985313e-07,
|
|
"logits/chosen": -1.9614832401275635,
|
|
"logits/rejected": -2.0468192100524902,
|
|
"logps/chosen": -0.30598872900009155,
|
|
"logps/rejected": -0.30102452635765076,
|
|
"loss": 0.4347690939903259,
|
|
"loss/core": 0.4347690939903259,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.2950398921966553,
|
|
"rewards/margins": 1.2899543046951294,
|
|
"rewards/rejected": 1.0050859451293945,
|
|
"step": 635
|
|
},
|
|
{
|
|
"epoch": 0.6114893108802102,
|
|
"grad_norm": 2.59375,
|
|
"learning_rate": 1.175201839416988e-07,
|
|
"logits/chosen": -2.1878843307495117,
|
|
"logits/rejected": -2.2639148235321045,
|
|
"logps/chosen": -0.3026634752750397,
|
|
"logps/rejected": -0.3000384271144867,
|
|
"loss": 0.3829612135887146,
|
|
"loss/core": 0.3829612135887146,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.9458287954330444,
|
|
"rewards/margins": 1.161188006401062,
|
|
"rewards/rejected": 0.7846406102180481,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.6162665711214619,
|
|
"grad_norm": 15.0,
|
|
"learning_rate": 1.1343389856433658e-07,
|
|
"logits/chosen": -2.133284091949463,
|
|
"logits/rejected": -2.1109793186187744,
|
|
"logps/chosen": -0.2913122773170471,
|
|
"logps/rejected": -0.3065160810947418,
|
|
"loss": 0.20538568496704102,
|
|
"loss/core": 0.20538568496704102,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.6671216487884521,
|
|
"rewards/margins": 0.5487877726554871,
|
|
"rewards/rejected": 1.1183340549468994,
|
|
"step": 645
|
|
},
|
|
{
|
|
"epoch": 0.6210438313627135,
|
|
"grad_norm": 3.78125,
|
|
"learning_rate": 1.0939897011258e-07,
|
|
"logits/chosen": -2.198296546936035,
|
|
"logits/rejected": -2.151305913925171,
|
|
"logps/chosen": -0.2899485230445862,
|
|
"logps/rejected": -0.30108729004859924,
|
|
"loss": 0.1469523161649704,
|
|
"loss/core": 0.1469523161649704,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.3288962841033936,
|
|
"rewards/margins": 0.5087905526161194,
|
|
"rewards/rejected": 0.8201059103012085,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.6258210916039652,
|
|
"grad_norm": 53.0,
|
|
"learning_rate": 1.0541691595800336e-07,
|
|
"logits/chosen": -2.037231922149658,
|
|
"logits/rejected": -2.0994153022766113,
|
|
"logps/chosen": -0.28296002745628357,
|
|
"logps/rejected": -0.29361408948898315,
|
|
"loss": 0.24921326339244843,
|
|
"loss/core": 0.24921326339244843,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.997532844543457,
|
|
"rewards/margins": 1.048076868057251,
|
|
"rewards/rejected": 0.9494560360908508,
|
|
"step": 655
|
|
},
|
|
{
|
|
"epoch": 0.6305983518452167,
|
|
"grad_norm": 2.390625,
|
|
"learning_rate": 1.0148923358832021e-07,
|
|
"logits/chosen": -2.0072593688964844,
|
|
"logits/rejected": -2.000403642654419,
|
|
"logps/chosen": -0.29590821266174316,
|
|
"logps/rejected": -0.3091500699520111,
|
|
"loss": 1.2735823392868042,
|
|
"loss/core": 1.2735823392868042,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.7570533752441406,
|
|
"rewards/margins": 1.4806814193725586,
|
|
"rewards/rejected": 1.2763724327087402,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.6353756120864684,
|
|
"grad_norm": 134.0,
|
|
"learning_rate": 9.761740004423926e-08,
|
|
"logits/chosen": -2.169792652130127,
|
|
"logits/rejected": -2.1452460289001465,
|
|
"logps/chosen": -0.28445929288864136,
|
|
"logps/rejected": -0.2821694612503052,
|
|
"loss": 0.505510151386261,
|
|
"loss/core": 0.505510151386261,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.346339702606201,
|
|
"rewards/margins": 0.8737912178039551,
|
|
"rewards/rejected": 1.472548484802246,
|
|
"step": 665
|
|
},
|
|
{
|
|
"epoch": 0.64015287232772,
|
|
"grad_norm": 1.421875,
|
|
"learning_rate": 9.380287136400999e-08,
|
|
"logits/chosen": -1.958132028579712,
|
|
"logits/rejected": -2.0062148571014404,
|
|
"logps/chosen": -0.2736981213092804,
|
|
"logps/rejected": -0.32142865657806396,
|
|
"loss": 0.9827208518981934,
|
|
"loss/core": 0.9827208518981934,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.6511759757995605,
|
|
"rewards/margins": 2.406550168991089,
|
|
"rewards/rejected": 1.2446256875991821,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.6449301325689717,
|
|
"grad_norm": 15.375,
|
|
"learning_rate": 9.004708203586628e-08,
|
|
"logits/chosen": -2.010897159576416,
|
|
"logits/rejected": -2.05232834815979,
|
|
"logps/chosen": -0.28901082277297974,
|
|
"logps/rejected": -0.2891398072242737,
|
|
"loss": 0.12457512319087982,
|
|
"loss/core": 0.12457512319087982,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.6163723468780518,
|
|
"rewards/margins": 0.7447583079338074,
|
|
"rewards/rejected": 0.8716139793395996,
|
|
"step": 675
|
|
},
|
|
{
|
|
"epoch": 0.6497073928102234,
|
|
"grad_norm": 1.7265625,
|
|
"learning_rate": 8.635144445857407e-08,
|
|
"logits/chosen": -2.133448362350464,
|
|
"logits/rejected": -2.0533690452575684,
|
|
"logps/chosen": -0.28220346570014954,
|
|
"logps/rejected": -0.2999503016471863,
|
|
"loss": 0.41169777512550354,
|
|
"loss/core": 0.41169777512550354,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.1444594860076904,
|
|
"rewards/margins": 0.8426515460014343,
|
|
"rewards/rejected": 1.3018076419830322,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.654484653051475,
|
|
"grad_norm": 4.28125,
|
|
"learning_rate": 8.271734841028552e-08,
|
|
"logits/chosen": -2.0622639656066895,
|
|
"logits/rejected": -2.0482723712921143,
|
|
"logps/chosen": -0.2543559670448303,
|
|
"logps/rejected": -0.2783207893371582,
|
|
"loss": 0.9395847320556641,
|
|
"loss/core": 0.9395847320556641,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.8786520957946777,
|
|
"rewards/margins": 1.356764554977417,
|
|
"rewards/rejected": 1.5218875408172607,
|
|
"step": 685
|
|
},
|
|
{
|
|
"epoch": 0.6592619132927267,
|
|
"grad_norm": 3.203125,
|
|
"learning_rate": 7.91461605259007e-08,
|
|
"logits/chosen": -1.8647575378417969,
|
|
"logits/rejected": -1.887168526649475,
|
|
"logps/chosen": -0.3094645142555237,
|
|
"logps/rejected": -0.32861852645874023,
|
|
"loss": 0.20163805782794952,
|
|
"loss/core": 0.20163805782794952,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.7553918361663818,
|
|
"rewards/margins": 0.6693966388702393,
|
|
"rewards/rejected": 1.0859953165054321,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 0.6640391735339782,
|
|
"grad_norm": 9.375,
|
|
"learning_rate": 7.563922378313217e-08,
|
|
"logits/chosen": -2.2469770908355713,
|
|
"logits/rejected": -2.256824016571045,
|
|
"logps/chosen": -0.3077448308467865,
|
|
"logps/rejected": -0.2938057780265808,
|
|
"loss": 0.3713051676750183,
|
|
"loss/core": 0.3713051676750183,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.9559170007705688,
|
|
"rewards/margins": 1.282508134841919,
|
|
"rewards/rejected": 0.6734089851379395,
|
|
"step": 695
|
|
},
|
|
{
|
|
"epoch": 0.6688164337752299,
|
|
"grad_norm": 388.0,
|
|
"learning_rate": 7.219785699746572e-08,
|
|
"logits/chosen": -2.08168888092041,
|
|
"logits/rejected": -2.1877474784851074,
|
|
"logps/chosen": -0.2970959544181824,
|
|
"logps/rejected": -0.3301224112510681,
|
|
"loss": 0.46775469183921814,
|
|
"loss/core": 0.46775469183921814,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.227623224258423,
|
|
"rewards/margins": 0.7473882436752319,
|
|
"rewards/rejected": 1.4802348613739014,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.6735936940164815,
|
|
"grad_norm": 84.5,
|
|
"learning_rate": 6.882335432620779e-08,
|
|
"logits/chosen": -2.1753158569335938,
|
|
"logits/rejected": -2.1324844360351562,
|
|
"logps/chosen": -0.28070613741874695,
|
|
"logps/rejected": -0.2675221264362335,
|
|
"loss": 0.8060401082038879,
|
|
"loss/core": 0.8060401082038879,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.4011025428771973,
|
|
"rewards/margins": 1.8703250885009766,
|
|
"rewards/rejected": 1.5307778120040894,
|
|
"step": 705
|
|
},
|
|
{
|
|
"epoch": 0.6783709542577332,
|
|
"grad_norm": 370.0,
|
|
"learning_rate": 6.551698478180589e-08,
|
|
"logits/chosen": -1.862591028213501,
|
|
"logits/rejected": -1.8508355617523193,
|
|
"logps/chosen": -0.27737393975257874,
|
|
"logps/rejected": -0.3012697696685791,
|
|
"loss": 1.3350226879119873,
|
|
"loss/core": 1.3350226879119873,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.5951266288757324,
|
|
"rewards/margins": 1.7325704097747803,
|
|
"rewards/rejected": 1.8625566959381104,
|
|
"step": 710
|
|
},
|
|
{
|
|
"epoch": 0.6831482144989849,
|
|
"grad_norm": 3.78125,
|
|
"learning_rate": 6.22799917546252e-08,
|
|
"logits/chosen": -2.230132579803467,
|
|
"logits/rejected": -2.2110886573791504,
|
|
"logps/chosen": -0.2842472493648529,
|
|
"logps/rejected": -0.3385520279407501,
|
|
"loss": 0.5101768374443054,
|
|
"loss/core": 0.5101768374443054,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.8642584085464478,
|
|
"rewards/margins": 1.1836979389190674,
|
|
"rewards/rejected": 0.6805603504180908,
|
|
"step": 715
|
|
},
|
|
{
|
|
"epoch": 0.6879254747402365,
|
|
"grad_norm": 23.625,
|
|
"learning_rate": 5.9113592545359944e-08,
|
|
"logits/chosen": -1.9779284000396729,
|
|
"logits/rejected": -1.9853112697601318,
|
|
"logps/chosen": -0.2979108393192291,
|
|
"logps/rejected": -0.30363336205482483,
|
|
"loss": 0.4835893511772156,
|
|
"loss/core": 0.4835893511772156,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.0863685607910156,
|
|
"rewards/margins": 0.6202920079231262,
|
|
"rewards/rejected": 1.4660766124725342,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.6927027349814882,
|
|
"grad_norm": 3.953125,
|
|
"learning_rate": 5.601897790725643e-08,
|
|
"logits/chosen": -2.102151393890381,
|
|
"logits/rejected": -2.2167160511016846,
|
|
"logps/chosen": -0.28644677996635437,
|
|
"logps/rejected": -0.27808719873428345,
|
|
"loss": 0.17492833733558655,
|
|
"loss/core": 0.17492833733558655,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.5772634744644165,
|
|
"rewards/margins": 0.8539497256278992,
|
|
"rewards/rejected": 0.7233136892318726,
|
|
"step": 725
|
|
},
|
|
{
|
|
"epoch": 0.6974799952227397,
|
|
"grad_norm": 398.0,
|
|
"learning_rate": 5.299731159831952e-08,
|
|
"logits/chosen": -2.1961236000061035,
|
|
"logits/rejected": -2.246107339859009,
|
|
"logps/chosen": -0.287611722946167,
|
|
"logps/rejected": -0.2948874533176422,
|
|
"loss": 0.19452638924121857,
|
|
"loss/core": 0.19452638924121857,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.0076961517333984,
|
|
"rewards/margins": 0.14150258898735046,
|
|
"rewards/rejected": 0.8661934733390808,
|
|
"step": 730
|
|
},
|
|
{
|
|
"epoch": 0.7022572554639914,
|
|
"grad_norm": 58.0,
|
|
"learning_rate": 5.0049729943671013e-08,
|
|
"logits/chosen": -2.0291497707366943,
|
|
"logits/rejected": -2.0758984088897705,
|
|
"logps/chosen": -0.32085052132606506,
|
|
"logps/rejected": -0.31806251406669617,
|
|
"loss": 0.14255869388580322,
|
|
"loss/core": 0.14255869388580322,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.4092836380004883,
|
|
"rewards/margins": 0.7487727403640747,
|
|
"rewards/rejected": 0.6605110168457031,
|
|
"step": 735
|
|
},
|
|
{
|
|
"epoch": 0.707034515705243,
|
|
"grad_norm": 5.46875,
|
|
"learning_rate": 4.7177341408223994e-08,
|
|
"logits/chosen": -2.077390193939209,
|
|
"logits/rejected": -2.095350742340088,
|
|
"logps/chosen": -0.3161494731903076,
|
|
"logps/rejected": -0.3177900016307831,
|
|
"loss": 1.4744199514389038,
|
|
"loss/core": 1.4744199514389038,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.533155918121338,
|
|
"rewards/margins": 0.720278799533844,
|
|
"rewards/rejected": 1.8128772974014282,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.7118117759464947,
|
|
"grad_norm": 384.0,
|
|
"learning_rate": 4.438122617983442e-08,
|
|
"logits/chosen": -2.031482696533203,
|
|
"logits/rejected": -2.0756478309631348,
|
|
"logps/chosen": -0.29046431183815,
|
|
"logps/rejected": -0.2840021848678589,
|
|
"loss": 0.611270010471344,
|
|
"loss/core": 0.611270010471344,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.1211209297180176,
|
|
"rewards/margins": 1.6258805990219116,
|
|
"rewards/rejected": 1.4952404499053955,
|
|
"step": 745
|
|
},
|
|
{
|
|
"epoch": 0.7165890361877463,
|
|
"grad_norm": 318.0,
|
|
"learning_rate": 4.1662435763087114e-08,
|
|
"logits/chosen": -2.16795015335083,
|
|
"logits/rejected": -2.160215377807617,
|
|
"logps/chosen": -0.3122718930244446,
|
|
"logps/rejected": -0.3045649826526642,
|
|
"loss": 0.5550835728645325,
|
|
"loss/core": 0.5550835728645325,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.2003018856048584,
|
|
"rewards/margins": -0.04670139029622078,
|
|
"rewards/rejected": 1.2470033168792725,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 0.721366296428998,
|
|
"grad_norm": 23.125,
|
|
"learning_rate": 3.902199258386732e-08,
|
|
"logits/chosen": -1.905664086341858,
|
|
"logits/rejected": -1.815201997756958,
|
|
"logps/chosen": -0.3100651502609253,
|
|
"logps/rejected": -0.2970584034919739,
|
|
"loss": 0.3808279037475586,
|
|
"loss/core": 0.3808279037475586,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.371333122253418,
|
|
"rewards/margins": 0.689149796962738,
|
|
"rewards/rejected": 1.6821835041046143,
|
|
"step": 755
|
|
},
|
|
{
|
|
"epoch": 0.7261435566702497,
|
|
"grad_norm": 20.0,
|
|
"learning_rate": 3.646088960486862e-08,
|
|
"logits/chosen": -2.1599106788635254,
|
|
"logits/rejected": -2.178335189819336,
|
|
"logps/chosen": -0.3025417923927307,
|
|
"logps/rejected": -0.2942703664302826,
|
|
"loss": 0.27844661474227905,
|
|
"loss/core": 0.27844661474227905,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.9924026727676392,
|
|
"rewards/margins": 1.1353952884674072,
|
|
"rewards/rejected": 0.8570073843002319,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 0.7309208169115012,
|
|
"grad_norm": 41.25,
|
|
"learning_rate": 3.398008995217988e-08,
|
|
"logits/chosen": -2.05729603767395,
|
|
"logits/rejected": -2.128960132598877,
|
|
"logps/chosen": -0.2781466543674469,
|
|
"logps/rejected": -0.2730472981929779,
|
|
"loss": 0.23510298132896423,
|
|
"loss/core": 0.23510298132896423,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.773431420326233,
|
|
"rewards/margins": 0.7215307354927063,
|
|
"rewards/rejected": 1.0519007444381714,
|
|
"step": 765
|
|
},
|
|
{
|
|
"epoch": 0.7356980771527529,
|
|
"grad_norm": 50.25,
|
|
"learning_rate": 3.1580526553093315e-08,
|
|
"logits/chosen": -1.9602434635162354,
|
|
"logits/rejected": -1.927735686302185,
|
|
"logps/chosen": -0.31485968828201294,
|
|
"logps/rejected": -0.28551191091537476,
|
|
"loss": 2.1671090126037598,
|
|
"loss/core": 2.1671090126037598,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.6825318336486816,
|
|
"rewards/margins": 1.338300108909607,
|
|
"rewards/rejected": 2.3442318439483643,
|
|
"step": 770
|
|
},
|
|
{
|
|
"epoch": 0.7404753373940045,
|
|
"grad_norm": 1.8984375,
|
|
"learning_rate": 2.9263101785268252e-08,
|
|
"logits/chosen": -2.0744194984436035,
|
|
"logits/rejected": -2.105100154876709,
|
|
"logps/chosen": -0.3076358139514923,
|
|
"logps/rejected": -0.3005218207836151,
|
|
"loss": 0.14362755417823792,
|
|
"loss/core": 0.14362755417823792,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.5125480890274048,
|
|
"rewards/margins": 0.5728277564048767,
|
|
"rewards/rejected": 0.9397203326225281,
|
|
"step": 775
|
|
},
|
|
{
|
|
"epoch": 0.7452525976352562,
|
|
"grad_norm": 19.125,
|
|
"learning_rate": 2.7028687137384264e-08,
|
|
"logits/chosen": -2.156768798828125,
|
|
"logits/rejected": -2.170663356781006,
|
|
"logps/chosen": -0.3037753999233246,
|
|
"logps/rejected": -0.3085966110229492,
|
|
"loss": 0.22171640396118164,
|
|
"loss/core": 0.22171640396118164,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.8746483325958252,
|
|
"rewards/margins": 0.6551548838615417,
|
|
"rewards/rejected": 1.2194935083389282,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 0.7500298578765078,
|
|
"grad_norm": 1.6328125,
|
|
"learning_rate": 2.4878122881409447e-08,
|
|
"logits/chosen": -1.9704217910766602,
|
|
"logits/rejected": -1.92739999294281,
|
|
"logps/chosen": -0.3119404911994934,
|
|
"logps/rejected": -0.2957368493080139,
|
|
"loss": 1.2243354320526123,
|
|
"loss/core": 1.2243354320526123,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 3.421541690826416,
|
|
"rewards/margins": 2.0960593223571777,
|
|
"rewards/rejected": 1.3254823684692383,
|
|
"step": 785
|
|
},
|
|
{
|
|
"epoch": 0.7548071181177595,
|
|
"grad_norm": 244.0,
|
|
"learning_rate": 2.2812217756608937e-08,
|
|
"logits/chosen": -2.1468310356140137,
|
|
"logits/rejected": -1.9746036529541016,
|
|
"logps/chosen": -0.28714293241500854,
|
|
"logps/rejected": -0.3162146210670471,
|
|
"loss": 1.1724170446395874,
|
|
"loss/core": 1.1724170446395874,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.608785390853882,
|
|
"rewards/margins": 0.6583928465843201,
|
|
"rewards/rejected": 1.9503923654556274,
|
|
"step": 790
|
|
},
|
|
{
|
|
"epoch": 0.7595843783590112,
|
|
"grad_norm": 24.125,
|
|
"learning_rate": 2.0831748665410763e-08,
|
|
"logits/chosen": -1.9510393142700195,
|
|
"logits/rejected": -1.968506097793579,
|
|
"logps/chosen": -0.30975866317749023,
|
|
"logps/rejected": -0.2976404130458832,
|
|
"loss": 1.044623851776123,
|
|
"loss/core": 1.044623851776123,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.357090473175049,
|
|
"rewards/margins": 1.1862857341766357,
|
|
"rewards/rejected": 1.1708046197891235,
|
|
"step": 795
|
|
},
|
|
{
|
|
"epoch": 0.7643616386002627,
|
|
"grad_norm": 4.53125,
|
|
"learning_rate": 1.8937460381244967e-08,
|
|
"logits/chosen": -2.131096363067627,
|
|
"logits/rejected": -2.1090807914733887,
|
|
"logps/chosen": -0.2900455892086029,
|
|
"logps/rejected": -0.28949636220932007,
|
|
"loss": 0.3053150773048401,
|
|
"loss/core": 0.3053150773048401,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.288881778717041,
|
|
"rewards/margins": 1.1297084093093872,
|
|
"rewards/rejected": 1.1591734886169434,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.7691388988415144,
|
|
"grad_norm": 9.5625,
|
|
"learning_rate": 1.713006526846439e-08,
|
|
"logits/chosen": -2.0039420127868652,
|
|
"logits/rejected": -1.9931786060333252,
|
|
"logps/chosen": -0.27616554498672485,
|
|
"logps/rejected": -0.27139753103256226,
|
|
"loss": 0.5071675181388855,
|
|
"loss/core": 0.5071675181388855,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.244047164916992,
|
|
"rewards/margins": 0.266120582818985,
|
|
"rewards/rejected": 1.9779266119003296,
|
|
"step": 805
|
|
},
|
|
{
|
|
"epoch": 0.773916159082766,
|
|
"grad_norm": 28.75,
|
|
"learning_rate": 1.541024301445404e-08,
|
|
"logits/chosen": -2.2073111534118652,
|
|
"logits/rejected": -2.1122589111328125,
|
|
"logps/chosen": -0.3092059791088104,
|
|
"logps/rejected": -0.32349738478660583,
|
|
"loss": 0.23964479565620422,
|
|
"loss/core": 0.23964479565620422,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.5212091207504272,
|
|
"rewards/margins": 0.11138463020324707,
|
|
"rewards/rejected": 1.409824252128601,
|
|
"step": 810
|
|
},
|
|
{
|
|
"epoch": 0.7786934193240177,
|
|
"grad_norm": 35.0,
|
|
"learning_rate": 1.3778640374027983e-08,
|
|
"logits/chosen": -2.0743167400360107,
|
|
"logits/rejected": -2.162884473800659,
|
|
"logps/chosen": -0.2836160361766815,
|
|
"logps/rejected": -0.28452104330062866,
|
|
"loss": 0.24607639014720917,
|
|
"loss/core": 0.24607639014720917,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.1251344680786133,
|
|
"rewards/margins": 1.0335019826889038,
|
|
"rewards/rejected": 1.09163236618042,
|
|
"step": 815
|
|
},
|
|
{
|
|
"epoch": 0.7834706795652693,
|
|
"grad_norm": 3.296875,
|
|
"learning_rate": 1.2235870926211616e-08,
|
|
"logits/chosen": -2.079317569732666,
|
|
"logits/rejected": -2.112313747406006,
|
|
"logps/chosen": -0.288349449634552,
|
|
"logps/rejected": -0.31876736879348755,
|
|
"loss": 0.2414264678955078,
|
|
"loss/core": 0.2414264678955078,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.9595178365707397,
|
|
"rewards/margins": 1.2891201972961426,
|
|
"rewards/rejected": 0.6703976988792419,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 0.788247939806521,
|
|
"grad_norm": 60.5,
|
|
"learning_rate": 1.0782514843499652e-08,
|
|
"logits/chosen": -2.1222786903381348,
|
|
"logits/rejected": -2.2262613773345947,
|
|
"logps/chosen": -0.30225175619125366,
|
|
"logps/rejected": -0.2850930094718933,
|
|
"loss": 0.308658629655838,
|
|
"loss/core": 0.308658629655838,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.461728096008301,
|
|
"rewards/margins": 1.250454068183899,
|
|
"rewards/rejected": 1.2112740278244019,
|
|
"step": 825
|
|
},
|
|
{
|
|
"epoch": 0.7930252000477725,
|
|
"grad_norm": 8.625,
|
|
"learning_rate": 9.419118673676924e-09,
|
|
"logits/chosen": -2.156327962875366,
|
|
"logits/rejected": -2.1050965785980225,
|
|
"logps/chosen": -0.2788023054599762,
|
|
"logps/rejected": -0.3164371848106384,
|
|
"loss": 0.30850037932395935,
|
|
"loss/core": 0.30850037932395935,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.7236570119857788,
|
|
"rewards/margins": 0.15539640188217163,
|
|
"rewards/rejected": 1.5682607889175415,
|
|
"step": 830
|
|
},
|
|
{
|
|
"epoch": 0.7978024602890242,
|
|
"grad_norm": 173.0,
|
|
"learning_rate": 8.14619513428405e-09,
|
|
"logits/chosen": -2.0721898078918457,
|
|
"logits/rejected": -2.071643590927124,
|
|
"logps/chosen": -0.2557886242866516,
|
|
"logps/rejected": -0.30318066477775574,
|
|
"loss": 0.3867768943309784,
|
|
"loss/core": 0.3867768943309784,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.3934996128082275,
|
|
"rewards/margins": 1.5836869478225708,
|
|
"rewards/rejected": 0.8098125457763672,
|
|
"step": 835
|
|
},
|
|
{
|
|
"epoch": 0.8025797205302759,
|
|
"grad_norm": 2672.0,
|
|
"learning_rate": 6.96422291980539e-09,
|
|
"logits/chosen": -2.0129635334014893,
|
|
"logits/rejected": -2.0625948905944824,
|
|
"logps/chosen": -0.32838812470436096,
|
|
"logps/rejected": -0.3022240698337555,
|
|
"loss": 0.8802303075790405,
|
|
"loss/core": 0.8802303075790405,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.7439942359924316,
|
|
"rewards/margins": 1.7150169610977173,
|
|
"rewards/rejected": 1.0289772748947144,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 0.8073569807715275,
|
|
"grad_norm": 3.296875,
|
|
"learning_rate": 5.8736465216517594e-09,
|
|
"logits/chosen": -2.1509671211242676,
|
|
"logits/rejected": -2.22468638420105,
|
|
"logps/chosen": -0.29492026567459106,
|
|
"logps/rejected": -0.3140087425708771,
|
|
"loss": 0.15912021696567535,
|
|
"loss/core": 0.15912021696567535,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.4049441814422607,
|
|
"rewards/margins": 0.977611243724823,
|
|
"rewards/rejected": 0.4273327887058258,
|
|
"step": 845
|
|
},
|
|
{
|
|
"epoch": 0.8121342410127792,
|
|
"grad_norm": 37.0,
|
|
"learning_rate": 4.874876061005173e-09,
|
|
"logits/chosen": -2.2683005332946777,
|
|
"logits/rejected": -2.3160834312438965,
|
|
"logps/chosen": -0.25317704677581787,
|
|
"logps/rejected": -0.2489752322435379,
|
|
"loss": 0.0958312377333641,
|
|
"loss/core": 0.0958312377333641,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.54629647731781,
|
|
"rewards/margins": 0.6603418588638306,
|
|
"rewards/rejected": 0.8859546780586243,
|
|
"step": 850
|
|
},
|
|
{
|
|
"epoch": 0.8169115012540308,
|
|
"grad_norm": 59.5,
|
|
"learning_rate": 3.968287134589188e-09,
|
|
"logits/chosen": -1.9828602075576782,
|
|
"logits/rejected": -2.0203537940979004,
|
|
"logps/chosen": -0.30375003814697266,
|
|
"logps/rejected": -0.2930416464805603,
|
|
"loss": 0.8324758410453796,
|
|
"loss/core": 0.8324758410453796,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.6595799922943115,
|
|
"rewards/margins": 1.4085386991500854,
|
|
"rewards/rejected": 1.251041293144226,
|
|
"step": 855
|
|
},
|
|
{
|
|
"epoch": 0.8216887614952825,
|
|
"grad_norm": 2.515625,
|
|
"learning_rate": 3.154220673422192e-09,
|
|
"logits/chosen": -1.9286237955093384,
|
|
"logits/rejected": -2.016119956970215,
|
|
"logps/chosen": -0.3028337359428406,
|
|
"logps/rejected": -0.30019161105155945,
|
|
"loss": 1.6009376049041748,
|
|
"loss/core": 1.6009376049041748,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.053446292877197,
|
|
"rewards/margins": 2.1368956565856934,
|
|
"rewards/rejected": 1.916550874710083,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 0.826466021736534,
|
|
"grad_norm": 51.25,
|
|
"learning_rate": 2.4329828146074096e-09,
|
|
"logits/chosen": -1.9394134283065796,
|
|
"logits/rejected": -2.0103347301483154,
|
|
"logps/chosen": -0.2917220890522003,
|
|
"logps/rejected": -0.2960386276245117,
|
|
"loss": 0.590233325958252,
|
|
"loss/core": 0.590233325958252,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.443800926208496,
|
|
"rewards/margins": 0.8796428442001343,
|
|
"rewards/rejected": 1.5641582012176514,
|
|
"step": 865
|
|
},
|
|
{
|
|
"epoch": 0.8312432819777857,
|
|
"grad_norm": 13.125,
|
|
"learning_rate": 1.8048447862070714e-09,
|
|
"logits/chosen": -2.090069532394409,
|
|
"logits/rejected": -2.098034381866455,
|
|
"logps/chosen": -0.3098277449607849,
|
|
"logps/rejected": -0.29889780282974243,
|
|
"loss": 1.0631767511367798,
|
|
"loss/core": 1.0631767511367798,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.1978888511657715,
|
|
"rewards/margins": 2.2793126106262207,
|
|
"rewards/rejected": 0.9185762405395508,
|
|
"step": 870
|
|
},
|
|
{
|
|
"epoch": 0.8360205422190374,
|
|
"grad_norm": 7.15625,
|
|
"learning_rate": 1.2700428052447033e-09,
|
|
"logits/chosen": -2.1596736907958984,
|
|
"logits/rejected": -2.207904100418091,
|
|
"logps/chosen": -0.30997827649116516,
|
|
"logps/rejected": -0.3309651017189026,
|
|
"loss": 0.7209604382514954,
|
|
"loss/core": 0.7209604382514954,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.8416748046875,
|
|
"rewards/margins": 0.8194574117660522,
|
|
"rewards/rejected": 1.0222175121307373,
|
|
"step": 875
|
|
},
|
|
{
|
|
"epoch": 0.840797802460289,
|
|
"grad_norm": 62.25,
|
|
"learning_rate": 8.287779888734858e-10,
|
|
"logits/chosen": -2.0957977771759033,
|
|
"logits/rejected": -2.079482078552246,
|
|
"logps/chosen": -0.291581928730011,
|
|
"logps/rejected": -0.3347880244255066,
|
|
"loss": 0.9310186505317688,
|
|
"loss/core": 0.9310186505317688,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.448652982711792,
|
|
"rewards/margins": 0.716099739074707,
|
|
"rewards/rejected": 1.7325531244277954,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 0.8455750627015407,
|
|
"grad_norm": 23.75,
|
|
"learning_rate": 4.812162787445062e-10,
|
|
"logits/chosen": -2.0210766792297363,
|
|
"logits/rejected": -2.097437858581543,
|
|
"logps/chosen": -0.28354987502098083,
|
|
"logps/rejected": -0.27728238701820374,
|
|
"loss": 0.42107605934143066,
|
|
"loss/core": 0.42107605934143066,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.7150750160217285,
|
|
"rewards/margins": 1.5944372415542603,
|
|
"rewards/rejected": 1.1206376552581787,
|
|
"step": 885
|
|
},
|
|
{
|
|
"epoch": 0.8503523229427923,
|
|
"grad_norm": 296.0,
|
|
"learning_rate": 2.2748837860270265e-10,
|
|
"logits/chosen": -1.9509280920028687,
|
|
"logits/rejected": -2.072624921798706,
|
|
"logps/chosen": -0.3031921684741974,
|
|
"logps/rejected": -0.2749203145503998,
|
|
"loss": 0.6249431371688843,
|
|
"loss/core": 0.6249431371688843,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.4241182804107666,
|
|
"rewards/margins": 0.9915217161178589,
|
|
"rewards/rejected": 1.4325960874557495,
|
|
"step": 890
|
|
},
|
|
{
|
|
"epoch": 0.855129583184044,
|
|
"grad_norm": 52.0,
|
|
"learning_rate": 6.768970513457151e-11,
|
|
"logits/chosen": -2.0221574306488037,
|
|
"logits/rejected": -2.0542521476745605,
|
|
"logps/chosen": -0.27334579825401306,
|
|
"logps/rejected": -0.291655033826828,
|
|
"loss": 0.3214346766471863,
|
|
"loss/core": 0.3214346766471863,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.2918426990509033,
|
|
"rewards/margins": 0.789661705493927,
|
|
"rewards/rejected": 1.502180814743042,
|
|
"step": 895
|
|
},
|
|
{
|
|
"epoch": 0.8599068434252956,
|
|
"grad_norm": 17.125,
|
|
"learning_rate": 1.8803520859811406e-12,
|
|
"logits/chosen": -2.0327274799346924,
|
|
"logits/rejected": -2.152374744415283,
|
|
"logps/chosen": -0.271484911441803,
|
|
"logps/rejected": -0.2696631848812103,
|
|
"loss": 0.9997545480728149,
|
|
"loss/core": 0.9997545480728149,
|
|
"rewards/accuracies": 0.987500011920929,
|
|
"rewards/chosen": 3.091799259185791,
|
|
"rewards/margins": 2.2562103271484375,
|
|
"rewards/rejected": 0.8355890512466431,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.8599068434252956,
|
|
"step": 900,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.7102717447943158,
|
|
"train_runtime": 7093.1544,
|
|
"train_samples_per_second": 2.03,
|
|
"train_steps_per_second": 0.127
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 900,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 900,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|