2924 lines
101 KiB
JSON
2924 lines
101 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.8599068434252956,
|
|
"eval_steps": 500,
|
|
"global_step": 900,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.004777260241251642,
|
|
"grad_norm": 47.0,
|
|
"learning_rate": 2.222222222222222e-08,
|
|
"logits/chosen": -1.8765161037445068,
|
|
"logits/rejected": -1.8362525701522827,
|
|
"logps/chosen": -0.3087114691734314,
|
|
"logps/rejected": -0.3054303526878357,
|
|
"loss": 1.0474976301193237,
|
|
"loss/core": 1.0474976301193237,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.4050393104553223,
|
|
"rewards/margins": 1.9889495372772217,
|
|
"rewards/rejected": 1.416089653968811,
|
|
"step": 5
|
|
},
|
|
{
|
|
"epoch": 0.009554520482503284,
|
|
"grad_norm": 15.125,
|
|
"learning_rate": 5e-08,
|
|
"logits/chosen": -1.8804371356964111,
|
|
"logits/rejected": -1.8574333190917969,
|
|
"logps/chosen": -0.29082149267196655,
|
|
"logps/rejected": -0.27977004647254944,
|
|
"loss": 1.056221842765808,
|
|
"loss/core": 1.056221842765808,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.9182629585266113,
|
|
"rewards/margins": 1.5795632600784302,
|
|
"rewards/rejected": 1.3386995792388916,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.014331780723754926,
|
|
"grad_norm": 7.03125,
|
|
"learning_rate": 7.777777777777778e-08,
|
|
"logits/chosen": -1.6232883930206299,
|
|
"logits/rejected": -1.6565492153167725,
|
|
"logps/chosen": -0.2895023226737976,
|
|
"logps/rejected": -0.28725099563598633,
|
|
"loss": 1.042079210281372,
|
|
"loss/core": 1.042079210281372,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.9274177551269531,
|
|
"rewards/margins": 1.0600453615188599,
|
|
"rewards/rejected": 0.8673725128173828,
|
|
"step": 15
|
|
},
|
|
{
|
|
"epoch": 0.01910904096500657,
|
|
"grad_norm": 7.65625,
|
|
"learning_rate": 1.0555555555555555e-07,
|
|
"logits/chosen": -1.892533302307129,
|
|
"logits/rejected": -1.9024165868759155,
|
|
"logps/chosen": -0.28663748502731323,
|
|
"logps/rejected": -0.2839517891407013,
|
|
"loss": 1.0445116758346558,
|
|
"loss/core": 1.0445116758346558,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.6367988586425781,
|
|
"rewards/margins": 1.0847914218902588,
|
|
"rewards/rejected": 0.5520075559616089,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.02388630120625821,
|
|
"grad_norm": 17.25,
|
|
"learning_rate": 1.3333333333333334e-07,
|
|
"logits/chosen": -1.929903268814087,
|
|
"logits/rejected": -1.9212831258773804,
|
|
"logps/chosen": -0.27033165097236633,
|
|
"logps/rejected": -0.2790217995643616,
|
|
"loss": 1.028629183769226,
|
|
"loss/core": 1.028629183769226,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.8104407787322998,
|
|
"rewards/margins": 0.6213824152946472,
|
|
"rewards/rejected": 1.1890584230422974,
|
|
"step": 25
|
|
},
|
|
{
|
|
"epoch": 0.028663561447509853,
|
|
"grad_norm": 46.5,
|
|
"learning_rate": 1.611111111111111e-07,
|
|
"logits/chosen": -1.8559471368789673,
|
|
"logits/rejected": -1.8298406600952148,
|
|
"logps/chosen": -0.31229084730148315,
|
|
"logps/rejected": -0.27183204889297485,
|
|
"loss": 1.1012113094329834,
|
|
"loss/core": 1.1012113094329834,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.737351179122925,
|
|
"rewards/margins": 0.9300357103347778,
|
|
"rewards/rejected": 1.8073152303695679,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.033440821688761495,
|
|
"grad_norm": 9.3125,
|
|
"learning_rate": 1.8888888888888888e-07,
|
|
"logits/chosen": -1.848472237586975,
|
|
"logits/rejected": -1.9320148229599,
|
|
"logps/chosen": -0.2827574610710144,
|
|
"logps/rejected": -0.26435643434524536,
|
|
"loss": 1.06340754032135,
|
|
"loss/core": 1.06340754032135,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.8763984441757202,
|
|
"rewards/margins": 1.0876319408416748,
|
|
"rewards/rejected": 0.7887665629386902,
|
|
"step": 35
|
|
},
|
|
{
|
|
"epoch": 0.03821808193001314,
|
|
"grad_norm": 10.8125,
|
|
"learning_rate": 2.1666666666666667e-07,
|
|
"logits/chosen": -1.5899121761322021,
|
|
"logits/rejected": -1.6173330545425415,
|
|
"logps/chosen": -0.2993370592594147,
|
|
"logps/rejected": -0.30007094144821167,
|
|
"loss": 1.0462119579315186,
|
|
"loss/core": 1.0462119579315186,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.3351786136627197,
|
|
"rewards/margins": 1.363264799118042,
|
|
"rewards/rejected": 0.9719139933586121,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.04299534217126478,
|
|
"grad_norm": 34.75,
|
|
"learning_rate": 2.4444444444444445e-07,
|
|
"logits/chosen": -1.8352829217910767,
|
|
"logits/rejected": -1.8310245275497437,
|
|
"logps/chosen": -0.2917270064353943,
|
|
"logps/rejected": -0.2925964891910553,
|
|
"loss": 1.0427584648132324,
|
|
"loss/core": 1.0427584648132324,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.797705888748169,
|
|
"rewards/margins": 0.967245876789093,
|
|
"rewards/rejected": 1.8304599523544312,
|
|
"step": 45
|
|
},
|
|
{
|
|
"epoch": 0.04777260241251642,
|
|
"grad_norm": 4.25,
|
|
"learning_rate": 2.7222222222222216e-07,
|
|
"logits/chosen": -1.7508865594863892,
|
|
"logits/rejected": -1.819305419921875,
|
|
"logps/chosen": -0.2858908176422119,
|
|
"logps/rejected": -0.290945827960968,
|
|
"loss": 1.0335471630096436,
|
|
"loss/core": 1.0335471630096436,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.8175321817398071,
|
|
"rewards/margins": 0.4548220634460449,
|
|
"rewards/rejected": 1.3627102375030518,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.05254986265376806,
|
|
"grad_norm": 6.375,
|
|
"learning_rate": 3e-07,
|
|
"logits/chosen": -1.7931705713272095,
|
|
"logits/rejected": -1.942427396774292,
|
|
"logps/chosen": -0.29617246985435486,
|
|
"logps/rejected": -0.2928510010242462,
|
|
"loss": 1.0443193912506104,
|
|
"loss/core": 1.0443193912506104,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.6830313205718994,
|
|
"rewards/margins": 0.7598537802696228,
|
|
"rewards/rejected": 0.9231774210929871,
|
|
"step": 55
|
|
},
|
|
{
|
|
"epoch": 0.057327122895019705,
|
|
"grad_norm": 23.75,
|
|
"learning_rate": 3.2777777777777776e-07,
|
|
"logits/chosen": -1.921984314918518,
|
|
"logits/rejected": -1.9806658029556274,
|
|
"logps/chosen": -0.2908121645450592,
|
|
"logps/rejected": -0.28627273440361023,
|
|
"loss": 1.0474377870559692,
|
|
"loss/core": 1.0474377870559692,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.452622652053833,
|
|
"rewards/margins": 0.369498610496521,
|
|
"rewards/rejected": 1.0831242799758911,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.06210438313627135,
|
|
"grad_norm": 35.75,
|
|
"learning_rate": 3.5555555555555553e-07,
|
|
"logits/chosen": -1.824541449546814,
|
|
"logits/rejected": -1.8805049657821655,
|
|
"logps/chosen": -0.3165293037891388,
|
|
"logps/rejected": -0.3103017210960388,
|
|
"loss": 1.0472676753997803,
|
|
"loss/core": 1.0472676753997803,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.4452695846557617,
|
|
"rewards/margins": 1.5458446741104126,
|
|
"rewards/rejected": 0.8994248509407043,
|
|
"step": 65
|
|
},
|
|
{
|
|
"epoch": 0.06688164337752299,
|
|
"grad_norm": 62.25,
|
|
"learning_rate": 3.8333333333333335e-07,
|
|
"logits/chosen": -1.7961727380752563,
|
|
"logits/rejected": -1.8802435398101807,
|
|
"logps/chosen": -0.3014804720878601,
|
|
"logps/rejected": -0.3035959005355835,
|
|
"loss": 1.0426177978515625,
|
|
"loss/core": 1.0426177978515625,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.323197841644287,
|
|
"rewards/margins": 1.3843536376953125,
|
|
"rewards/rejected": 0.9388440847396851,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.07165890361877464,
|
|
"grad_norm": 7.71875,
|
|
"learning_rate": 4.1111111111111107e-07,
|
|
"logits/chosen": -1.7457157373428345,
|
|
"logits/rejected": -1.7732973098754883,
|
|
"logps/chosen": -0.29039743542671204,
|
|
"logps/rejected": -0.29679733514785767,
|
|
"loss": 1.031226396560669,
|
|
"loss/core": 1.031226396560669,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.5424000024795532,
|
|
"rewards/margins": 0.5238652229309082,
|
|
"rewards/rejected": 1.018534779548645,
|
|
"step": 75
|
|
},
|
|
{
|
|
"epoch": 0.07643616386002627,
|
|
"grad_norm": 8.9375,
|
|
"learning_rate": 4.3888888888888884e-07,
|
|
"logits/chosen": -1.8642539978027344,
|
|
"logits/rejected": -1.9320249557495117,
|
|
"logps/chosen": -0.3098042905330658,
|
|
"logps/rejected": -0.3067478537559509,
|
|
"loss": 1.0435469150543213,
|
|
"loss/core": 1.0435469150543213,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.2842977046966553,
|
|
"rewards/margins": 0.5746160745620728,
|
|
"rewards/rejected": 0.7096816301345825,
|
|
"step": 80
|
|
},
|
|
{
|
|
"epoch": 0.08121342410127792,
|
|
"grad_norm": 5.34375,
|
|
"learning_rate": 4.6666666666666666e-07,
|
|
"logits/chosen": -1.7921772003173828,
|
|
"logits/rejected": -1.7714446783065796,
|
|
"logps/chosen": -0.2924823760986328,
|
|
"logps/rejected": -0.2849774658679962,
|
|
"loss": 1.0487070083618164,
|
|
"loss/core": 1.0487070083618164,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.4415981769561768,
|
|
"rewards/margins": 0.7337201833724976,
|
|
"rewards/rejected": 1.7078783512115479,
|
|
"step": 85
|
|
},
|
|
{
|
|
"epoch": 0.08599068434252956,
|
|
"grad_norm": 9.8125,
|
|
"learning_rate": 4.944444444444445e-07,
|
|
"logits/chosen": -1.7024269104003906,
|
|
"logits/rejected": -1.8251234292984009,
|
|
"logps/chosen": -0.30760306119918823,
|
|
"logps/rejected": -0.30106034874916077,
|
|
"loss": 1.0480482578277588,
|
|
"loss/core": 1.0480482578277588,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.881970167160034,
|
|
"rewards/margins": 2.2146878242492676,
|
|
"rewards/rejected": 0.6672819256782532,
|
|
"step": 90
|
|
},
|
|
{
|
|
"epoch": 0.09076794458378121,
|
|
"grad_norm": 29.875,
|
|
"learning_rate": 4.999699149323369e-07,
|
|
"logits/chosen": -1.787375807762146,
|
|
"logits/rejected": -1.7270876169204712,
|
|
"logps/chosen": -0.28113657236099243,
|
|
"logps/rejected": -0.3039243221282959,
|
|
"loss": 1.0137240886688232,
|
|
"loss/core": 1.0137240886688232,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.1713666915893555,
|
|
"rewards/margins": 0.8056703805923462,
|
|
"rewards/rejected": 1.3656964302062988,
|
|
"step": 95
|
|
},
|
|
{
|
|
"epoch": 0.09554520482503284,
|
|
"grad_norm": 5.6875,
|
|
"learning_rate": 4.998477067547739e-07,
|
|
"logits/chosen": -1.668386697769165,
|
|
"logits/rejected": -1.639217734336853,
|
|
"logps/chosen": -0.2826381325721741,
|
|
"logps/rejected": -0.30979663133621216,
|
|
"loss": 1.0171899795532227,
|
|
"loss/core": 1.0171899795532227,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.973217725753784,
|
|
"rewards/margins": 1.4058278799057007,
|
|
"rewards/rejected": 1.567389726638794,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.10032246506628449,
|
|
"grad_norm": 7.84375,
|
|
"learning_rate": 4.996315410727229e-07,
|
|
"logits/chosen": -1.54190993309021,
|
|
"logits/rejected": -1.6111608743667603,
|
|
"logps/chosen": -0.27607348561286926,
|
|
"logps/rejected": -0.2650708854198456,
|
|
"loss": 1.0601215362548828,
|
|
"loss/core": 1.0601215362548828,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.1852011680603027,
|
|
"rewards/margins": 1.8053677082061768,
|
|
"rewards/rejected": 1.3798331022262573,
|
|
"step": 105
|
|
},
|
|
{
|
|
"epoch": 0.10509972530753613,
|
|
"grad_norm": 4.90625,
|
|
"learning_rate": 4.993214991772562e-07,
|
|
"logits/chosen": -1.7653499841690063,
|
|
"logits/rejected": -1.7720153331756592,
|
|
"logps/chosen": -0.2952477037906647,
|
|
"logps/rejected": -0.3086181879043579,
|
|
"loss": 1.0259872674942017,
|
|
"loss/core": 1.0259872674942017,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.194749355316162,
|
|
"rewards/margins": 1.190367579460144,
|
|
"rewards/rejected": 1.0043818950653076,
|
|
"step": 110
|
|
},
|
|
{
|
|
"epoch": 0.10987698554878778,
|
|
"grad_norm": 78.0,
|
|
"learning_rate": 4.989176976624511e-07,
|
|
"logits/chosen": -1.7536379098892212,
|
|
"logits/rejected": -1.7781728506088257,
|
|
"logps/chosen": -0.29186949133872986,
|
|
"logps/rejected": -0.2834053635597229,
|
|
"loss": 1.0515763759613037,
|
|
"loss/core": 1.0515763759613037,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.4836719036102295,
|
|
"rewards/margins": 2.2081823348999023,
|
|
"rewards/rejected": 1.2754894495010376,
|
|
"step": 115
|
|
},
|
|
{
|
|
"epoch": 0.11465424579003941,
|
|
"grad_norm": 34.0,
|
|
"learning_rate": 4.984202883815428e-07,
|
|
"logits/chosen": -1.9431312084197998,
|
|
"logits/rejected": -1.8143190145492554,
|
|
"logps/chosen": -0.26071271300315857,
|
|
"logps/rejected": -0.2756553292274475,
|
|
"loss": 1.0258220434188843,
|
|
"loss/core": 1.0258220434188843,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.8046910762786865,
|
|
"rewards/margins": 0.35107293725013733,
|
|
"rewards/rejected": 1.4536181688308716,
|
|
"step": 120
|
|
},
|
|
{
|
|
"epoch": 0.11943150603129106,
|
|
"grad_norm": 4.03125,
|
|
"learning_rate": 4.978294583898195e-07,
|
|
"logits/chosen": -1.7581638097763062,
|
|
"logits/rejected": -1.8171669244766235,
|
|
"logps/chosen": -0.2824077606201172,
|
|
"logps/rejected": -0.27793461084365845,
|
|
"loss": 1.0521032810211182,
|
|
"loss/core": 1.0521032810211182,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 4.071238040924072,
|
|
"rewards/margins": 3.047459125518799,
|
|
"rewards/rejected": 1.0237786769866943,
|
|
"step": 125
|
|
},
|
|
{
|
|
"epoch": 0.1242087662725427,
|
|
"grad_norm": 91.5,
|
|
"learning_rate": 4.971454298742779e-07,
|
|
"logits/chosen": -1.738824486732483,
|
|
"logits/rejected": -1.773324966430664,
|
|
"logps/chosen": -0.29637548327445984,
|
|
"logps/rejected": -0.292829692363739,
|
|
"loss": 1.0435963869094849,
|
|
"loss/core": 1.0435963869094849,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.691030979156494,
|
|
"rewards/margins": 0.9570609331130981,
|
|
"rewards/rejected": 1.733970284461975,
|
|
"step": 130
|
|
},
|
|
{
|
|
"epoch": 0.12898602651379434,
|
|
"grad_norm": 11.9375,
|
|
"learning_rate": 4.963684600700678e-07,
|
|
"logits/chosen": -1.7135483026504517,
|
|
"logits/rejected": -1.7729370594024658,
|
|
"logps/chosen": -0.27682191133499146,
|
|
"logps/rejected": -0.26342201232910156,
|
|
"loss": 1.0573183298110962,
|
|
"loss/core": 1.0573183298110962,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.7214828729629517,
|
|
"rewards/margins": 0.8246873617172241,
|
|
"rewards/rejected": 0.8967955708503723,
|
|
"step": 135
|
|
},
|
|
{
|
|
"epoch": 0.13376328675504598,
|
|
"grad_norm": 4.9375,
|
|
"learning_rate": 4.954988411637571e-07,
|
|
"logits/chosen": -1.7595752477645874,
|
|
"logits/rejected": -1.805037498474121,
|
|
"logps/chosen": -0.2989395260810852,
|
|
"logps/rejected": -0.29219749569892883,
|
|
"loss": 1.0499845743179321,
|
|
"loss/core": 1.0499845743179321,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.22503924369812,
|
|
"rewards/margins": 1.3095721006393433,
|
|
"rewards/rejected": 0.9154669046401978,
|
|
"step": 140
|
|
},
|
|
{
|
|
"epoch": 0.13854054699629761,
|
|
"grad_norm": 17.25,
|
|
"learning_rate": 4.945369001834514e-07,
|
|
"logits/chosen": -1.6619952917099,
|
|
"logits/rejected": -1.6887853145599365,
|
|
"logps/chosen": -0.3317473828792572,
|
|
"logps/rejected": -0.31527823209762573,
|
|
"loss": 1.0616098642349243,
|
|
"loss/core": 1.0616098642349243,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.8598458766937256,
|
|
"rewards/margins": 0.7766186594963074,
|
|
"rewards/rejected": 1.0832273960113525,
|
|
"step": 145
|
|
},
|
|
{
|
|
"epoch": 0.14331780723754928,
|
|
"grad_norm": 6.21875,
|
|
"learning_rate": 4.93482998875813e-07,
|
|
"logits/chosen": -1.6465572118759155,
|
|
"logits/rejected": -1.7042934894561768,
|
|
"logps/chosen": -0.310824453830719,
|
|
"logps/rejected": -0.3171466886997223,
|
|
"loss": 1.0322322845458984,
|
|
"loss/core": 1.0322322845458984,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.5014384984970093,
|
|
"rewards/margins": 0.929470419883728,
|
|
"rewards/rejected": 0.571968138217926,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.1480950674788009,
|
|
"grad_norm": 14.75,
|
|
"learning_rate": 4.923375335700223e-07,
|
|
"logits/chosen": -1.9649051427841187,
|
|
"logits/rejected": -1.9704341888427734,
|
|
"logps/chosen": -0.3165343403816223,
|
|
"logps/rejected": -0.303557425737381,
|
|
"loss": 1.056438684463501,
|
|
"loss/core": 1.056438684463501,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.594120740890503,
|
|
"rewards/margins": 0.7040458917617798,
|
|
"rewards/rejected": 0.8900747299194336,
|
|
"step": 155
|
|
},
|
|
{
|
|
"epoch": 0.15287232772005255,
|
|
"grad_norm": 146.0,
|
|
"learning_rate": 4.911009350287347e-07,
|
|
"logits/chosen": -1.7473423480987549,
|
|
"logits/rejected": -1.7553294897079468,
|
|
"logps/chosen": -0.29741325974464417,
|
|
"logps/rejected": -0.2803717255592346,
|
|
"loss": 1.0651330947875977,
|
|
"loss/core": 1.0651330947875977,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.707282543182373,
|
|
"rewards/margins": 2.542534351348877,
|
|
"rewards/rejected": 1.164747953414917,
|
|
"step": 160
|
|
},
|
|
{
|
|
"epoch": 0.15764958796130418,
|
|
"grad_norm": 9.4375,
|
|
"learning_rate": 4.897736682860885e-07,
|
|
"logits/chosen": -1.839644193649292,
|
|
"logits/rejected": -1.9340137243270874,
|
|
"logps/chosen": -0.3177702724933624,
|
|
"logps/rejected": -0.3130636215209961,
|
|
"loss": 1.049741268157959,
|
|
"loss/core": 1.049741268157959,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.8438761234283447,
|
|
"rewards/margins": 0.8960213661193848,
|
|
"rewards/rejected": 0.9478548765182495,
|
|
"step": 165
|
|
},
|
|
{
|
|
"epoch": 0.16242684820255585,
|
|
"grad_norm": 14.375,
|
|
"learning_rate": 4.883562324728241e-07,
|
|
"logits/chosen": -1.9273979663848877,
|
|
"logits/rejected": -2.001987934112549,
|
|
"logps/chosen": -0.2734295725822449,
|
|
"logps/rejected": -0.2835696339607239,
|
|
"loss": 1.0265181064605713,
|
|
"loss/core": 1.0265181064605713,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.5538994073867798,
|
|
"rewards/margins": 0.5848657488822937,
|
|
"rewards/rejected": 0.9690335988998413,
|
|
"step": 170
|
|
},
|
|
{
|
|
"epoch": 0.16720410844380748,
|
|
"grad_norm": 17.5,
|
|
"learning_rate": 4.868491606285823e-07,
|
|
"logits/chosen": -1.6086976528167725,
|
|
"logits/rejected": -1.6097809076309204,
|
|
"logps/chosen": -0.2869703769683838,
|
|
"logps/rejected": -0.2960742712020874,
|
|
"loss": 1.0377720594406128,
|
|
"loss/core": 1.0377720594406128,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 5.263535499572754,
|
|
"rewards/margins": 3.293330669403076,
|
|
"rewards/rejected": 1.9702045917510986,
|
|
"step": 175
|
|
},
|
|
{
|
|
"epoch": 0.17198136868505912,
|
|
"grad_norm": 4.375,
|
|
"learning_rate": 4.852530195014489e-07,
|
|
"logits/chosen": -1.8328279256820679,
|
|
"logits/rejected": -1.9173707962036133,
|
|
"logps/chosen": -0.28484848141670227,
|
|
"logps/rejected": -0.2925505042076111,
|
|
"loss": 1.0417985916137695,
|
|
"loss/core": 1.0417985916137695,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 3.295905590057373,
|
|
"rewards/margins": 1.7848495244979858,
|
|
"rewards/rejected": 1.5110561847686768,
|
|
"step": 180
|
|
},
|
|
{
|
|
"epoch": 0.17675862892631075,
|
|
"grad_norm": 6.59375,
|
|
"learning_rate": 4.835684093348244e-07,
|
|
"logits/chosen": -1.7860227823257446,
|
|
"logits/rejected": -1.84549880027771,
|
|
"logps/chosen": -0.25924015045166016,
|
|
"logps/rejected": -0.26765456795692444,
|
|
"loss": 1.0282881259918213,
|
|
"loss/core": 1.0282881259918213,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.9171737432479858,
|
|
"rewards/margins": 1.1408088207244873,
|
|
"rewards/rejected": 0.7763648629188538,
|
|
"step": 185
|
|
},
|
|
{
|
|
"epoch": 0.18153588916756241,
|
|
"grad_norm": 5.3125,
|
|
"learning_rate": 4.817959636416969e-07,
|
|
"logits/chosen": -1.6373240947723389,
|
|
"logits/rejected": -1.6969833374023438,
|
|
"logps/chosen": -0.3073750138282776,
|
|
"logps/rejected": -0.30846190452575684,
|
|
"loss": 1.0419862270355225,
|
|
"loss/core": 1.0419862270355225,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.9135539531707764,
|
|
"rewards/margins": 1.0026248693466187,
|
|
"rewards/rejected": 0.9109293222427368,
|
|
"step": 190
|
|
},
|
|
{
|
|
"epoch": 0.18631314940881405,
|
|
"grad_norm": 6.9375,
|
|
"learning_rate": 4.799363489664039e-07,
|
|
"logits/chosen": -1.808831810951233,
|
|
"logits/rejected": -1.7658703327178955,
|
|
"logps/chosen": -0.2862377464771271,
|
|
"logps/rejected": -0.3009548783302307,
|
|
"loss": 1.0329413414001465,
|
|
"loss/core": 1.0329413414001465,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.1667561531066895,
|
|
"rewards/margins": 0.9283055067062378,
|
|
"rewards/rejected": 1.2384504079818726,
|
|
"step": 195
|
|
},
|
|
{
|
|
"epoch": 0.19109040965006568,
|
|
"grad_norm": 24.875,
|
|
"learning_rate": 4.779902646339721e-07,
|
|
"logits/chosen": -1.722137451171875,
|
|
"logits/rejected": -1.6766093969345093,
|
|
"logps/chosen": -0.29227524995803833,
|
|
"logps/rejected": -0.289249062538147,
|
|
"loss": 1.0445283651351929,
|
|
"loss/core": 1.0445283651351929,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.8198704719543457,
|
|
"rewards/margins": 1.2550045251846313,
|
|
"rewards/rejected": 1.5648664236068726,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.19586766989131732,
|
|
"grad_norm": 18.625,
|
|
"learning_rate": 4.759584424871301e-07,
|
|
"logits/chosen": -1.7560946941375732,
|
|
"logits/rejected": -1.7568002939224243,
|
|
"logps/chosen": -0.3940495252609253,
|
|
"logps/rejected": -0.3073170483112335,
|
|
"loss": 1.2059910297393799,
|
|
"loss/core": 1.2059910297393799,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.591503620147705,
|
|
"rewards/margins": 1.2615159749984741,
|
|
"rewards/rejected": 1.329987645149231,
|
|
"step": 205
|
|
},
|
|
{
|
|
"epoch": 0.20064493013256898,
|
|
"grad_norm": 5.125,
|
|
"learning_rate": 4.738416466110917e-07,
|
|
"logits/chosen": -1.9233392477035522,
|
|
"logits/rejected": -2.00313663482666,
|
|
"logps/chosen": -0.25894150137901306,
|
|
"logps/rejected": -0.2547500729560852,
|
|
"loss": 1.0453473329544067,
|
|
"loss/core": 1.0453473329544067,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.7126867771148682,
|
|
"rewards/margins": 0.995030403137207,
|
|
"rewards/rejected": 0.7176561951637268,
|
|
"step": 210
|
|
},
|
|
{
|
|
"epoch": 0.20542219037382062,
|
|
"grad_norm": 51.75,
|
|
"learning_rate": 4.716406730462153e-07,
|
|
"logits/chosen": -1.8582137823104858,
|
|
"logits/rejected": -1.8355731964111328,
|
|
"logps/chosen": -0.28673282265663147,
|
|
"logps/rejected": -0.2773427367210388,
|
|
"loss": 1.0536106824874878,
|
|
"loss/core": 1.0536106824874878,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.7520334720611572,
|
|
"rewards/margins": 1.0252889394760132,
|
|
"rewards/rejected": 1.7267444133758545,
|
|
"step": 215
|
|
},
|
|
{
|
|
"epoch": 0.21019945061507225,
|
|
"grad_norm": 5.46875,
|
|
"learning_rate": 4.693563494886454e-07,
|
|
"logits/chosen": -1.6601266860961914,
|
|
"logits/rejected": -1.832810640335083,
|
|
"logps/chosen": -0.2959999144077301,
|
|
"logps/rejected": -0.28008145093917847,
|
|
"loss": 1.061637043952942,
|
|
"loss/core": 1.061637043952942,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.475043535232544,
|
|
"rewards/margins": 2.701432466506958,
|
|
"rewards/rejected": 0.7736114263534546,
|
|
"step": 220
|
|
},
|
|
{
|
|
"epoch": 0.2149767108563239,
|
|
"grad_norm": 35.0,
|
|
"learning_rate": 4.6698953497905016e-07,
|
|
"logits/chosen": -1.826503038406372,
|
|
"logits/rejected": -1.9233640432357788,
|
|
"logps/chosen": -0.27310609817504883,
|
|
"logps/rejected": -0.27189555764198303,
|
|
"loss": 1.040917158126831,
|
|
"loss/core": 1.040917158126831,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.2421226501464844,
|
|
"rewards/margins": 2.2241339683532715,
|
|
"rewards/rejected": 1.017989158630371,
|
|
"step": 225
|
|
},
|
|
{
|
|
"epoch": 0.21975397109757555,
|
|
"grad_norm": 124.0,
|
|
"learning_rate": 4.645411195795709e-07,
|
|
"logits/chosen": -1.8059947490692139,
|
|
"logits/rejected": -1.8729541301727295,
|
|
"logps/chosen": -0.25209683179855347,
|
|
"logps/rejected": -0.2518470287322998,
|
|
"loss": 1.0411286354064941,
|
|
"loss/core": 1.0411286354064941,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.6358604431152344,
|
|
"rewards/margins": 2.28558087348938,
|
|
"rewards/rejected": 1.3502795696258545,
|
|
"step": 230
|
|
},
|
|
{
|
|
"epoch": 0.2245312313388272,
|
|
"grad_norm": 5.90625,
|
|
"learning_rate": 4.6201202403910643e-07,
|
|
"logits/chosen": -2.0312843322753906,
|
|
"logits/rejected": -2.0044198036193848,
|
|
"logps/chosen": -0.2595166563987732,
|
|
"logps/rejected": -0.2711992859840393,
|
|
"loss": 1.0278807878494263,
|
|
"loss/core": 1.0278807878494263,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.072014331817627,
|
|
"rewards/margins": 0.8968809247016907,
|
|
"rewards/rejected": 1.1751338243484497,
|
|
"step": 235
|
|
},
|
|
{
|
|
"epoch": 0.22930849158007882,
|
|
"grad_norm": 14.6875,
|
|
"learning_rate": 4.594031994470573e-07,
|
|
"logits/chosen": -1.8727977275848389,
|
|
"logits/rejected": -1.8875232934951782,
|
|
"logps/chosen": -0.27784401178359985,
|
|
"logps/rejected": -0.29739147424697876,
|
|
"loss": 1.0186278820037842,
|
|
"loss/core": 1.0186278820037842,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.4038894176483154,
|
|
"rewards/margins": 1.5082296133041382,
|
|
"rewards/rejected": 1.8956598043441772,
|
|
"step": 240
|
|
},
|
|
{
|
|
"epoch": 0.23408575182133046,
|
|
"grad_norm": 71.5,
|
|
"learning_rate": 4.567156268756593e-07,
|
|
"logits/chosen": -1.5276188850402832,
|
|
"logits/rejected": -1.57925283908844,
|
|
"logps/chosen": -0.2935658097267151,
|
|
"logps/rejected": -0.3003990352153778,
|
|
"loss": 1.0357589721679688,
|
|
"loss/core": 1.0357589721679688,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.293689250946045,
|
|
"rewards/margins": 1.7028945684432983,
|
|
"rewards/rejected": 1.5907951593399048,
|
|
"step": 245
|
|
},
|
|
{
|
|
"epoch": 0.23886301206258212,
|
|
"grad_norm": 4.125,
|
|
"learning_rate": 4.5395031701104303e-07,
|
|
"logits/chosen": -1.9313945770263672,
|
|
"logits/rejected": -1.8916925191879272,
|
|
"logps/chosen": -0.2886388301849365,
|
|
"logps/rejected": -0.28957533836364746,
|
|
"loss": 1.037282943725586,
|
|
"loss/core": 1.037282943725586,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.5945872068405151,
|
|
"rewards/margins": 0.4880189895629883,
|
|
"rewards/rejected": 1.1065683364868164,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.24364027230383375,
|
|
"grad_norm": 3.671875,
|
|
"learning_rate": 4.511083097731555e-07,
|
|
"logits/chosen": -1.8803619146347046,
|
|
"logits/rejected": -1.8885911703109741,
|
|
"logps/chosen": -0.30258265137672424,
|
|
"logps/rejected": -0.30648648738861084,
|
|
"loss": 1.0346689224243164,
|
|
"loss/core": 1.0346689224243164,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 1.496037483215332,
|
|
"rewards/margins": 0.9408340454101562,
|
|
"rewards/rejected": 0.555203378200531,
|
|
"step": 255
|
|
},
|
|
{
|
|
"epoch": 0.2484175325450854,
|
|
"grad_norm": 4.59375,
|
|
"learning_rate": 4.481906739246894e-07,
|
|
"logits/chosen": -1.963661551475525,
|
|
"logits/rejected": -1.9137052297592163,
|
|
"logps/chosen": -0.27444905042648315,
|
|
"logps/rejected": -0.27626100182533264,
|
|
"loss": 1.0373988151550293,
|
|
"loss/core": 1.0373988151550293,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.3847882747650146,
|
|
"rewards/margins": 1.301588773727417,
|
|
"rewards/rejected": 1.0831997394561768,
|
|
"step": 260
|
|
},
|
|
{
|
|
"epoch": 0.25319479278633705,
|
|
"grad_norm": 5.75,
|
|
"learning_rate": 4.451985066691648e-07,
|
|
"logits/chosen": -2.0017013549804688,
|
|
"logits/rejected": -2.0215392112731934,
|
|
"logps/chosen": -0.2834641933441162,
|
|
"logps/rejected": -0.2866167724132538,
|
|
"loss": 1.0352154970169067,
|
|
"loss/core": 1.0352154970169067,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.4251301288604736,
|
|
"rewards/margins": 0.7156028151512146,
|
|
"rewards/rejected": 0.709527313709259,
|
|
"step": 265
|
|
},
|
|
{
|
|
"epoch": 0.2579720530275887,
|
|
"grad_norm": 12.0,
|
|
"learning_rate": 4.421329332383158e-07,
|
|
"logits/chosen": -1.7916221618652344,
|
|
"logits/rejected": -1.9190409183502197,
|
|
"logps/chosen": -0.3161014914512634,
|
|
"logps/rejected": -0.293036550283432,
|
|
"loss": 1.0780491828918457,
|
|
"loss/core": 1.0780491828918457,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.8342010974884033,
|
|
"rewards/margins": 1.156964898109436,
|
|
"rewards/rejected": 0.677236020565033,
|
|
"step": 270
|
|
},
|
|
{
|
|
"epoch": 0.2627493132688403,
|
|
"grad_norm": 10.6875,
|
|
"learning_rate": 4.3899510646893696e-07,
|
|
"logits/chosen": -1.8899133205413818,
|
|
"logits/rejected": -1.8713350296020508,
|
|
"logps/chosen": -0.2559998631477356,
|
|
"logps/rejected": -0.2608237862586975,
|
|
"loss": 1.0367724895477295,
|
|
"loss/core": 1.0367724895477295,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.4705395698547363,
|
|
"rewards/margins": 1.2723934650421143,
|
|
"rewards/rejected": 1.1981456279754639,
|
|
"step": 275
|
|
},
|
|
{
|
|
"epoch": 0.26752657351009196,
|
|
"grad_norm": 22.75,
|
|
"learning_rate": 4.357862063693485e-07,
|
|
"logits/chosen": -1.6495206356048584,
|
|
"logits/rejected": -1.69220769405365,
|
|
"logps/chosen": -0.3172406554222107,
|
|
"logps/rejected": -0.32189950346946716,
|
|
"loss": 1.0348529815673828,
|
|
"loss/core": 1.0348529815673828,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.4244751930236816,
|
|
"rewards/margins": 1.04317307472229,
|
|
"rewards/rejected": 1.3813021183013916,
|
|
"step": 280
|
|
},
|
|
{
|
|
"epoch": 0.2723038337513436,
|
|
"grad_norm": 18.625,
|
|
"learning_rate": 4.3250743967564364e-07,
|
|
"logits/chosen": -1.7316668033599854,
|
|
"logits/rejected": -1.6830451488494873,
|
|
"logps/chosen": -0.26700717210769653,
|
|
"logps/rejected": -0.2771323621273041,
|
|
"loss": 1.0284992456436157,
|
|
"loss/core": 1.0284992456436157,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.8682186603546143,
|
|
"rewards/margins": 1.2967889308929443,
|
|
"rewards/rejected": 1.5714294910430908,
|
|
"step": 285
|
|
},
|
|
{
|
|
"epoch": 0.27708109399259523,
|
|
"grad_norm": 10.9375,
|
|
"learning_rate": 4.29160039397884e-07,
|
|
"logits/chosen": -1.8204319477081299,
|
|
"logits/rejected": -1.9146995544433594,
|
|
"logps/chosen": -0.29401618242263794,
|
|
"logps/rejected": -0.2924644351005554,
|
|
"loss": 1.0413658618927002,
|
|
"loss/core": 1.0413658618927002,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.0374536514282227,
|
|
"rewards/margins": 1.1444836854934692,
|
|
"rewards/rejected": 0.8929699063301086,
|
|
"step": 290
|
|
},
|
|
{
|
|
"epoch": 0.28185835423384686,
|
|
"grad_norm": 68.0,
|
|
"learning_rate": 4.2574526435641546e-07,
|
|
"logits/chosen": -1.6919562816619873,
|
|
"logits/rejected": -1.6904808282852173,
|
|
"logps/chosen": -0.27512839436531067,
|
|
"logps/rejected": -0.2824132442474365,
|
|
"loss": 1.033488154411316,
|
|
"loss/core": 1.033488154411316,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.2550134658813477,
|
|
"rewards/margins": 0.5635195970535278,
|
|
"rewards/rejected": 1.6914937496185303,
|
|
"step": 295
|
|
},
|
|
{
|
|
"epoch": 0.28663561447509855,
|
|
"grad_norm": 14.5,
|
|
"learning_rate": 4.22264398708477e-07,
|
|
"logits/chosen": -1.6543853282928467,
|
|
"logits/rejected": -1.6278355121612549,
|
|
"logps/chosen": -0.2689480185508728,
|
|
"logps/rejected": -0.28801828622817993,
|
|
"loss": 1.0174682140350342,
|
|
"loss/core": 1.0174682140350342,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.770953416824341,
|
|
"rewards/margins": 1.4299379587173462,
|
|
"rewards/rejected": 1.3410152196884155,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.2914128747163502,
|
|
"grad_norm": 19.0,
|
|
"learning_rate": 4.187187514652819e-07,
|
|
"logits/chosen": -1.756967544555664,
|
|
"logits/rejected": -1.7581822872161865,
|
|
"logps/chosen": -0.2890106439590454,
|
|
"logps/rejected": -0.30179038643836975,
|
|
"loss": 1.028267502784729,
|
|
"loss/core": 1.028267502784729,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.826168417930603,
|
|
"rewards/margins": 0.5155857801437378,
|
|
"rewards/rejected": 1.3105827569961548,
|
|
"step": 305
|
|
},
|
|
{
|
|
"epoch": 0.2961901349576018,
|
|
"grad_norm": 31.5,
|
|
"learning_rate": 4.151096559997519e-07,
|
|
"logits/chosen": -1.8081289529800415,
|
|
"logits/rejected": -1.7069460153579712,
|
|
"logps/chosen": -0.2982190251350403,
|
|
"logps/rejected": -0.3032774031162262,
|
|
"loss": 1.0330473184585571,
|
|
"loss/core": 1.0330473184585571,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.1015119552612305,
|
|
"rewards/margins": 0.7455414533615112,
|
|
"rewards/rejected": 1.3559703826904297,
|
|
"step": 310
|
|
},
|
|
{
|
|
"epoch": 0.30096739519885346,
|
|
"grad_norm": 6.1875,
|
|
"learning_rate": 4.114384695450905e-07,
|
|
"logits/chosen": -1.9606285095214844,
|
|
"logits/rejected": -1.9415584802627563,
|
|
"logps/chosen": -0.2907028794288635,
|
|
"logps/rejected": -0.28985944390296936,
|
|
"loss": 1.0457550287246704,
|
|
"loss/core": 1.0457550287246704,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.5825884342193604,
|
|
"rewards/margins": 0.5970422029495239,
|
|
"rewards/rejected": 0.9855462312698364,
|
|
"step": 315
|
|
},
|
|
{
|
|
"epoch": 0.3057446554401051,
|
|
"grad_norm": 6.625,
|
|
"learning_rate": 4.077065726843828e-07,
|
|
"logits/chosen": -1.654712438583374,
|
|
"logits/rejected": -1.5861515998840332,
|
|
"logps/chosen": -0.28932780027389526,
|
|
"logps/rejected": -0.276142954826355,
|
|
"loss": 1.0589449405670166,
|
|
"loss/core": 1.0589449405670166,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.8658580780029297,
|
|
"rewards/margins": 0.9593901634216309,
|
|
"rewards/rejected": 1.9064680337905884,
|
|
"step": 320
|
|
},
|
|
{
|
|
"epoch": 0.31052191568135673,
|
|
"grad_norm": 8.9375,
|
|
"learning_rate": 4.039153688314145e-07,
|
|
"logits/chosen": -1.6774059534072876,
|
|
"logits/rejected": -1.7040441036224365,
|
|
"logps/chosen": -0.29178744554519653,
|
|
"logps/rejected": -0.2866431474685669,
|
|
"loss": 1.0456210374832153,
|
|
"loss/core": 1.0456210374832153,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.5071741342544556,
|
|
"rewards/margins": -0.009084177203476429,
|
|
"rewards/rejected": 1.5162582397460938,
|
|
"step": 325
|
|
},
|
|
{
|
|
"epoch": 0.31529917592260837,
|
|
"grad_norm": 12.6875,
|
|
"learning_rate": 4.0006628370290613e-07,
|
|
"logits/chosen": -1.8318054676055908,
|
|
"logits/rejected": -1.81647527217865,
|
|
"logps/chosen": -0.2811618745326996,
|
|
"logps/rejected": -0.28358012437820435,
|
|
"loss": 1.03645920753479,
|
|
"loss/core": 1.03645920753479,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.583585023880005,
|
|
"rewards/margins": 1.4332873821258545,
|
|
"rewards/rejected": 1.15029776096344,
|
|
"step": 330
|
|
},
|
|
{
|
|
"epoch": 0.32007643616386,
|
|
"grad_norm": 40.5,
|
|
"learning_rate": 3.9616076478235826e-07,
|
|
"logits/chosen": -1.6614739894866943,
|
|
"logits/rejected": -1.6278941631317139,
|
|
"logps/chosen": -0.28862547874450684,
|
|
"logps/rejected": -0.2881496548652649,
|
|
"loss": 1.0447041988372803,
|
|
"loss/core": 1.0447041988372803,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.6951169967651367,
|
|
"rewards/margins": 1.3128235340118408,
|
|
"rewards/rejected": 1.382293462753296,
|
|
"step": 335
|
|
},
|
|
{
|
|
"epoch": 0.3248536964051117,
|
|
"grad_norm": 11.125,
|
|
"learning_rate": 3.922002807757129e-07,
|
|
"logits/chosen": -1.8974721431732178,
|
|
"logits/rejected": -1.9225714206695557,
|
|
"logps/chosen": -0.29584866762161255,
|
|
"logps/rejected": -0.3060780167579651,
|
|
"loss": 1.0265626907348633,
|
|
"loss/core": 1.0265626907348633,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.664690613746643,
|
|
"rewards/margins": 0.8293659090995789,
|
|
"rewards/rejected": 0.8353245854377747,
|
|
"step": 340
|
|
},
|
|
{
|
|
"epoch": 0.3296309566463633,
|
|
"grad_norm": 13.875,
|
|
"learning_rate": 3.8818632105903315e-07,
|
|
"logits/chosen": -1.673317551612854,
|
|
"logits/rejected": -1.7775061130523682,
|
|
"logps/chosen": -0.2768518328666687,
|
|
"logps/rejected": -0.2922056317329407,
|
|
"loss": 1.0234686136245728,
|
|
"loss/core": 1.0234686136245728,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.881852626800537,
|
|
"rewards/margins": 0.5690313577651978,
|
|
"rewards/rejected": 2.31282114982605,
|
|
"step": 345
|
|
},
|
|
{
|
|
"epoch": 0.33440821688761496,
|
|
"grad_norm": 24.5,
|
|
"learning_rate": 3.841203951184094e-07,
|
|
"logits/chosen": -1.9435956478118896,
|
|
"logits/rejected": -1.8745100498199463,
|
|
"logps/chosen": -0.28658443689346313,
|
|
"logps/rejected": -0.2910448908805847,
|
|
"loss": 1.0343317985534668,
|
|
"loss/core": 1.0343317985534668,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.736780881881714,
|
|
"rewards/margins": 2.516613483428955,
|
|
"rewards/rejected": 1.220167636871338,
|
|
"step": 350
|
|
},
|
|
{
|
|
"epoch": 0.3391854771288666,
|
|
"grad_norm": 4.6875,
|
|
"learning_rate": 3.800040319823038e-07,
|
|
"logits/chosen": -1.682003378868103,
|
|
"logits/rejected": -1.7151581048965454,
|
|
"logps/chosen": -0.3114383816719055,
|
|
"logps/rejected": -0.3123466372489929,
|
|
"loss": 1.043797492980957,
|
|
"loss/core": 1.043797492980957,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 1.962998390197754,
|
|
"rewards/margins": 1.1944884061813354,
|
|
"rewards/rejected": 0.7685100436210632,
|
|
"step": 355
|
|
},
|
|
{
|
|
"epoch": 0.34396273737011823,
|
|
"grad_norm": 26.875,
|
|
"learning_rate": 3.75838779646545e-07,
|
|
"logits/chosen": -1.719026803970337,
|
|
"logits/rejected": -1.6863863468170166,
|
|
"logps/chosen": -0.31425201892852783,
|
|
"logps/rejected": -0.32097968459129333,
|
|
"loss": 1.0321420431137085,
|
|
"loss/core": 1.0321420431137085,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.7521181106567383,
|
|
"rewards/margins": -0.033901017159223557,
|
|
"rewards/rejected": 1.7860190868377686,
|
|
"step": 360
|
|
},
|
|
{
|
|
"epoch": 0.34873999761136987,
|
|
"grad_norm": 27.75,
|
|
"learning_rate": 3.7162620449218993e-07,
|
|
"logits/chosen": -1.8344494104385376,
|
|
"logits/rejected": -1.8459513187408447,
|
|
"logps/chosen": -0.29098817706108093,
|
|
"logps/rejected": -0.28455573320388794,
|
|
"loss": 1.0480910539627075,
|
|
"loss/core": 1.0480910539627075,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.697824478149414,
|
|
"rewards/margins": 2.8228657245635986,
|
|
"rewards/rejected": 0.8749583959579468,
|
|
"step": 365
|
|
},
|
|
{
|
|
"epoch": 0.3535172578526215,
|
|
"grad_norm": 10.75,
|
|
"learning_rate": 3.673678906964727e-07,
|
|
"logits/chosen": -1.622668981552124,
|
|
"logits/rejected": -1.6612154245376587,
|
|
"logps/chosen": -0.2679634094238281,
|
|
"logps/rejected": -0.2821475863456726,
|
|
"loss": 1.0219076871871948,
|
|
"loss/core": 1.0219076871871948,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.9509494304656982,
|
|
"rewards/margins": 1.733298659324646,
|
|
"rewards/rejected": 1.2176507711410522,
|
|
"step": 370
|
|
},
|
|
{
|
|
"epoch": 0.35829451809387314,
|
|
"grad_norm": 14.25,
|
|
"learning_rate": 3.6306543963705936e-07,
|
|
"logits/chosen": -1.685569405555725,
|
|
"logits/rejected": -1.6597143411636353,
|
|
"logps/chosen": -0.30852535367012024,
|
|
"logps/rejected": -0.2950063943862915,
|
|
"loss": 1.0666342973709106,
|
|
"loss/core": 1.0666342973709106,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 4.885121822357178,
|
|
"rewards/margins": 2.607081174850464,
|
|
"rewards/rejected": 2.278040647506714,
|
|
"step": 375
|
|
},
|
|
{
|
|
"epoch": 0.36307177833512483,
|
|
"grad_norm": 24.375,
|
|
"learning_rate": 3.5872046928983623e-07,
|
|
"logits/chosen": -1.7393958568572998,
|
|
"logits/rejected": -1.839385747909546,
|
|
"logps/chosen": -0.2777525782585144,
|
|
"logps/rejected": -0.28187328577041626,
|
|
"loss": 1.037045955657959,
|
|
"loss/core": 1.037045955657959,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.303683042526245,
|
|
"rewards/margins": 1.0037339925765991,
|
|
"rewards/rejected": 1.299949049949646,
|
|
"step": 380
|
|
},
|
|
{
|
|
"epoch": 0.36784903857637646,
|
|
"grad_norm": 17.875,
|
|
"learning_rate": 3.5433461362045447e-07,
|
|
"logits/chosen": -1.6332000494003296,
|
|
"logits/rejected": -1.6447935104370117,
|
|
"logps/chosen": -0.3114931285381317,
|
|
"logps/rejected": -0.28601643443107605,
|
|
"loss": 1.073043704032898,
|
|
"loss/core": 1.073043704032898,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.6627743244171143,
|
|
"rewards/margins": 0.6046234369277954,
|
|
"rewards/rejected": 1.0581510066986084,
|
|
"step": 385
|
|
},
|
|
{
|
|
"epoch": 0.3726262988176281,
|
|
"grad_norm": 5.6875,
|
|
"learning_rate": 3.4990952196986305e-07,
|
|
"logits/chosen": -1.9358383417129517,
|
|
"logits/rejected": -1.9621446132659912,
|
|
"logps/chosen": -0.27814939618110657,
|
|
"logps/rejected": -0.28488942980766296,
|
|
"loss": 1.0306986570358276,
|
|
"loss/core": 1.0306986570358276,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.2591309547424316,
|
|
"rewards/margins": 1.023651361465454,
|
|
"rewards/rejected": 1.2354795932769775,
|
|
"step": 390
|
|
},
|
|
{
|
|
"epoch": 0.37740355905887973,
|
|
"grad_norm": 6.875,
|
|
"learning_rate": 3.4544685843405875e-07,
|
|
"logits/chosen": -1.9065141677856445,
|
|
"logits/rejected": -1.8645994663238525,
|
|
"logps/chosen": -0.3078649640083313,
|
|
"logps/rejected": -0.302230566740036,
|
|
"loss": 1.0477346181869507,
|
|
"loss/core": 1.0477346181869507,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.9888968467712402,
|
|
"rewards/margins": 1.2866131067276,
|
|
"rewards/rejected": 1.7022842168807983,
|
|
"step": 395
|
|
},
|
|
{
|
|
"epoch": 0.38218081930013137,
|
|
"grad_norm": 4.15625,
|
|
"learning_rate": 3.4094830123828786e-07,
|
|
"logits/chosen": -1.7724406719207764,
|
|
"logits/rejected": -1.9576416015625,
|
|
"logps/chosen": -0.2747136950492859,
|
|
"logps/rejected": -0.2948540151119232,
|
|
"loss": 1.0195709466934204,
|
|
"loss/core": 1.0195709466934204,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.7523436546325684,
|
|
"rewards/margins": 0.6447616815567017,
|
|
"rewards/rejected": 2.1075825691223145,
|
|
"step": 400
|
|
},
|
|
{
|
|
"epoch": 0.386958079541383,
|
|
"grad_norm": 42.0,
|
|
"learning_rate": 3.3641554210593414e-07,
|
|
"logits/chosen": -1.838844895362854,
|
|
"logits/rejected": -1.8915517330169678,
|
|
"logps/chosen": -0.32020050287246704,
|
|
"logps/rejected": -0.28831055760383606,
|
|
"loss": 1.0958722829818726,
|
|
"loss/core": 1.0958722829818726,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.624875545501709,
|
|
"rewards/margins": 1.3493173122406006,
|
|
"rewards/rejected": 1.2755582332611084,
|
|
"step": 405
|
|
},
|
|
{
|
|
"epoch": 0.39173533978263464,
|
|
"grad_norm": 5.5,
|
|
"learning_rate": 3.3185028562233107e-07,
|
|
"logits/chosen": -1.8786590099334717,
|
|
"logits/rejected": -1.8944612741470337,
|
|
"logps/chosen": -0.26763150095939636,
|
|
"logps/rejected": -0.2702597975730896,
|
|
"loss": 1.038240671157837,
|
|
"loss/core": 1.038240671157837,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.777128219604492,
|
|
"rewards/margins": 1.4700839519500732,
|
|
"rewards/rejected": 1.3070439100265503,
|
|
"step": 410
|
|
},
|
|
{
|
|
"epoch": 0.3965126000238863,
|
|
"grad_norm": 50.0,
|
|
"learning_rate": 3.272542485937368e-07,
|
|
"logits/chosen": -1.6480621099472046,
|
|
"logits/rejected": -1.7010635137557983,
|
|
"logps/chosen": -0.2993305027484894,
|
|
"logps/rejected": -0.2903633117675781,
|
|
"loss": 1.056257963180542,
|
|
"loss/core": 1.056257963180542,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.3192853927612305,
|
|
"rewards/margins": 1.6964190006256104,
|
|
"rewards/rejected": 0.6228662729263306,
|
|
"step": 415
|
|
},
|
|
{
|
|
"epoch": 0.40128986026513797,
|
|
"grad_norm": 8.625,
|
|
"learning_rate": 3.226291594017137e-07,
|
|
"logits/chosen": -1.556668996810913,
|
|
"logits/rejected": -1.5511045455932617,
|
|
"logps/chosen": -0.2851598262786865,
|
|
"logps/rejected": -0.288934588432312,
|
|
"loss": 1.0358564853668213,
|
|
"loss/core": 1.0358564853668213,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.2491157054901123,
|
|
"rewards/margins": 1.3990309238433838,
|
|
"rewards/rejected": 1.850084900856018,
|
|
"step": 420
|
|
},
|
|
{
|
|
"epoch": 0.4060671205063896,
|
|
"grad_norm": 18.625,
|
|
"learning_rate": 3.1797675735315454e-07,
|
|
"logits/chosen": -1.6744778156280518,
|
|
"logits/rejected": -1.7476119995117188,
|
|
"logps/chosen": -0.2897992730140686,
|
|
"logps/rejected": -0.2777308523654938,
|
|
"loss": 1.0568740367889404,
|
|
"loss/core": 1.0568740367889404,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.9222044944763184,
|
|
"rewards/margins": 0.9556252360343933,
|
|
"rewards/rejected": 1.9665788412094116,
|
|
"step": 425
|
|
},
|
|
{
|
|
"epoch": 0.41084438074764124,
|
|
"grad_norm": 6.375,
|
|
"learning_rate": 3.1329879202620047e-07,
|
|
"logits/chosen": -1.8092721700668335,
|
|
"logits/rejected": -1.8535629510879517,
|
|
"logps/chosen": -0.3078886866569519,
|
|
"logps/rejected": -0.30679917335510254,
|
|
"loss": 1.04254150390625,
|
|
"loss/core": 1.04254150390625,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.51043438911438,
|
|
"rewards/margins": 1.6923158168792725,
|
|
"rewards/rejected": 0.818118691444397,
|
|
"step": 430
|
|
},
|
|
{
|
|
"epoch": 0.41562164098889287,
|
|
"grad_norm": 15.1875,
|
|
"learning_rate": 3.0859702261229613e-07,
|
|
"logits/chosen": -1.84933602809906,
|
|
"logits/rejected": -1.77957284450531,
|
|
"logps/chosen": -0.28597092628479004,
|
|
"logps/rejected": -0.2642970383167267,
|
|
"loss": 1.0709216594696045,
|
|
"loss/core": 1.0709216594696045,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.4492688179016113,
|
|
"rewards/margins": 1.2216014862060547,
|
|
"rewards/rejected": 1.2276675701141357,
|
|
"step": 435
|
|
},
|
|
{
|
|
"epoch": 0.4203989012301445,
|
|
"grad_norm": 44.5,
|
|
"learning_rate": 3.0387321725463e-07,
|
|
"logits/chosen": -1.6569089889526367,
|
|
"logits/rejected": -1.6702871322631836,
|
|
"logps/chosen": -0.31207168102264404,
|
|
"logps/rejected": -0.3157859444618225,
|
|
"loss": 1.0342601537704468,
|
|
"loss/core": 1.0342601537704468,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.2018513679504395,
|
|
"rewards/margins": 1.3161648511886597,
|
|
"rewards/rejected": 0.8856865167617798,
|
|
"step": 440
|
|
},
|
|
{
|
|
"epoch": 0.42517616147139614,
|
|
"grad_norm": 79.0,
|
|
"learning_rate": 2.991291523832075e-07,
|
|
"logits/chosen": -1.7703174352645874,
|
|
"logits/rejected": -1.7698230743408203,
|
|
"logps/chosen": -0.2817544937133789,
|
|
"logps/rejected": -0.2641424536705017,
|
|
"loss": 1.0657778978347778,
|
|
"loss/core": 1.0657778978347778,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.6899399757385254,
|
|
"rewards/margins": 1.9449743032455444,
|
|
"rewards/rejected": 1.7449655532836914,
|
|
"step": 445
|
|
},
|
|
{
|
|
"epoch": 0.4299534217126478,
|
|
"grad_norm": 7.625,
|
|
"learning_rate": 2.943666120468088e-07,
|
|
"logits/chosen": -1.723353624343872,
|
|
"logits/rejected": -1.7594738006591797,
|
|
"logps/chosen": -0.30141833424568176,
|
|
"logps/rejected": -0.2993377447128296,
|
|
"loss": 1.0418169498443604,
|
|
"loss/core": 1.0418169498443604,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.9154335260391235,
|
|
"rewards/margins": 1.0606777667999268,
|
|
"rewards/rejected": 0.8547555804252625,
|
|
"step": 450
|
|
},
|
|
{
|
|
"epoch": 0.4347306819538994,
|
|
"grad_norm": 14.125,
|
|
"learning_rate": 2.8958738724208073e-07,
|
|
"logits/chosen": -1.8625282049179077,
|
|
"logits/rejected": -1.9700651168823242,
|
|
"logps/chosen": -0.26571571826934814,
|
|
"logps/rejected": -0.2809098958969116,
|
|
"loss": 1.0212137699127197,
|
|
"loss/core": 1.0212137699127197,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.1734156608581543,
|
|
"rewards/margins": 0.8710582852363586,
|
|
"rewards/rejected": 1.30235755443573,
|
|
"step": 455
|
|
},
|
|
{
|
|
"epoch": 0.4395079421951511,
|
|
"grad_norm": 7.1875,
|
|
"learning_rate": 2.8479327524001633e-07,
|
|
"logits/chosen": -1.8570115566253662,
|
|
"logits/rejected": -1.7697608470916748,
|
|
"logps/chosen": -0.2919974625110626,
|
|
"logps/rejected": -0.29211685061454773,
|
|
"loss": 1.039764165878296,
|
|
"loss/core": 1.039764165878296,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.0760490894317627,
|
|
"rewards/margins": 1.1579499244689941,
|
|
"rewards/rejected": 0.9180992245674133,
|
|
"step": 460
|
|
},
|
|
{
|
|
"epoch": 0.44428520243640274,
|
|
"grad_norm": 18.375,
|
|
"learning_rate": 2.7998607891007493e-07,
|
|
"logits/chosen": -2.035067081451416,
|
|
"logits/rejected": -2.0927367210388184,
|
|
"logps/chosen": -0.3042372763156891,
|
|
"logps/rejected": -0.3184296488761902,
|
|
"loss": 1.069641351699829,
|
|
"loss/core": 1.069641351699829,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.6732687950134277,
|
|
"rewards/margins": 1.3037307262420654,
|
|
"rewards/rejected": 1.3695380687713623,
|
|
"step": 465
|
|
},
|
|
{
|
|
"epoch": 0.4490624626776544,
|
|
"grad_norm": 8.875,
|
|
"learning_rate": 2.7516760604219616e-07,
|
|
"logits/chosen": -1.872999906539917,
|
|
"logits/rejected": -1.875903844833374,
|
|
"logps/chosen": -0.296085923910141,
|
|
"logps/rejected": -0.2649732828140259,
|
|
"loss": 1.0810202360153198,
|
|
"loss/core": 1.0810202360153198,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.5348351001739502,
|
|
"rewards/margins": 0.5164709091186523,
|
|
"rewards/rejected": 1.0183641910552979,
|
|
"step": 470
|
|
},
|
|
{
|
|
"epoch": 0.453839722918906,
|
|
"grad_norm": 5.25,
|
|
"learning_rate": 2.703396686669646e-07,
|
|
"logits/chosen": -1.8748611211776733,
|
|
"logits/rejected": -1.8687400817871094,
|
|
"logps/chosen": -0.3014170825481415,
|
|
"logps/rejected": -0.2969133257865906,
|
|
"loss": 1.044512391090393,
|
|
"loss/core": 1.044512391090393,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.9237457513809204,
|
|
"rewards/margins": 1.39292573928833,
|
|
"rewards/rejected": 0.5308200120925903,
|
|
"step": 475
|
|
},
|
|
{
|
|
"epoch": 0.45861698316015764,
|
|
"grad_norm": 44.5,
|
|
"learning_rate": 2.6550408237417884e-07,
|
|
"logits/chosen": -1.7492471933364868,
|
|
"logits/rejected": -1.7116022109985352,
|
|
"logps/chosen": -0.3013203740119934,
|
|
"logps/rejected": -0.31782329082489014,
|
|
"loss": 1.023486852645874,
|
|
"loss/core": 1.023486852645874,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.455150604248047,
|
|
"rewards/margins": 1.1551944017410278,
|
|
"rewards/rejected": 1.2999564409255981,
|
|
"step": 480
|
|
},
|
|
{
|
|
"epoch": 0.4633942434014093,
|
|
"grad_norm": 30.75,
|
|
"learning_rate": 2.6066266563008265e-07,
|
|
"logits/chosen": -1.691613793373108,
|
|
"logits/rejected": -1.6590354442596436,
|
|
"logps/chosen": -0.2971816658973694,
|
|
"logps/rejected": -0.29611891508102417,
|
|
"loss": 1.040845513343811,
|
|
"loss/core": 1.040845513343811,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.0947299003601074,
|
|
"rewards/margins": 1.0051168203353882,
|
|
"rewards/rejected": 1.0896131992340088,
|
|
"step": 485
|
|
},
|
|
{
|
|
"epoch": 0.4681715036426609,
|
|
"grad_norm": 9.125,
|
|
"learning_rate": 2.5581723909351404e-07,
|
|
"logits/chosen": -1.6561720371246338,
|
|
"logits/rejected": -1.6545698642730713,
|
|
"logps/chosen": -0.3090967535972595,
|
|
"logps/rejected": -0.30728867650032043,
|
|
"loss": 1.0426571369171143,
|
|
"loss/core": 1.0426571369171143,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.3853068351745605,
|
|
"rewards/margins": 1.2928699254989624,
|
|
"rewards/rejected": 1.0924367904663086,
|
|
"step": 490
|
|
},
|
|
{
|
|
"epoch": 0.47294876388391255,
|
|
"grad_norm": 12.1875,
|
|
"learning_rate": 2.509696249312301e-07,
|
|
"logits/chosen": -1.7215149402618408,
|
|
"logits/rejected": -1.759275197982788,
|
|
"logps/chosen": -0.28365054726600647,
|
|
"logps/rejected": -0.2850833535194397,
|
|
"loss": 1.0367029905319214,
|
|
"loss/core": 1.0367029905319214,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.926408052444458,
|
|
"rewards/margins": 0.9314010739326477,
|
|
"rewards/rejected": 0.9950069189071655,
|
|
"step": 495
|
|
},
|
|
{
|
|
"epoch": 0.47772602412516424,
|
|
"grad_norm": 63.25,
|
|
"learning_rate": 2.461216461326642e-07,
|
|
"logits/chosen": -1.6816461086273193,
|
|
"logits/rejected": -1.7013599872589111,
|
|
"logps/chosen": -0.2813529670238495,
|
|
"logps/rejected": -0.28367528319358826,
|
|
"loss": 1.0371912717819214,
|
|
"loss/core": 1.0371912717819214,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.650829315185547,
|
|
"rewards/margins": 1.9984947443008423,
|
|
"rewards/rejected": 1.6523345708847046,
|
|
"step": 500
|
|
},
|
|
{
|
|
"epoch": 0.4825032843664159,
|
|
"grad_norm": 8.1875,
|
|
"learning_rate": 2.412751258243748e-07,
|
|
"logits/chosen": -1.6414563655853271,
|
|
"logits/rejected": -1.585834264755249,
|
|
"logps/chosen": -0.3117072582244873,
|
|
"logps/rejected": -0.3133184313774109,
|
|
"loss": 1.0436599254608154,
|
|
"loss/core": 1.0436599254608154,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.4120712280273438,
|
|
"rewards/margins": 2.0417308807373047,
|
|
"rewards/rejected": 1.3703404664993286,
|
|
"step": 505
|
|
},
|
|
{
|
|
"epoch": 0.4872805446076675,
|
|
"grad_norm": 10.1875,
|
|
"learning_rate": 2.3643188658444158e-07,
|
|
"logits/chosen": -1.901231050491333,
|
|
"logits/rejected": -1.9024569988250732,
|
|
"logps/chosen": -0.29130491614341736,
|
|
"logps/rejected": -0.28371134400367737,
|
|
"loss": 1.05287504196167,
|
|
"loss/core": 1.05287504196167,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.3334412574768066,
|
|
"rewards/margins": 1.1235178709030151,
|
|
"rewards/rejected": 1.209923505783081,
|
|
"step": 510
|
|
},
|
|
{
|
|
"epoch": 0.49205780484891914,
|
|
"grad_norm": 28.375,
|
|
"learning_rate": 2.315937497570688e-07,
|
|
"logits/chosen": -1.5855246782302856,
|
|
"logits/rejected": -1.6642907857894897,
|
|
"logps/chosen": -0.2806105613708496,
|
|
"logps/rejected": -0.2813439965248108,
|
|
"loss": 1.0392553806304932,
|
|
"loss/core": 1.0392553806304932,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.8368934392929077,
|
|
"rewards/margins": 0.04681912809610367,
|
|
"rewards/rejected": 1.7900745868682861,
|
|
"step": 515
|
|
},
|
|
{
|
|
"epoch": 0.4968350650901708,
|
|
"grad_norm": 19.875,
|
|
"learning_rate": 2.2676253476765194e-07,
|
|
"logits/chosen": -1.7813889980316162,
|
|
"logits/rejected": -1.7595758438110352,
|
|
"logps/chosen": -0.29206109046936035,
|
|
"logps/rejected": -0.28826069831848145,
|
|
"loss": 1.0510475635528564,
|
|
"loss/core": 1.0510475635528564,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.1681056022644043,
|
|
"rewards/margins": 1.8101484775543213,
|
|
"rewards/rejected": 1.357957363128662,
|
|
"step": 520
|
|
},
|
|
{
|
|
"epoch": 0.5016123253314224,
|
|
"grad_norm": 6.0625,
|
|
"learning_rate": 2.2194005843856633e-07,
|
|
"logits/chosen": -1.7787355184555054,
|
|
"logits/rejected": -1.8631689548492432,
|
|
"logps/chosen": -0.2766266465187073,
|
|
"logps/rejected": -0.27404943108558655,
|
|
"loss": 1.042309045791626,
|
|
"loss/core": 1.042309045791626,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.1334118843078613,
|
|
"rewards/margins": 1.3973788022994995,
|
|
"rewards/rejected": 0.7360331416130066,
|
|
"step": 525
|
|
},
|
|
{
|
|
"epoch": 0.5063895855726741,
|
|
"grad_norm": 5.28125,
|
|
"learning_rate": 2.1712813430593434e-07,
|
|
"logits/chosen": -1.885209321975708,
|
|
"logits/rejected": -1.9635217189788818,
|
|
"logps/chosen": -0.28333884477615356,
|
|
"logps/rejected": -0.29169341921806335,
|
|
"loss": 1.0290471315383911,
|
|
"loss/core": 1.0290471315383911,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.6382904052734375,
|
|
"rewards/margins": 0.8100762367248535,
|
|
"rewards/rejected": 0.8282139897346497,
|
|
"step": 530
|
|
},
|
|
{
|
|
"epoch": 0.5111668458139257,
|
|
"grad_norm": 5.5625,
|
|
"learning_rate": 2.123285719376292e-07,
|
|
"logits/chosen": -1.7140337228775024,
|
|
"logits/rejected": -1.700222373008728,
|
|
"logps/chosen": -0.2663342356681824,
|
|
"logps/rejected": -0.2759917378425598,
|
|
"loss": 1.0282187461853027,
|
|
"loss/core": 1.0282187461853027,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.3432140350341797,
|
|
"rewards/margins": 2.151822566986084,
|
|
"rewards/rejected": 1.1913913488388062,
|
|
"step": 535
|
|
},
|
|
{
|
|
"epoch": 0.5159441060551774,
|
|
"grad_norm": 39.25,
|
|
"learning_rate": 2.0754317625276982e-07,
|
|
"logits/chosen": -1.79038405418396,
|
|
"logits/rejected": -1.9031089544296265,
|
|
"logps/chosen": -0.29101166129112244,
|
|
"logps/rejected": -0.2809355556964874,
|
|
"loss": 1.0550512075424194,
|
|
"loss/core": 1.0550512075424194,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.919968366622925,
|
|
"rewards/margins": 2.102735996246338,
|
|
"rewards/rejected": 0.8172324299812317,
|
|
"step": 540
|
|
},
|
|
{
|
|
"epoch": 0.520721366296429,
|
|
"grad_norm": 35.5,
|
|
"learning_rate": 2.0277374684296498e-07,
|
|
"logits/chosen": -1.715163230895996,
|
|
"logits/rejected": -1.8029130697250366,
|
|
"logps/chosen": -0.30172231793403625,
|
|
"logps/rejected": -0.32220780849456787,
|
|
"loss": 1.0138914585113525,
|
|
"loss/core": 1.0138914585113525,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.903212070465088,
|
|
"rewards/margins": 1.434679627418518,
|
|
"rewards/rejected": 1.4685324430465698,
|
|
"step": 545
|
|
},
|
|
{
|
|
"epoch": 0.5254986265376806,
|
|
"grad_norm": 5.90625,
|
|
"learning_rate": 1.980220772955602e-07,
|
|
"logits/chosen": -1.949233055114746,
|
|
"logits/rejected": -2.019287347793579,
|
|
"logps/chosen": -0.2586565315723419,
|
|
"logps/rejected": -0.2541942596435547,
|
|
"loss": 1.0455553531646729,
|
|
"loss/core": 1.0455553531646729,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.489222288131714,
|
|
"rewards/margins": 0.4238598942756653,
|
|
"rewards/rejected": 2.0653622150421143,
|
|
"step": 550
|
|
},
|
|
{
|
|
"epoch": 0.5302758867789323,
|
|
"grad_norm": 19.125,
|
|
"learning_rate": 1.932899545191433e-07,
|
|
"logits/chosen": -1.9635541439056396,
|
|
"logits/rejected": -1.9265022277832031,
|
|
"logps/chosen": -0.290753573179245,
|
|
"logps/rejected": -0.27892938256263733,
|
|
"loss": 1.0557947158813477,
|
|
"loss/core": 1.0557947158813477,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.1324610710144043,
|
|
"rewards/margins": 0.5951575040817261,
|
|
"rewards/rejected": 1.5373035669326782,
|
|
"step": 555
|
|
},
|
|
{
|
|
"epoch": 0.5350531470201839,
|
|
"grad_norm": 77.0,
|
|
"learning_rate": 1.885791580715609e-07,
|
|
"logits/chosen": -1.6048539876937866,
|
|
"logits/rejected": -1.6119636297225952,
|
|
"logps/chosen": -0.28494495153427124,
|
|
"logps/rejected": -0.286663681268692,
|
|
"loss": 1.038069725036621,
|
|
"loss/core": 1.038069725036621,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.65592098236084,
|
|
"rewards/margins": 0.7274275422096252,
|
|
"rewards/rejected": 1.9284934997558594,
|
|
"step": 560
|
|
},
|
|
{
|
|
"epoch": 0.5398304072614356,
|
|
"grad_norm": 6.84375,
|
|
"learning_rate": 1.8389145949069951e-07,
|
|
"logits/chosen": -1.6353938579559326,
|
|
"logits/rejected": -1.6788675785064697,
|
|
"logps/chosen": -0.2736624479293823,
|
|
"logps/rejected": -0.26380524039268494,
|
|
"loss": 1.052668809890747,
|
|
"loss/core": 1.052668809890747,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.4656434059143066,
|
|
"rewards/margins": 1.5169572830200195,
|
|
"rewards/rejected": 1.9486862421035767,
|
|
"step": 565
|
|
},
|
|
{
|
|
"epoch": 0.5446076675026872,
|
|
"grad_norm": 45.25,
|
|
"learning_rate": 1.792286216282824e-07,
|
|
"logits/chosen": -1.558429479598999,
|
|
"logits/rejected": -1.5371347665786743,
|
|
"logps/chosen": -0.2772481441497803,
|
|
"logps/rejected": -0.2760164737701416,
|
|
"loss": 1.0409849882125854,
|
|
"loss/core": 1.0409849882125854,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.2676281929016113,
|
|
"rewards/margins": 0.6158636212348938,
|
|
"rewards/rejected": 1.6517642736434937,
|
|
"step": 570
|
|
},
|
|
{
|
|
"epoch": 0.5493849277439389,
|
|
"grad_norm": 3.640625,
|
|
"learning_rate": 1.745923979869336e-07,
|
|
"logits/chosen": -1.8342043161392212,
|
|
"logits/rejected": -1.8309091329574585,
|
|
"logps/chosen": -0.2796408534049988,
|
|
"logps/rejected": -0.2815830409526825,
|
|
"loss": 1.0364043712615967,
|
|
"loss/core": 1.0364043712615967,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.265101194381714,
|
|
"rewards/margins": 1.3904941082000732,
|
|
"rewards/rejected": 0.8746069669723511,
|
|
"step": 575
|
|
},
|
|
{
|
|
"epoch": 0.5541621879851905,
|
|
"grad_norm": 7.4375,
|
|
"learning_rate": 1.6998453206075708e-07,
|
|
"logits/chosen": -2.014143466949463,
|
|
"logits/rejected": -2.0165810585021973,
|
|
"logps/chosen": -0.2994392216205597,
|
|
"logps/rejected": -0.30162468552589417,
|
|
"loss": 1.037367582321167,
|
|
"loss/core": 1.037367582321167,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.404824137687683,
|
|
"rewards/margins": 0.4138920307159424,
|
|
"rewards/rejected": 0.990932285785675,
|
|
"step": 580
|
|
},
|
|
{
|
|
"epoch": 0.5589394482264421,
|
|
"grad_norm": 5.90625,
|
|
"learning_rate": 1.6540675667967973e-07,
|
|
"logits/chosen": -1.724384069442749,
|
|
"logits/rejected": -1.7589279413223267,
|
|
"logps/chosen": -0.30380764603614807,
|
|
"logps/rejected": -0.2969801127910614,
|
|
"loss": 1.0485934019088745,
|
|
"loss/core": 1.0485934019088745,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.2653725147247314,
|
|
"rewards/margins": 1.247117042541504,
|
|
"rewards/rejected": 1.0182554721832275,
|
|
"step": 585
|
|
},
|
|
{
|
|
"epoch": 0.5637167084676937,
|
|
"grad_norm": 11.5625,
|
|
"learning_rate": 1.60860793357805e-07,
|
|
"logits/chosen": -1.806414008140564,
|
|
"logits/rejected": -1.8376471996307373,
|
|
"logps/chosen": -0.2740582823753357,
|
|
"logps/rejected": -0.27855148911476135,
|
|
"loss": 1.033927083015442,
|
|
"loss/core": 1.033927083015442,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.1178157329559326,
|
|
"rewards/margins": 0.7777595520019531,
|
|
"rewards/rejected": 1.3400561809539795,
|
|
"step": 590
|
|
},
|
|
{
|
|
"epoch": 0.5684939687089454,
|
|
"grad_norm": 10.5,
|
|
"learning_rate": 1.5634835164602196e-07,
|
|
"logits/chosen": -1.6470069885253906,
|
|
"logits/rejected": -1.700169563293457,
|
|
"logps/chosen": -0.2837096154689789,
|
|
"logps/rejected": -0.28879398107528687,
|
|
"loss": 1.033031940460205,
|
|
"loss/core": 1.033031940460205,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.160616636276245,
|
|
"rewards/margins": 0.9769798517227173,
|
|
"rewards/rejected": 1.1836367845535278,
|
|
"step": 595
|
|
},
|
|
{
|
|
"epoch": 0.5732712289501971,
|
|
"grad_norm": 13.625,
|
|
"learning_rate": 1.518711284891132e-07,
|
|
"logits/chosen": -1.9625580310821533,
|
|
"logits/rejected": -1.928464651107788,
|
|
"logps/chosen": -0.2618710696697235,
|
|
"logps/rejected": -0.27233943343162537,
|
|
"loss": 1.0305383205413818,
|
|
"loss/core": 1.0305383205413818,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.9427595138549805,
|
|
"rewards/margins": 0.7403771877288818,
|
|
"rewards/rejected": 1.2023823261260986,
|
|
"step": 600
|
|
},
|
|
{
|
|
"epoch": 0.5780484891914487,
|
|
"grad_norm": 79.5,
|
|
"learning_rate": 1.47430807587603e-07,
|
|
"logits/chosen": -1.7923481464385986,
|
|
"logits/rejected": -1.9285895824432373,
|
|
"logps/chosen": -0.3259024918079376,
|
|
"logps/rejected": -0.33158808946609497,
|
|
"loss": 1.0331567525863647,
|
|
"loss/core": 1.0331567525863647,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.143375873565674,
|
|
"rewards/margins": 1.2113240957260132,
|
|
"rewards/rejected": 0.9320517778396606,
|
|
"step": 605
|
|
},
|
|
{
|
|
"epoch": 0.5828257494327004,
|
|
"grad_norm": 18.125,
|
|
"learning_rate": 1.430290587645865e-07,
|
|
"logits/chosen": -1.8447580337524414,
|
|
"logits/rejected": -1.826443076133728,
|
|
"logps/chosen": -0.2840345799922943,
|
|
"logps/rejected": -0.29238995909690857,
|
|
"loss": 1.0294818878173828,
|
|
"loss/core": 1.0294818878173828,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.1355230808258057,
|
|
"rewards/margins": 0.9689822196960449,
|
|
"rewards/rejected": 1.1665408611297607,
|
|
"step": 610
|
|
},
|
|
{
|
|
"epoch": 0.587603009673952,
|
|
"grad_norm": 10.75,
|
|
"learning_rate": 1.3866753733777765e-07,
|
|
"logits/chosen": -1.8555551767349243,
|
|
"logits/rejected": -1.9416917562484741,
|
|
"logps/chosen": -0.27575385570526123,
|
|
"logps/rejected": -0.3149864971637726,
|
|
"loss": 1.0164090394973755,
|
|
"loss/core": 1.0164090394973755,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.061546802520752,
|
|
"rewards/margins": 1.4506876468658447,
|
|
"rewards/rejected": 1.6108589172363281,
|
|
"step": 615
|
|
},
|
|
{
|
|
"epoch": 0.5923802699152036,
|
|
"grad_norm": 46.25,
|
|
"learning_rate": 1.343478834970121e-07,
|
|
"logits/chosen": -1.8785676956176758,
|
|
"logits/rejected": -1.8584808111190796,
|
|
"logps/chosen": -0.2844165861606598,
|
|
"logps/rejected": -0.29962268471717834,
|
|
"loss": 1.0305485725402832,
|
|
"loss/core": 1.0305485725402832,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.5849804878234863,
|
|
"rewards/margins": 1.1563302278518677,
|
|
"rewards/rejected": 1.4286500215530396,
|
|
"step": 620
|
|
},
|
|
{
|
|
"epoch": 0.5971575301564552,
|
|
"grad_norm": 11.375,
|
|
"learning_rate": 1.3007172168743852e-07,
|
|
"logits/chosen": -1.8624359369277954,
|
|
"logits/rejected": -1.852315902709961,
|
|
"logps/chosen": -0.2829623222351074,
|
|
"logps/rejected": -0.30127206444740295,
|
|
"loss": 1.0275830030441284,
|
|
"loss/core": 1.0275830030441284,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.9424110651016235,
|
|
"rewards/margins": 0.5080059170722961,
|
|
"rewards/rejected": 1.4344053268432617,
|
|
"step": 625
|
|
},
|
|
{
|
|
"epoch": 0.6019347903977069,
|
|
"grad_norm": 26.25,
|
|
"learning_rate": 1.25840659998631e-07,
|
|
"logits/chosen": -1.7425400018692017,
|
|
"logits/rejected": -1.77998948097229,
|
|
"logps/chosen": -0.29986807703971863,
|
|
"logps/rejected": -0.3102700114250183,
|
|
"loss": 1.0461167097091675,
|
|
"loss/core": 1.0461167097091675,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.9104256629943848,
|
|
"rewards/margins": 1.9246628284454346,
|
|
"rewards/rejected": 0.9857630729675293,
|
|
"step": 630
|
|
},
|
|
{
|
|
"epoch": 0.6067120506389586,
|
|
"grad_norm": 18.75,
|
|
"learning_rate": 1.2165628955985313e-07,
|
|
"logits/chosen": -1.8899660110473633,
|
|
"logits/rejected": -1.788923978805542,
|
|
"logps/chosen": -0.30364158749580383,
|
|
"logps/rejected": -0.30772846937179565,
|
|
"loss": 1.0387496948242188,
|
|
"loss/core": 1.0387496948242188,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.7164779901504517,
|
|
"rewards/margins": 0.41386109590530396,
|
|
"rewards/rejected": 1.302616834640503,
|
|
"step": 635
|
|
},
|
|
{
|
|
"epoch": 0.6114893108802102,
|
|
"grad_norm": 11.3125,
|
|
"learning_rate": 1.175201839416988e-07,
|
|
"logits/chosen": -1.7163013219833374,
|
|
"logits/rejected": -1.7982505559921265,
|
|
"logps/chosen": -0.2873172163963318,
|
|
"logps/rejected": -0.29000353813171387,
|
|
"loss": 1.036372184753418,
|
|
"loss/core": 1.036372184753418,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.6618483066558838,
|
|
"rewards/margins": 1.0304663181304932,
|
|
"rewards/rejected": 0.6313816905021667,
|
|
"step": 640
|
|
},
|
|
{
|
|
"epoch": 0.6162665711214619,
|
|
"grad_norm": 54.0,
|
|
"learning_rate": 1.1343389856433658e-07,
|
|
"logits/chosen": -1.9578911066055298,
|
|
"logits/rejected": -1.9668989181518555,
|
|
"logps/chosen": -0.2846335470676422,
|
|
"logps/rejected": -0.27758553624153137,
|
|
"loss": 1.0494635105133057,
|
|
"loss/core": 1.0494635105133057,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.0247325897216797,
|
|
"rewards/margins": 0.21105141937732697,
|
|
"rewards/rejected": 1.8136810064315796,
|
|
"step": 645
|
|
},
|
|
{
|
|
"epoch": 0.6210438313627135,
|
|
"grad_norm": 70.5,
|
|
"learning_rate": 1.0939897011258e-07,
|
|
"logits/chosen": -1.6655654907226562,
|
|
"logits/rejected": -1.7558271884918213,
|
|
"logps/chosen": -0.29102975130081177,
|
|
"logps/rejected": -0.2871955335140228,
|
|
"loss": 1.046040654182434,
|
|
"loss/core": 1.046040654182434,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.1432576179504395,
|
|
"rewards/margins": 1.0412713289260864,
|
|
"rewards/rejected": 2.1019864082336426,
|
|
"step": 650
|
|
},
|
|
{
|
|
"epoch": 0.6258210916039652,
|
|
"grad_norm": 65.0,
|
|
"learning_rate": 1.0541691595800336e-07,
|
|
"logits/chosen": -1.9179801940917969,
|
|
"logits/rejected": -1.831691026687622,
|
|
"logps/chosen": -0.2815897464752197,
|
|
"logps/rejected": -0.29244643449783325,
|
|
"loss": 1.026608943939209,
|
|
"loss/core": 1.026608943939209,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.7700217962265015,
|
|
"rewards/margins": 0.5304393768310547,
|
|
"rewards/rejected": 1.2395825386047363,
|
|
"step": 655
|
|
},
|
|
{
|
|
"epoch": 0.6305983518452167,
|
|
"grad_norm": 5.0,
|
|
"learning_rate": 1.0148923358832021e-07,
|
|
"logits/chosen": -1.7689975500106812,
|
|
"logits/rejected": -1.8483155965805054,
|
|
"logps/chosen": -0.2766217589378357,
|
|
"logps/rejected": -0.2837319076061249,
|
|
"loss": 1.0300475358963013,
|
|
"loss/core": 1.0300475358963013,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.812479019165039,
|
|
"rewards/margins": 0.43489259481430054,
|
|
"rewards/rejected": 1.3775863647460938,
|
|
"step": 660
|
|
},
|
|
{
|
|
"epoch": 0.6353756120864684,
|
|
"grad_norm": 8.1875,
|
|
"learning_rate": 9.761740004423926e-08,
|
|
"logits/chosen": -1.8706638813018799,
|
|
"logits/rejected": -1.864198088645935,
|
|
"logps/chosen": -0.2770681381225586,
|
|
"logps/rejected": -0.2515154182910919,
|
|
"loss": 1.073243498802185,
|
|
"loss/core": 1.073243498802185,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.2681612968444824,
|
|
"rewards/margins": 1.0078423023223877,
|
|
"rewards/rejected": 1.2603189945220947,
|
|
"step": 665
|
|
},
|
|
{
|
|
"epoch": 0.64015287232772,
|
|
"grad_norm": 5.125,
|
|
"learning_rate": 9.380287136400999e-08,
|
|
"logits/chosen": -1.8545506000518799,
|
|
"logits/rejected": -1.9372055530548096,
|
|
"logps/chosen": -0.2920622229576111,
|
|
"logps/rejected": -0.2942867577075958,
|
|
"loss": 1.0373417139053345,
|
|
"loss/core": 1.0373417139053345,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.3302066326141357,
|
|
"rewards/margins": 0.37538617849349976,
|
|
"rewards/rejected": 0.954820454120636,
|
|
"step": 670
|
|
},
|
|
{
|
|
"epoch": 0.6449301325689717,
|
|
"grad_norm": 19.375,
|
|
"learning_rate": 9.004708203586628e-08,
|
|
"logits/chosen": -2.0307302474975586,
|
|
"logits/rejected": -1.9755264520645142,
|
|
"logps/chosen": -0.28046339750289917,
|
|
"logps/rejected": -0.29223278164863586,
|
|
"loss": 1.0251643657684326,
|
|
"loss/core": 1.0251643657684326,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.621219277381897,
|
|
"rewards/margins": 0.32391253113746643,
|
|
"rewards/rejected": 1.2973066568374634,
|
|
"step": 675
|
|
},
|
|
{
|
|
"epoch": 0.6497073928102234,
|
|
"grad_norm": 11.6875,
|
|
"learning_rate": 8.635144445857407e-08,
|
|
"logits/chosen": -1.800567626953125,
|
|
"logits/rejected": -1.8191674947738647,
|
|
"logps/chosen": -0.2679905295372009,
|
|
"logps/rejected": -0.27527937293052673,
|
|
"loss": 1.0352510213851929,
|
|
"loss/core": 1.0352510213851929,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.176344871520996,
|
|
"rewards/margins": 1.9697097539901733,
|
|
"rewards/rejected": 1.206634759902954,
|
|
"step": 680
|
|
},
|
|
{
|
|
"epoch": 0.654484653051475,
|
|
"grad_norm": 35.25,
|
|
"learning_rate": 8.271734841028552e-08,
|
|
"logits/chosen": -1.8452231884002686,
|
|
"logits/rejected": -1.8123219013214111,
|
|
"logps/chosen": -0.2911657691001892,
|
|
"logps/rejected": -0.2622499167919159,
|
|
"loss": 1.0803478956222534,
|
|
"loss/core": 1.0803478956222534,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.31235933303833,
|
|
"rewards/margins": -0.02849596180021763,
|
|
"rewards/rejected": 2.340855121612549,
|
|
"step": 685
|
|
},
|
|
{
|
|
"epoch": 0.6592619132927267,
|
|
"grad_norm": 40.0,
|
|
"learning_rate": 7.91461605259007e-08,
|
|
"logits/chosen": -1.593219518661499,
|
|
"logits/rejected": -1.6734249591827393,
|
|
"logps/chosen": -0.31815314292907715,
|
|
"logps/rejected": -0.32347214221954346,
|
|
"loss": 1.0386189222335815,
|
|
"loss/core": 1.0386189222335815,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.939263105392456,
|
|
"rewards/margins": 0.8087685704231262,
|
|
"rewards/rejected": 2.1304945945739746,
|
|
"step": 690
|
|
},
|
|
{
|
|
"epoch": 0.6640391735339782,
|
|
"grad_norm": 18.625,
|
|
"learning_rate": 7.563922378313217e-08,
|
|
"logits/chosen": -1.778198480606079,
|
|
"logits/rejected": -1.7811349630355835,
|
|
"logps/chosen": -0.2683827579021454,
|
|
"logps/rejected": -0.2815011143684387,
|
|
"loss": 1.0239763259887695,
|
|
"loss/core": 1.0239763259887695,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.358581781387329,
|
|
"rewards/margins": 1.180909514427185,
|
|
"rewards/rejected": 1.1776721477508545,
|
|
"step": 695
|
|
},
|
|
{
|
|
"epoch": 0.6688164337752299,
|
|
"grad_norm": 29.25,
|
|
"learning_rate": 7.219785699746572e-08,
|
|
"logits/chosen": -1.9994821548461914,
|
|
"logits/rejected": -1.947238564491272,
|
|
"logps/chosen": -0.286782830953598,
|
|
"logps/rejected": -0.27130791544914246,
|
|
"loss": 1.0630942583084106,
|
|
"loss/core": 1.0630942583084106,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.2841906547546387,
|
|
"rewards/margins": 1.6503528356552124,
|
|
"rewards/rejected": 1.633837342262268,
|
|
"step": 700
|
|
},
|
|
{
|
|
"epoch": 0.6735936940164815,
|
|
"grad_norm": 4.53125,
|
|
"learning_rate": 6.882335432620779e-08,
|
|
"logits/chosen": -1.8093124628067017,
|
|
"logits/rejected": -1.754615068435669,
|
|
"logps/chosen": -0.3254457712173462,
|
|
"logps/rejected": -0.31252914667129517,
|
|
"loss": 1.0568554401397705,
|
|
"loss/core": 1.0568554401397705,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.3327386379241943,
|
|
"rewards/margins": 0.8064883947372437,
|
|
"rewards/rejected": 0.5262502431869507,
|
|
"step": 705
|
|
},
|
|
{
|
|
"epoch": 0.6783709542577332,
|
|
"grad_norm": 19.875,
|
|
"learning_rate": 6.551698478180589e-08,
|
|
"logits/chosen": -1.6687633991241455,
|
|
"logits/rejected": -1.6874847412109375,
|
|
"logps/chosen": -0.2778703570365906,
|
|
"logps/rejected": -0.28912389278411865,
|
|
"loss": 1.025848150253296,
|
|
"loss/core": 1.025848150253296,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.4708938598632812,
|
|
"rewards/margins": 1.6537853479385376,
|
|
"rewards/rejected": 0.8171082735061646,
|
|
"step": 710
|
|
},
|
|
{
|
|
"epoch": 0.6831482144989849,
|
|
"grad_norm": 9.0,
|
|
"learning_rate": 6.22799917546252e-08,
|
|
"logits/chosen": -1.8202791213989258,
|
|
"logits/rejected": -1.9110924005508423,
|
|
"logps/chosen": -0.2837766408920288,
|
|
"logps/rejected": -0.28179988265037537,
|
|
"loss": 1.0426539182662964,
|
|
"loss/core": 1.0426539182662964,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.3514044284820557,
|
|
"rewards/margins": 1.4219096899032593,
|
|
"rewards/rejected": 0.9294947385787964,
|
|
"step": 715
|
|
},
|
|
{
|
|
"epoch": 0.6879254747402365,
|
|
"grad_norm": 7.40625,
|
|
"learning_rate": 5.9113592545359944e-08,
|
|
"logits/chosen": -1.9379427433013916,
|
|
"logits/rejected": -1.941803216934204,
|
|
"logps/chosen": -0.2888217568397522,
|
|
"logps/rejected": -0.3106004297733307,
|
|
"loss": 1.0178791284561157,
|
|
"loss/core": 1.0178791284561157,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.548539638519287,
|
|
"rewards/margins": 1.4435575008392334,
|
|
"rewards/rejected": 1.1049822568893433,
|
|
"step": 720
|
|
},
|
|
{
|
|
"epoch": 0.6927027349814882,
|
|
"grad_norm": 3.9375,
|
|
"learning_rate": 5.601897790725643e-08,
|
|
"logits/chosen": -1.8667323589324951,
|
|
"logits/rejected": -1.8213609457015991,
|
|
"logps/chosen": -0.28370243310928345,
|
|
"logps/rejected": -0.2769761383533478,
|
|
"loss": 1.0477391481399536,
|
|
"loss/core": 1.0477391481399536,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.1843743324279785,
|
|
"rewards/margins": 0.06037312000989914,
|
|
"rewards/rejected": 1.124001145362854,
|
|
"step": 725
|
|
},
|
|
{
|
|
"epoch": 0.6974799952227397,
|
|
"grad_norm": 22.125,
|
|
"learning_rate": 5.299731159831952e-08,
|
|
"logits/chosen": -1.7045841217041016,
|
|
"logits/rejected": -1.7231441736221313,
|
|
"logps/chosen": -0.3042663633823395,
|
|
"logps/rejected": -0.3213575482368469,
|
|
"loss": 1.0313047170639038,
|
|
"loss/core": 1.0313047170639038,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.7923970222473145,
|
|
"rewards/margins": 1.8490235805511475,
|
|
"rewards/rejected": 0.9433733820915222,
|
|
"step": 730
|
|
},
|
|
{
|
|
"epoch": 0.7022572554639914,
|
|
"grad_norm": 44.75,
|
|
"learning_rate": 5.0049729943671013e-08,
|
|
"logits/chosen": -1.915514349937439,
|
|
"logits/rejected": -1.888715386390686,
|
|
"logps/chosen": -0.2653517723083496,
|
|
"logps/rejected": -0.27663201093673706,
|
|
"loss": 1.0275311470031738,
|
|
"loss/core": 1.0275311470031738,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.803576707839966,
|
|
"rewards/margins": 1.734165906906128,
|
|
"rewards/rejected": 1.0694106817245483,
|
|
"step": 735
|
|
},
|
|
{
|
|
"epoch": 0.707034515705243,
|
|
"grad_norm": 30.5,
|
|
"learning_rate": 4.7177341408223994e-08,
|
|
"logits/chosen": -1.6809492111206055,
|
|
"logits/rejected": -1.6404021978378296,
|
|
"logps/chosen": -0.29158157110214233,
|
|
"logps/rejected": -0.3046492338180542,
|
|
"loss": 1.0248371362686157,
|
|
"loss/core": 1.0248371362686157,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.1834161281585693,
|
|
"rewards/margins": 0.5413880348205566,
|
|
"rewards/rejected": 1.6420282125473022,
|
|
"step": 740
|
|
},
|
|
{
|
|
"epoch": 0.7118117759464947,
|
|
"grad_norm": 6.375,
|
|
"learning_rate": 4.438122617983442e-08,
|
|
"logits/chosen": -1.887942910194397,
|
|
"logits/rejected": -1.8876336812973022,
|
|
"logps/chosen": -0.2923053801059723,
|
|
"logps/rejected": -0.2827454209327698,
|
|
"loss": 1.0520001649856567,
|
|
"loss/core": 1.0520001649856567,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.3299145698547363,
|
|
"rewards/margins": 1.1022189855575562,
|
|
"rewards/rejected": 1.2276954650878906,
|
|
"step": 745
|
|
},
|
|
{
|
|
"epoch": 0.7165890361877463,
|
|
"grad_norm": 36.0,
|
|
"learning_rate": 4.1662435763087114e-08,
|
|
"logits/chosen": -1.841260552406311,
|
|
"logits/rejected": -1.7571378946304321,
|
|
"logps/chosen": -0.2794218361377716,
|
|
"logps/rejected": -0.3028506338596344,
|
|
"loss": 1.012818694114685,
|
|
"loss/core": 1.012818694114685,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.2234444618225098,
|
|
"rewards/margins": 0.845853328704834,
|
|
"rewards/rejected": 1.3775911331176758,
|
|
"step": 750
|
|
},
|
|
{
|
|
"epoch": 0.721366296428998,
|
|
"grad_norm": 5.46875,
|
|
"learning_rate": 3.902199258386732e-08,
|
|
"logits/chosen": -1.8437637090682983,
|
|
"logits/rejected": -1.8173859119415283,
|
|
"logps/chosen": -0.27545225620269775,
|
|
"logps/rejected": -0.2872912287712097,
|
|
"loss": 1.0292212963104248,
|
|
"loss/core": 1.0292212963104248,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.603247880935669,
|
|
"rewards/margins": 1.7060693502426147,
|
|
"rewards/rejected": 1.8971784114837646,
|
|
"step": 755
|
|
},
|
|
{
|
|
"epoch": 0.7261435566702497,
|
|
"grad_norm": 6.96875,
|
|
"learning_rate": 3.646088960486862e-08,
|
|
"logits/chosen": -1.6895129680633545,
|
|
"logits/rejected": -1.7431033849716187,
|
|
"logps/chosen": -0.3080654442310333,
|
|
"logps/rejected": -0.3540648818016052,
|
|
"loss": 1.0325762033462524,
|
|
"loss/core": 1.0325762033462524,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.53886079788208,
|
|
"rewards/margins": 1.5185824632644653,
|
|
"rewards/rejected": 2.020278215408325,
|
|
"step": 760
|
|
},
|
|
{
|
|
"epoch": 0.7309208169115012,
|
|
"grad_norm": 8.4375,
|
|
"learning_rate": 3.398008995217988e-08,
|
|
"logits/chosen": -1.964421033859253,
|
|
"logits/rejected": -1.953046441078186,
|
|
"logps/chosen": -0.2779175639152527,
|
|
"logps/rejected": -0.26899924874305725,
|
|
"loss": 1.0525156259536743,
|
|
"loss/core": 1.0525156259536743,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.6339603662490845,
|
|
"rewards/margins": 0.7983501553535461,
|
|
"rewards/rejected": 0.8356102705001831,
|
|
"step": 765
|
|
},
|
|
{
|
|
"epoch": 0.7356980771527529,
|
|
"grad_norm": 108.0,
|
|
"learning_rate": 3.1580526553093315e-08,
|
|
"logits/chosen": -1.877881407737732,
|
|
"logits/rejected": -1.9419214725494385,
|
|
"logps/chosen": -0.28730475902557373,
|
|
"logps/rejected": -0.2802114188671112,
|
|
"loss": 1.0498204231262207,
|
|
"loss/core": 1.0498204231262207,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.2819581031799316,
|
|
"rewards/margins": 1.5368332862854004,
|
|
"rewards/rejected": 0.7451248168945312,
|
|
"step": 770
|
|
},
|
|
{
|
|
"epoch": 0.7404753373940045,
|
|
"grad_norm": 5.125,
|
|
"learning_rate": 2.9263101785268252e-08,
|
|
"logits/chosen": -1.890252709388733,
|
|
"logits/rejected": -1.786828637123108,
|
|
"logps/chosen": -0.27267634868621826,
|
|
"logps/rejected": -0.2676258087158203,
|
|
"loss": 1.0456647872924805,
|
|
"loss/core": 1.0456647872924805,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.8578557968139648,
|
|
"rewards/margins": 0.4590323567390442,
|
|
"rewards/rejected": 1.3988234996795654,
|
|
"step": 775
|
|
},
|
|
{
|
|
"epoch": 0.7452525976352562,
|
|
"grad_norm": 268.0,
|
|
"learning_rate": 2.7028687137384264e-08,
|
|
"logits/chosen": -1.9074313640594482,
|
|
"logits/rejected": -1.8822380304336548,
|
|
"logps/chosen": -0.2915792465209961,
|
|
"logps/rejected": -0.293667197227478,
|
|
"loss": 1.037200927734375,
|
|
"loss/core": 1.037200927734375,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.0063960552215576,
|
|
"rewards/margins": 1.0704381465911865,
|
|
"rewards/rejected": 1.935957670211792,
|
|
"step": 780
|
|
},
|
|
{
|
|
"epoch": 0.7500298578765078,
|
|
"grad_norm": 9.6875,
|
|
"learning_rate": 2.4878122881409447e-08,
|
|
"logits/chosen": -1.8461768627166748,
|
|
"logits/rejected": -1.8658325672149658,
|
|
"logps/chosen": -0.2820442020893097,
|
|
"logps/rejected": -0.2832142412662506,
|
|
"loss": 1.0385828018188477,
|
|
"loss/core": 1.0385828018188477,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.9002225399017334,
|
|
"rewards/margins": 1.1294481754302979,
|
|
"rewards/rejected": 0.7707743644714355,
|
|
"step": 785
|
|
},
|
|
{
|
|
"epoch": 0.7548071181177595,
|
|
"grad_norm": 36.25,
|
|
"learning_rate": 2.2812217756608937e-08,
|
|
"logits/chosen": -1.825455904006958,
|
|
"logits/rejected": -1.8060061931610107,
|
|
"logps/chosen": -0.2775079011917114,
|
|
"logps/rejected": -0.27117595076560974,
|
|
"loss": 1.0483758449554443,
|
|
"loss/core": 1.0483758449554443,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.307039260864258,
|
|
"rewards/margins": 2.0355687141418457,
|
|
"rewards/rejected": 1.271470546722412,
|
|
"step": 790
|
|
},
|
|
{
|
|
"epoch": 0.7595843783590112,
|
|
"grad_norm": 61.5,
|
|
"learning_rate": 2.0831748665410763e-08,
|
|
"logits/chosen": -1.770493745803833,
|
|
"logits/rejected": -1.7957115173339844,
|
|
"logps/chosen": -0.27306801080703735,
|
|
"logps/rejected": -0.28165358304977417,
|
|
"loss": 1.0311884880065918,
|
|
"loss/core": 1.0311884880065918,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.751323699951172,
|
|
"rewards/margins": 2.812521457672119,
|
|
"rewards/rejected": 0.9388027191162109,
|
|
"step": 795
|
|
},
|
|
{
|
|
"epoch": 0.7643616386002627,
|
|
"grad_norm": 10.5,
|
|
"learning_rate": 1.8937460381244967e-08,
|
|
"logits/chosen": -2.075711727142334,
|
|
"logits/rejected": -2.082383394241333,
|
|
"logps/chosen": -0.2964426577091217,
|
|
"logps/rejected": -0.30977433919906616,
|
|
"loss": 1.0251314640045166,
|
|
"loss/core": 1.0251314640045166,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.5174245834350586,
|
|
"rewards/margins": 1.0785468816757202,
|
|
"rewards/rejected": 1.4388777017593384,
|
|
"step": 800
|
|
},
|
|
{
|
|
"epoch": 0.7691388988415144,
|
|
"grad_norm": 7.65625,
|
|
"learning_rate": 1.713006526846439e-08,
|
|
"logits/chosen": -1.8648141622543335,
|
|
"logits/rejected": -1.8252522945404053,
|
|
"logps/chosen": -0.30121514201164246,
|
|
"logps/rejected": -0.3006502687931061,
|
|
"loss": 1.0409377813339233,
|
|
"loss/core": 1.0409377813339233,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 0.9633139371871948,
|
|
"rewards/margins": 0.3292653262615204,
|
|
"rewards/rejected": 0.634048581123352,
|
|
"step": 805
|
|
},
|
|
{
|
|
"epoch": 0.773916159082766,
|
|
"grad_norm": 28.125,
|
|
"learning_rate": 1.541024301445404e-08,
|
|
"logits/chosen": -1.6336872577667236,
|
|
"logits/rejected": -1.643832802772522,
|
|
"logps/chosen": -0.26690006256103516,
|
|
"logps/rejected": -0.26295873522758484,
|
|
"loss": 1.0444200038909912,
|
|
"loss/core": 1.0444200038909912,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.8206228017807007,
|
|
"rewards/margins": 0.1702648401260376,
|
|
"rewards/rejected": 1.6503578424453735,
|
|
"step": 810
|
|
},
|
|
{
|
|
"epoch": 0.7786934193240177,
|
|
"grad_norm": 5.84375,
|
|
"learning_rate": 1.3778640374027983e-08,
|
|
"logits/chosen": -1.751861333847046,
|
|
"logits/rejected": -1.9031435251235962,
|
|
"logps/chosen": -0.3171994984149933,
|
|
"logps/rejected": -0.3077738881111145,
|
|
"loss": 1.0605608224868774,
|
|
"loss/core": 1.0605608224868774,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.4510459899902344,
|
|
"rewards/margins": 1.3974026441574097,
|
|
"rewards/rejected": 1.0536432266235352,
|
|
"step": 815
|
|
},
|
|
{
|
|
"epoch": 0.7834706795652693,
|
|
"grad_norm": 19.0,
|
|
"learning_rate": 1.2235870926211616e-08,
|
|
"logits/chosen": -1.6588127613067627,
|
|
"logits/rejected": -1.6749852895736694,
|
|
"logps/chosen": -0.27939319610595703,
|
|
"logps/rejected": -0.3374519348144531,
|
|
"loss": 1.0130990743637085,
|
|
"loss/core": 1.0130990743637085,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.6268069744110107,
|
|
"rewards/margins": 0.6854991912841797,
|
|
"rewards/rejected": 1.9413076639175415,
|
|
"step": 820
|
|
},
|
|
{
|
|
"epoch": 0.788247939806521,
|
|
"grad_norm": 13.375,
|
|
"learning_rate": 1.0782514843499652e-08,
|
|
"logits/chosen": -1.7541849613189697,
|
|
"logits/rejected": -1.7715661525726318,
|
|
"logps/chosen": -0.29441070556640625,
|
|
"logps/rejected": -0.27596911787986755,
|
|
"loss": 1.0648963451385498,
|
|
"loss/core": 1.0648963451385498,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.5321407318115234,
|
|
"rewards/margins": 1.8472797870635986,
|
|
"rewards/rejected": 1.6848608255386353,
|
|
"step": 825
|
|
},
|
|
{
|
|
"epoch": 0.7930252000477725,
|
|
"grad_norm": 78.5,
|
|
"learning_rate": 9.419118673676924e-09,
|
|
"logits/chosen": -1.563860535621643,
|
|
"logits/rejected": -1.5621966123580933,
|
|
"logps/chosen": -0.3032872676849365,
|
|
"logps/rejected": -0.3065972328186035,
|
|
"loss": 1.0367621183395386,
|
|
"loss/core": 1.0367621183395386,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.807102680206299,
|
|
"rewards/margins": 1.6108064651489258,
|
|
"rewards/rejected": 1.1962958574295044,
|
|
"step": 830
|
|
},
|
|
{
|
|
"epoch": 0.7978024602890242,
|
|
"grad_norm": 6.03125,
|
|
"learning_rate": 8.14619513428405e-09,
|
|
"logits/chosen": -1.904526948928833,
|
|
"logits/rejected": -1.9740642309188843,
|
|
"logps/chosen": -0.29926085472106934,
|
|
"logps/rejected": -0.2885436713695526,
|
|
"loss": 1.054502248764038,
|
|
"loss/core": 1.054502248764038,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.6957647800445557,
|
|
"rewards/margins": 0.6506364941596985,
|
|
"rewards/rejected": 1.045128345489502,
|
|
"step": 835
|
|
},
|
|
{
|
|
"epoch": 0.8025797205302759,
|
|
"grad_norm": 9.9375,
|
|
"learning_rate": 6.96422291980539e-09,
|
|
"logits/chosen": -1.8588794469833374,
|
|
"logits/rejected": -1.90227472782135,
|
|
"logps/chosen": -0.2894556224346161,
|
|
"logps/rejected": -0.3027735650539398,
|
|
"loss": 1.02797532081604,
|
|
"loss/core": 1.02797532081604,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.031310796737671,
|
|
"rewards/margins": 0.43263769149780273,
|
|
"rewards/rejected": 1.5986733436584473,
|
|
"step": 840
|
|
},
|
|
{
|
|
"epoch": 0.8073569807715275,
|
|
"grad_norm": 17.25,
|
|
"learning_rate": 5.8736465216517594e-09,
|
|
"logits/chosen": -1.9031187295913696,
|
|
"logits/rejected": -1.9344947338104248,
|
|
"logps/chosen": -0.2742392420768738,
|
|
"logps/rejected": -0.2774720788002014,
|
|
"loss": 1.0499933958053589,
|
|
"loss/core": 1.0499933958053589,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.593908786773682,
|
|
"rewards/margins": 2.4118456840515137,
|
|
"rewards/rejected": 2.1820626258850098,
|
|
"step": 845
|
|
},
|
|
{
|
|
"epoch": 0.8121342410127792,
|
|
"grad_norm": 4.09375,
|
|
"learning_rate": 4.874876061005173e-09,
|
|
"logits/chosen": -1.8513599634170532,
|
|
"logits/rejected": -1.8989410400390625,
|
|
"logps/chosen": -0.2543167173862457,
|
|
"logps/rejected": -0.25180160999298096,
|
|
"loss": 1.0425068140029907,
|
|
"loss/core": 1.0425068140029907,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.7452361583709717,
|
|
"rewards/margins": 1.0755412578582764,
|
|
"rewards/rejected": 0.6696950197219849,
|
|
"step": 850
|
|
},
|
|
{
|
|
"epoch": 0.8169115012540308,
|
|
"grad_norm": 25.875,
|
|
"learning_rate": 3.968287134589188e-09,
|
|
"logits/chosen": -1.7955166101455688,
|
|
"logits/rejected": -1.8299312591552734,
|
|
"logps/chosen": -0.2676030695438385,
|
|
"logps/rejected": -0.2964601516723633,
|
|
"loss": 1.004606008529663,
|
|
"loss/core": 1.004606008529663,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.3115034103393555,
|
|
"rewards/margins": 0.41639384627342224,
|
|
"rewards/rejected": 1.8951095342636108,
|
|
"step": 855
|
|
},
|
|
{
|
|
"epoch": 0.8216887614952825,
|
|
"grad_norm": 5.375,
|
|
"learning_rate": 3.154220673422192e-09,
|
|
"logits/chosen": -1.8242244720458984,
|
|
"logits/rejected": -1.871952772140503,
|
|
"logps/chosen": -0.29963549971580505,
|
|
"logps/rejected": -0.3072584271430969,
|
|
"loss": 1.0296491384506226,
|
|
"loss/core": 1.0296491384506226,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.9225000143051147,
|
|
"rewards/margins": 0.8448880910873413,
|
|
"rewards/rejected": 1.0776121616363525,
|
|
"step": 860
|
|
},
|
|
{
|
|
"epoch": 0.826466021736534,
|
|
"grad_norm": 7.28125,
|
|
"learning_rate": 2.4329828146074096e-09,
|
|
"logits/chosen": -1.9124653339385986,
|
|
"logits/rejected": -2.0408012866973877,
|
|
"logps/chosen": -0.29619625210762024,
|
|
"logps/rejected": -0.2827211022377014,
|
|
"loss": 1.0616817474365234,
|
|
"loss/core": 1.0616817474365234,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.6343491077423096,
|
|
"rewards/margins": 1.424903392791748,
|
|
"rewards/rejected": 1.2094457149505615,
|
|
"step": 865
|
|
},
|
|
{
|
|
"epoch": 0.8312432819777857,
|
|
"grad_norm": 19.5,
|
|
"learning_rate": 1.8048447862070714e-09,
|
|
"logits/chosen": -1.5945602655410767,
|
|
"logits/rejected": -1.6320714950561523,
|
|
"logps/chosen": -0.2955728769302368,
|
|
"logps/rejected": -0.2850645184516907,
|
|
"loss": 1.061391830444336,
|
|
"loss/core": 1.061391830444336,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.463152885437012,
|
|
"rewards/margins": 2.405036449432373,
|
|
"rewards/rejected": 2.0581166744232178,
|
|
"step": 870
|
|
},
|
|
{
|
|
"epoch": 0.8360205422190374,
|
|
"grad_norm": 4.5,
|
|
"learning_rate": 1.2700428052447033e-09,
|
|
"logits/chosen": -1.7374908924102783,
|
|
"logits/rejected": -1.7853012084960938,
|
|
"logps/chosen": -0.3029218316078186,
|
|
"logps/rejected": -0.3070649802684784,
|
|
"loss": 1.0413432121276855,
|
|
"loss/core": 1.0413432121276855,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.9537794589996338,
|
|
"rewards/margins": 0.5955628156661987,
|
|
"rewards/rejected": 1.358216643333435,
|
|
"step": 875
|
|
},
|
|
{
|
|
"epoch": 0.840797802460289,
|
|
"grad_norm": 3.453125,
|
|
"learning_rate": 8.287779888734858e-10,
|
|
"logits/chosen": -1.7615150213241577,
|
|
"logits/rejected": -1.707476019859314,
|
|
"logps/chosen": -0.28448665142059326,
|
|
"logps/rejected": -0.3107626140117645,
|
|
"loss": 1.0223973989486694,
|
|
"loss/core": 1.0223973989486694,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.715723752975464,
|
|
"rewards/margins": 2.2797560691833496,
|
|
"rewards/rejected": 1.4359674453735352,
|
|
"step": 880
|
|
},
|
|
{
|
|
"epoch": 0.8455750627015407,
|
|
"grad_norm": 18.0,
|
|
"learning_rate": 4.812162787445062e-10,
|
|
"logits/chosen": -1.8650376796722412,
|
|
"logits/rejected": -1.8496296405792236,
|
|
"logps/chosen": -0.2981775403022766,
|
|
"logps/rejected": -0.30601340532302856,
|
|
"loss": 1.029161810874939,
|
|
"loss/core": 1.029161810874939,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.5521029233932495,
|
|
"rewards/margins": 0.7568989992141724,
|
|
"rewards/rejected": 0.7952039241790771,
|
|
"step": 885
|
|
},
|
|
{
|
|
"epoch": 0.8503523229427923,
|
|
"grad_norm": 11.4375,
|
|
"learning_rate": 2.2748837860270265e-10,
|
|
"logits/chosen": -1.8946914672851562,
|
|
"logits/rejected": -1.9093029499053955,
|
|
"logps/chosen": -0.2697398066520691,
|
|
"logps/rejected": -0.2905231714248657,
|
|
"loss": 1.019055962562561,
|
|
"loss/core": 1.019055962562561,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.8530833721160889,
|
|
"rewards/margins": 0.24989423155784607,
|
|
"rewards/rejected": 1.6031888723373413,
|
|
"step": 890
|
|
},
|
|
{
|
|
"epoch": 0.855129583184044,
|
|
"grad_norm": 65.0,
|
|
"learning_rate": 6.768970513457151e-11,
|
|
"logits/chosen": -1.7693462371826172,
|
|
"logits/rejected": -1.7002862691879272,
|
|
"logps/chosen": -0.2899450659751892,
|
|
"logps/rejected": -0.2910805940628052,
|
|
"loss": 1.038063645362854,
|
|
"loss/core": 1.038063645362854,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.1362929344177246,
|
|
"rewards/margins": 1.1503578424453735,
|
|
"rewards/rejected": 0.9859352111816406,
|
|
"step": 895
|
|
},
|
|
{
|
|
"epoch": 0.8599068434252956,
|
|
"grad_norm": 34.0,
|
|
"learning_rate": 1.8803520859811406e-12,
|
|
"logits/chosen": -1.7765915393829346,
|
|
"logits/rejected": -1.7241567373275757,
|
|
"logps/chosen": -0.2680931091308594,
|
|
"logps/rejected": -0.2755570411682129,
|
|
"loss": 1.0350791215896606,
|
|
"loss/core": 1.0350791215896606,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.9140121936798096,
|
|
"rewards/margins": 1.5131105184555054,
|
|
"rewards/rejected": 1.4009015560150146,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.8599068434252956,
|
|
"step": 900,
|
|
"total_flos": 0.0,
|
|
"train_loss": 1.0422146828969319,
|
|
"train_runtime": 8158.2235,
|
|
"train_samples_per_second": 1.765,
|
|
"train_steps_per_second": 0.11
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 900,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 900,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 2,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|