2924 lines
101 KiB
JSON
2924 lines
101 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 0.8599068434252956,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 900,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"epoch": 0.004777260241251642,
|
||
|
|
"grad_norm": 2.296875,
|
||
|
|
"learning_rate": 2.222222222222222e-08,
|
||
|
|
"logits/chosen": -2.3184335231781006,
|
||
|
|
"logits/rejected": -2.304084300994873,
|
||
|
|
"logps/chosen": -0.31140023469924927,
|
||
|
|
"logps/rejected": -0.29028692841529846,
|
||
|
|
"loss": 0.3946344256401062,
|
||
|
|
"loss/core": 0.3946344256401062,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.209475040435791,
|
||
|
|
"rewards/margins": 1.5164874792099,
|
||
|
|
"rewards/rejected": 0.6929876208305359,
|
||
|
|
"step": 5
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.009554520482503284,
|
||
|
|
"grad_norm": 183.0,
|
||
|
|
"learning_rate": 5e-08,
|
||
|
|
"logits/chosen": -2.064580202102661,
|
||
|
|
"logits/rejected": -2.0450222492218018,
|
||
|
|
"logps/chosen": -0.28674912452697754,
|
||
|
|
"logps/rejected": -0.3005256652832031,
|
||
|
|
"loss": 1.1264296770095825,
|
||
|
|
"loss/core": 1.1264296770095825,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.9985833168029785,
|
||
|
|
"rewards/margins": 0.33723002672195435,
|
||
|
|
"rewards/rejected": 2.66135311126709,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.014331780723754926,
|
||
|
|
"grad_norm": 12.625,
|
||
|
|
"learning_rate": 7.777777777777778e-08,
|
||
|
|
"logits/chosen": -2.228989601135254,
|
||
|
|
"logits/rejected": -2.2271804809570312,
|
||
|
|
"logps/chosen": -0.2594456374645233,
|
||
|
|
"logps/rejected": -0.2722489833831787,
|
||
|
|
"loss": 0.26212042570114136,
|
||
|
|
"loss/core": 0.26212042570114136,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.754608154296875,
|
||
|
|
"rewards/margins": 0.7076910734176636,
|
||
|
|
"rewards/rejected": 1.046917200088501,
|
||
|
|
"step": 15
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.01910904096500657,
|
||
|
|
"grad_norm": 4.59375,
|
||
|
|
"learning_rate": 1.0555555555555555e-07,
|
||
|
|
"logits/chosen": -2.1681175231933594,
|
||
|
|
"logits/rejected": -2.185380458831787,
|
||
|
|
"logps/chosen": -0.2745749354362488,
|
||
|
|
"logps/rejected": -0.26860952377319336,
|
||
|
|
"loss": 1.016000747680664,
|
||
|
|
"loss/core": 1.016000747680664,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.6250548362731934,
|
||
|
|
"rewards/margins": 2.429143190383911,
|
||
|
|
"rewards/rejected": 1.1959115266799927,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.02388630120625821,
|
||
|
|
"grad_norm": 4.40625,
|
||
|
|
"learning_rate": 1.3333333333333334e-07,
|
||
|
|
"logits/chosen": -2.010061740875244,
|
||
|
|
"logits/rejected": -2.1256415843963623,
|
||
|
|
"logps/chosen": -0.27929872274398804,
|
||
|
|
"logps/rejected": -0.2702064514160156,
|
||
|
|
"loss": 0.17983713746070862,
|
||
|
|
"loss/core": 0.17983713746070862,
|
||
|
|
"rewards/accuracies": 0.9750000238418579,
|
||
|
|
"rewards/chosen": 2.1464362144470215,
|
||
|
|
"rewards/margins": 1.1600453853607178,
|
||
|
|
"rewards/rejected": 0.9863905906677246,
|
||
|
|
"step": 25
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.028663561447509853,
|
||
|
|
"grad_norm": 5.78125,
|
||
|
|
"learning_rate": 1.611111111111111e-07,
|
||
|
|
"logits/chosen": -2.1331052780151367,
|
||
|
|
"logits/rejected": -2.1351206302642822,
|
||
|
|
"logps/chosen": -0.27982744574546814,
|
||
|
|
"logps/rejected": -0.26132339239120483,
|
||
|
|
"loss": 0.5742586851119995,
|
||
|
|
"loss/core": 0.5742586851119995,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.813447952270508,
|
||
|
|
"rewards/margins": 1.750083327293396,
|
||
|
|
"rewards/rejected": 1.0633647441864014,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.033440821688761495,
|
||
|
|
"grad_norm": 1.53125,
|
||
|
|
"learning_rate": 1.8888888888888888e-07,
|
||
|
|
"logits/chosen": -2.273294687271118,
|
||
|
|
"logits/rejected": -2.2828454971313477,
|
||
|
|
"logps/chosen": -0.2884656488895416,
|
||
|
|
"logps/rejected": -0.2767728865146637,
|
||
|
|
"loss": 0.8869408369064331,
|
||
|
|
"loss/core": 0.8869408369064331,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.5600028038024902,
|
||
|
|
"rewards/margins": 1.78105890750885,
|
||
|
|
"rewards/rejected": 0.7789438366889954,
|
||
|
|
"step": 35
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.03821808193001314,
|
||
|
|
"grad_norm": 27.75,
|
||
|
|
"learning_rate": 2.1666666666666667e-07,
|
||
|
|
"logits/chosen": -2.2540197372436523,
|
||
|
|
"logits/rejected": -2.351133108139038,
|
||
|
|
"logps/chosen": -0.2658698558807373,
|
||
|
|
"logps/rejected": -0.2907714247703552,
|
||
|
|
"loss": 0.6574953198432922,
|
||
|
|
"loss/core": 0.6574953198432922,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.926403522491455,
|
||
|
|
"rewards/margins": 2.233344554901123,
|
||
|
|
"rewards/rejected": 0.6930587887763977,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.04299534217126478,
|
||
|
|
"grad_norm": 4.46875,
|
||
|
|
"learning_rate": 2.4444444444444445e-07,
|
||
|
|
"logits/chosen": -2.090873956680298,
|
||
|
|
"logits/rejected": -2.14512300491333,
|
||
|
|
"logps/chosen": -0.27041956782341003,
|
||
|
|
"logps/rejected": -0.27646249532699585,
|
||
|
|
"loss": 0.8163895606994629,
|
||
|
|
"loss/core": 0.8163895606994629,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 3.0477185249328613,
|
||
|
|
"rewards/margins": 1.7669315338134766,
|
||
|
|
"rewards/rejected": 1.2807869911193848,
|
||
|
|
"step": 45
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.04777260241251642,
|
||
|
|
"grad_norm": 66.0,
|
||
|
|
"learning_rate": 2.7222222222222216e-07,
|
||
|
|
"logits/chosen": -2.188749074935913,
|
||
|
|
"logits/rejected": -2.2616398334503174,
|
||
|
|
"logps/chosen": -0.2877681851387024,
|
||
|
|
"logps/rejected": -0.26972633600234985,
|
||
|
|
"loss": 0.21321019530296326,
|
||
|
|
"loss/core": 0.21321019530296326,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 1.6582177877426147,
|
||
|
|
"rewards/margins": 0.44566529989242554,
|
||
|
|
"rewards/rejected": 1.2125524282455444,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.05254986265376806,
|
||
|
|
"grad_norm": 236.0,
|
||
|
|
"learning_rate": 3e-07,
|
||
|
|
"logits/chosen": -2.0755667686462402,
|
||
|
|
"logits/rejected": -2.1377971172332764,
|
||
|
|
"logps/chosen": -0.28878724575042725,
|
||
|
|
"logps/rejected": -0.28834742307662964,
|
||
|
|
"loss": 0.5084811449050903,
|
||
|
|
"loss/core": 0.5084811449050903,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.339972496032715,
|
||
|
|
"rewards/margins": 1.6370515823364258,
|
||
|
|
"rewards/rejected": 0.7029210329055786,
|
||
|
|
"step": 55
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.057327122895019705,
|
||
|
|
"grad_norm": 121.0,
|
||
|
|
"learning_rate": 3.2777777777777776e-07,
|
||
|
|
"logits/chosen": -2.1755266189575195,
|
||
|
|
"logits/rejected": -2.122792959213257,
|
||
|
|
"logps/chosen": -0.2767593562602997,
|
||
|
|
"logps/rejected": -0.2760574519634247,
|
||
|
|
"loss": 1.5027045011520386,
|
||
|
|
"loss/core": 1.5027045011520386,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.761892318725586,
|
||
|
|
"rewards/margins": 2.0544276237487793,
|
||
|
|
"rewards/rejected": 1.707464575767517,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.06210438313627135,
|
||
|
|
"grad_norm": 264.0,
|
||
|
|
"learning_rate": 3.5555555555555553e-07,
|
||
|
|
"logits/chosen": -2.1052839756011963,
|
||
|
|
"logits/rejected": -2.1663551330566406,
|
||
|
|
"logps/chosen": -0.2984526455402374,
|
||
|
|
"logps/rejected": -0.28494197130203247,
|
||
|
|
"loss": 1.0875555276870728,
|
||
|
|
"loss/core": 1.0875555276870728,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 3.2381019592285156,
|
||
|
|
"rewards/margins": 2.0958991050720215,
|
||
|
|
"rewards/rejected": 1.1422029733657837,
|
||
|
|
"step": 65
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.06688164337752299,
|
||
|
|
"grad_norm": 16.0,
|
||
|
|
"learning_rate": 3.8333333333333335e-07,
|
||
|
|
"logits/chosen": -2.134707450866699,
|
||
|
|
"logits/rejected": -2.046159267425537,
|
||
|
|
"logps/chosen": -0.2798652648925781,
|
||
|
|
"logps/rejected": -0.27488410472869873,
|
||
|
|
"loss": 0.4238152503967285,
|
||
|
|
"loss/core": 0.4238152503967285,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.4914584159851074,
|
||
|
|
"rewards/margins": 1.004212498664856,
|
||
|
|
"rewards/rejected": 1.4872459173202515,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07165890361877464,
|
||
|
|
"grad_norm": 1.65625,
|
||
|
|
"learning_rate": 4.1111111111111107e-07,
|
||
|
|
"logits/chosen": -2.0351903438568115,
|
||
|
|
"logits/rejected": -2.068025588989258,
|
||
|
|
"logps/chosen": -0.29988136887550354,
|
||
|
|
"logps/rejected": -0.3065613806247711,
|
||
|
|
"loss": 0.5185372829437256,
|
||
|
|
"loss/core": 0.5185372829437256,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.2677860260009766,
|
||
|
|
"rewards/margins": 1.2009799480438232,
|
||
|
|
"rewards/rejected": 1.0668058395385742,
|
||
|
|
"step": 75
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.07643616386002627,
|
||
|
|
"grad_norm": 0.8359375,
|
||
|
|
"learning_rate": 4.3888888888888884e-07,
|
||
|
|
"logits/chosen": -2.2651379108428955,
|
||
|
|
"logits/rejected": -2.265864849090576,
|
||
|
|
"logps/chosen": -0.2954421639442444,
|
||
|
|
"logps/rejected": -0.2877275347709656,
|
||
|
|
"loss": 0.18425166606903076,
|
||
|
|
"loss/core": 0.18425166606903076,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.3546873331069946,
|
||
|
|
"rewards/margins": 0.4060265123844147,
|
||
|
|
"rewards/rejected": 0.9486608505249023,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.08121342410127792,
|
||
|
|
"grad_norm": 744.0,
|
||
|
|
"learning_rate": 4.6666666666666666e-07,
|
||
|
|
"logits/chosen": -2.2166881561279297,
|
||
|
|
"logits/rejected": -2.2303507328033447,
|
||
|
|
"logps/chosen": -0.27470141649246216,
|
||
|
|
"logps/rejected": -0.2800918519496918,
|
||
|
|
"loss": 0.5587457418441772,
|
||
|
|
"loss/core": 0.5587457418441772,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 1.6758487224578857,
|
||
|
|
"rewards/margins": 0.03965097665786743,
|
||
|
|
"rewards/rejected": 1.6361976861953735,
|
||
|
|
"step": 85
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.08599068434252956,
|
||
|
|
"grad_norm": 15.6875,
|
||
|
|
"learning_rate": 4.944444444444445e-07,
|
||
|
|
"logits/chosen": -2.1294827461242676,
|
||
|
|
"logits/rejected": -2.2263193130493164,
|
||
|
|
"logps/chosen": -0.27022016048431396,
|
||
|
|
"logps/rejected": -0.31300783157348633,
|
||
|
|
"loss": 0.8455947041511536,
|
||
|
|
"loss/core": 0.8455947041511536,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.5631749629974365,
|
||
|
|
"rewards/margins": 1.0474013090133667,
|
||
|
|
"rewards/rejected": 1.515773892402649,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.09076794458378121,
|
||
|
|
"grad_norm": 22.0,
|
||
|
|
"learning_rate": 4.999699149323369e-07,
|
||
|
|
"logits/chosen": -2.137500524520874,
|
||
|
|
"logits/rejected": -2.064415454864502,
|
||
|
|
"logps/chosen": -0.28990474343299866,
|
||
|
|
"logps/rejected": -0.2970162034034729,
|
||
|
|
"loss": 0.4857604503631592,
|
||
|
|
"loss/core": 0.4857604503631592,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.2846360206604004,
|
||
|
|
"rewards/margins": 1.2684682607650757,
|
||
|
|
"rewards/rejected": 1.0161678791046143,
|
||
|
|
"step": 95
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.09554520482503284,
|
||
|
|
"grad_norm": 4.4375,
|
||
|
|
"learning_rate": 4.998477067547739e-07,
|
||
|
|
"logits/chosen": -2.11228346824646,
|
||
|
|
"logits/rejected": -2.1553733348846436,
|
||
|
|
"logps/chosen": -0.26913318037986755,
|
||
|
|
"logps/rejected": -0.2833011746406555,
|
||
|
|
"loss": 1.1113026142120361,
|
||
|
|
"loss/core": 1.1113026142120361,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.711238145828247,
|
||
|
|
"rewards/margins": 2.09954833984375,
|
||
|
|
"rewards/rejected": 1.611689567565918,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10032246506628449,
|
||
|
|
"grad_norm": 24.25,
|
||
|
|
"learning_rate": 4.996315410727229e-07,
|
||
|
|
"logits/chosen": -2.0640909671783447,
|
||
|
|
"logits/rejected": -2.0353593826293945,
|
||
|
|
"logps/chosen": -0.317560613155365,
|
||
|
|
"logps/rejected": -0.304330050945282,
|
||
|
|
"loss": 0.93501216173172,
|
||
|
|
"loss/core": 0.93501216173172,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 3.3537240028381348,
|
||
|
|
"rewards/margins": 2.5598361492156982,
|
||
|
|
"rewards/rejected": 0.7938874959945679,
|
||
|
|
"step": 105
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10509972530753613,
|
||
|
|
"grad_norm": 1.7890625,
|
||
|
|
"learning_rate": 4.993214991772562e-07,
|
||
|
|
"logits/chosen": -2.1243863105773926,
|
||
|
|
"logits/rejected": -2.126587390899658,
|
||
|
|
"logps/chosen": -0.2978929877281189,
|
||
|
|
"logps/rejected": -0.3149787485599518,
|
||
|
|
"loss": 0.20066983997821808,
|
||
|
|
"loss/core": 0.20066983997821808,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.6585355997085571,
|
||
|
|
"rewards/margins": 0.7344213724136353,
|
||
|
|
"rewards/rejected": 0.9241144061088562,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.10987698554878778,
|
||
|
|
"grad_norm": 10.8125,
|
||
|
|
"learning_rate": 4.989176976624511e-07,
|
||
|
|
"logits/chosen": -2.0023255348205566,
|
||
|
|
"logits/rejected": -2.1182520389556885,
|
||
|
|
"logps/chosen": -0.30129915475845337,
|
||
|
|
"logps/rejected": -0.29388052225112915,
|
||
|
|
"loss": 1.1868757009506226,
|
||
|
|
"loss/core": 1.1868757009506226,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.8971168994903564,
|
||
|
|
"rewards/margins": 1.7273696660995483,
|
||
|
|
"rewards/rejected": 1.1697468757629395,
|
||
|
|
"step": 115
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.11465424579003941,
|
||
|
|
"grad_norm": 51.0,
|
||
|
|
"learning_rate": 4.984202883815428e-07,
|
||
|
|
"logits/chosen": -2.2406866550445557,
|
||
|
|
"logits/rejected": -2.252915620803833,
|
||
|
|
"logps/chosen": -0.30535244941711426,
|
||
|
|
"logps/rejected": -0.30678510665893555,
|
||
|
|
"loss": 0.48086509108543396,
|
||
|
|
"loss/core": 0.48086509108543396,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.3497025966644287,
|
||
|
|
"rewards/margins": 1.1640708446502686,
|
||
|
|
"rewards/rejected": 1.1856318712234497,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.11943150603129106,
|
||
|
|
"grad_norm": 87.0,
|
||
|
|
"learning_rate": 4.978294583898195e-07,
|
||
|
|
"logits/chosen": -2.0294809341430664,
|
||
|
|
"logits/rejected": -1.9394248723983765,
|
||
|
|
"logps/chosen": -0.29434821009635925,
|
||
|
|
"logps/rejected": -0.3001564145088196,
|
||
|
|
"loss": 0.9685723185539246,
|
||
|
|
"loss/core": 0.9685723185539246,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 1.8707126379013062,
|
||
|
|
"rewards/margins": -0.364132821559906,
|
||
|
|
"rewards/rejected": 2.2348453998565674,
|
||
|
|
"step": 125
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1242087662725427,
|
||
|
|
"grad_norm": 58.5,
|
||
|
|
"learning_rate": 4.971454298742779e-07,
|
||
|
|
"logits/chosen": -2.348689079284668,
|
||
|
|
"logits/rejected": -2.265820026397705,
|
||
|
|
"logps/chosen": -0.27629876136779785,
|
||
|
|
"logps/rejected": -0.302666574716568,
|
||
|
|
"loss": 0.6233381032943726,
|
||
|
|
"loss/core": 0.6233381032943726,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.2184367179870605,
|
||
|
|
"rewards/margins": 0.2606551945209503,
|
||
|
|
"rewards/rejected": 1.9577815532684326,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.12898602651379434,
|
||
|
|
"grad_norm": 11.0625,
|
||
|
|
"learning_rate": 4.963684600700678e-07,
|
||
|
|
"logits/chosen": -2.0810961723327637,
|
||
|
|
"logits/rejected": -2.177532434463501,
|
||
|
|
"logps/chosen": -0.34504374861717224,
|
||
|
|
"logps/rejected": -0.3136570155620575,
|
||
|
|
"loss": 0.6652131676673889,
|
||
|
|
"loss/core": 0.6652131676673889,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.7010741233825684,
|
||
|
|
"rewards/margins": 1.2287189960479736,
|
||
|
|
"rewards/rejected": 1.4723551273345947,
|
||
|
|
"step": 135
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.13376328675504598,
|
||
|
|
"grad_norm": 57.75,
|
||
|
|
"learning_rate": 4.954988411637571e-07,
|
||
|
|
"logits/chosen": -1.9434146881103516,
|
||
|
|
"logits/rejected": -1.9310106039047241,
|
||
|
|
"logps/chosen": -0.3319775462150574,
|
||
|
|
"logps/rejected": -0.34657081961631775,
|
||
|
|
"loss": 1.2185124158859253,
|
||
|
|
"loss/core": 1.2185124158859253,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 3.100647449493408,
|
||
|
|
"rewards/margins": 0.7573388814926147,
|
||
|
|
"rewards/rejected": 2.343308925628662,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.13854054699629761,
|
||
|
|
"grad_norm": 6.4375,
|
||
|
|
"learning_rate": 4.945369001834514e-07,
|
||
|
|
"logits/chosen": -2.0375614166259766,
|
||
|
|
"logits/rejected": -2.0471043586730957,
|
||
|
|
"logps/chosen": -0.3008027672767639,
|
||
|
|
"logps/rejected": -0.28672724962234497,
|
||
|
|
"loss": 0.20575180649757385,
|
||
|
|
"loss/core": 0.20575180649757385,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.7885154485702515,
|
||
|
|
"rewards/margins": 0.952106773853302,
|
||
|
|
"rewards/rejected": 0.8364084362983704,
|
||
|
|
"step": 145
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.14331780723754928,
|
||
|
|
"grad_norm": 211.0,
|
||
|
|
"learning_rate": 4.93482998875813e-07,
|
||
|
|
"logits/chosen": -2.004568576812744,
|
||
|
|
"logits/rejected": -2.028262138366699,
|
||
|
|
"logps/chosen": -0.2854091227054596,
|
||
|
|
"logps/rejected": -0.27151355147361755,
|
||
|
|
"loss": 0.6710211038589478,
|
||
|
|
"loss/core": 0.6710211038589478,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.637166738510132,
|
||
|
|
"rewards/margins": 1.3770954608917236,
|
||
|
|
"rewards/rejected": 1.2600712776184082,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.1480950674788009,
|
||
|
|
"grad_norm": 5.90625,
|
||
|
|
"learning_rate": 4.923375335700223e-07,
|
||
|
|
"logits/chosen": -1.968898057937622,
|
||
|
|
"logits/rejected": -2.025012731552124,
|
||
|
|
"logps/chosen": -0.2497386932373047,
|
||
|
|
"logps/rejected": -0.2664481997489929,
|
||
|
|
"loss": 0.38974815607070923,
|
||
|
|
"loss/core": 0.38974815607070923,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.618589401245117,
|
||
|
|
"rewards/margins": 1.195473551750183,
|
||
|
|
"rewards/rejected": 1.423115849494934,
|
||
|
|
"step": 155
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.15287232772005255,
|
||
|
|
"grad_norm": 58.5,
|
||
|
|
"learning_rate": 4.911009350287347e-07,
|
||
|
|
"logits/chosen": -2.1655495166778564,
|
||
|
|
"logits/rejected": -2.139359951019287,
|
||
|
|
"logps/chosen": -0.27988573908805847,
|
||
|
|
"logps/rejected": -0.2946188449859619,
|
||
|
|
"loss": 0.5953173041343689,
|
||
|
|
"loss/core": 0.5953173041343689,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.644726276397705,
|
||
|
|
"rewards/margins": 1.5004968643188477,
|
||
|
|
"rewards/rejected": 1.144229531288147,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.15764958796130418,
|
||
|
|
"grad_norm": 5.3125,
|
||
|
|
"learning_rate": 4.897736682860885e-07,
|
||
|
|
"logits/chosen": -2.109489917755127,
|
||
|
|
"logits/rejected": -2.0843923091888428,
|
||
|
|
"logps/chosen": -0.3071993589401245,
|
||
|
|
"logps/rejected": -0.30547797679901123,
|
||
|
|
"loss": 0.1960802972316742,
|
||
|
|
"loss/core": 0.1960802972316742,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 1.7830066680908203,
|
||
|
|
"rewards/margins": 0.7959617376327515,
|
||
|
|
"rewards/rejected": 0.9870448112487793,
|
||
|
|
"step": 165
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.16242684820255585,
|
||
|
|
"grad_norm": 1.2578125,
|
||
|
|
"learning_rate": 4.883562324728241e-07,
|
||
|
|
"logits/chosen": -2.0669140815734863,
|
||
|
|
"logits/rejected": -1.9952644109725952,
|
||
|
|
"logps/chosen": -0.3106909692287445,
|
||
|
|
"logps/rejected": -0.3281523883342743,
|
||
|
|
"loss": 0.20468628406524658,
|
||
|
|
"loss/core": 0.20468628406524658,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.8208732604980469,
|
||
|
|
"rewards/margins": 1.240056037902832,
|
||
|
|
"rewards/rejected": 0.5808171033859253,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.16720410844380748,
|
||
|
|
"grad_norm": 70.5,
|
||
|
|
"learning_rate": 4.868491606285823e-07,
|
||
|
|
"logits/chosen": -2.2352006435394287,
|
||
|
|
"logits/rejected": -2.256232261657715,
|
||
|
|
"logps/chosen": -0.27431878447532654,
|
||
|
|
"logps/rejected": -0.28181517124176025,
|
||
|
|
"loss": 0.1919003427028656,
|
||
|
|
"loss/core": 0.1919003427028656,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.784128189086914,
|
||
|
|
"rewards/margins": 1.083038568496704,
|
||
|
|
"rewards/rejected": 0.7010896801948547,
|
||
|
|
"step": 175
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.17198136868505912,
|
||
|
|
"grad_norm": 14.6875,
|
||
|
|
"learning_rate": 4.852530195014489e-07,
|
||
|
|
"logits/chosen": -2.1959755420684814,
|
||
|
|
"logits/rejected": -2.148374080657959,
|
||
|
|
"logps/chosen": -0.2624792456626892,
|
||
|
|
"logps/rejected": -0.2835710942745209,
|
||
|
|
"loss": 0.6657207608222961,
|
||
|
|
"loss/core": 0.6657207608222961,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.457181692123413,
|
||
|
|
"rewards/margins": 1.1070095300674438,
|
||
|
|
"rewards/rejected": 1.3501724004745483,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.17675862892631075,
|
||
|
|
"grad_norm": 632.0,
|
||
|
|
"learning_rate": 4.835684093348244e-07,
|
||
|
|
"logits/chosen": -2.2028822898864746,
|
||
|
|
"logits/rejected": -2.2440648078918457,
|
||
|
|
"logps/chosen": -0.3046368956565857,
|
||
|
|
"logps/rejected": -0.3071895241737366,
|
||
|
|
"loss": 0.6476051807403564,
|
||
|
|
"loss/core": 0.6476051807403564,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 1.8896329402923584,
|
||
|
|
"rewards/margins": 0.37541091442108154,
|
||
|
|
"rewards/rejected": 1.5142220258712769,
|
||
|
|
"step": 185
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18153588916756241,
|
||
|
|
"grad_norm": 436.0,
|
||
|
|
"learning_rate": 4.817959636416969e-07,
|
||
|
|
"logits/chosen": -2.134538412094116,
|
||
|
|
"logits/rejected": -2.1165084838867188,
|
||
|
|
"logps/chosen": -0.2866383194923401,
|
||
|
|
"logps/rejected": -0.27144110202789307,
|
||
|
|
"loss": 0.6611530780792236,
|
||
|
|
"loss/core": 0.6611530780792236,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.6436305046081543,
|
||
|
|
"rewards/margins": 1.4646612405776978,
|
||
|
|
"rewards/rejected": 1.178969144821167,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.18631314940881405,
|
||
|
|
"grad_norm": 6.3125,
|
||
|
|
"learning_rate": 4.799363489664039e-07,
|
||
|
|
"logits/chosen": -1.9559043645858765,
|
||
|
|
"logits/rejected": -1.9407514333724976,
|
||
|
|
"logps/chosen": -0.28489893674850464,
|
||
|
|
"logps/rejected": -0.2946854531764984,
|
||
|
|
"loss": 0.2528533637523651,
|
||
|
|
"loss/core": 0.2528533637523651,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.681348204612732,
|
||
|
|
"rewards/margins": 0.3501024842262268,
|
||
|
|
"rewards/rejected": 1.33124577999115,
|
||
|
|
"step": 195
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.19109040965006568,
|
||
|
|
"grad_norm": 8.6875,
|
||
|
|
"learning_rate": 4.779902646339721e-07,
|
||
|
|
"logits/chosen": -2.1071603298187256,
|
||
|
|
"logits/rejected": -2.1380481719970703,
|
||
|
|
"logps/chosen": -0.2741830050945282,
|
||
|
|
"logps/rejected": -0.27496692538261414,
|
||
|
|
"loss": 0.46219339966773987,
|
||
|
|
"loss/core": 0.46219339966773987,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.5044608116149902,
|
||
|
|
"rewards/margins": 1.1724002361297607,
|
||
|
|
"rewards/rejected": 1.33206045627594,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.19586766989131732,
|
||
|
|
"grad_norm": 3.203125,
|
||
|
|
"learning_rate": 4.759584424871301e-07,
|
||
|
|
"logits/chosen": -2.0774598121643066,
|
||
|
|
"logits/rejected": -2.142284631729126,
|
||
|
|
"logps/chosen": -0.2829883396625519,
|
||
|
|
"logps/rejected": -0.29093074798583984,
|
||
|
|
"loss": 0.08258922398090363,
|
||
|
|
"loss/core": 0.08258922398090363,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.3614203929901123,
|
||
|
|
"rewards/margins": 0.7309275269508362,
|
||
|
|
"rewards/rejected": 0.6304928660392761,
|
||
|
|
"step": 205
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20064493013256898,
|
||
|
|
"grad_norm": 10.9375,
|
||
|
|
"learning_rate": 4.738416466110917e-07,
|
||
|
|
"logits/chosen": -2.0302059650421143,
|
||
|
|
"logits/rejected": -2.0086560249328613,
|
||
|
|
"logps/chosen": -0.288141131401062,
|
||
|
|
"logps/rejected": -0.2887657582759857,
|
||
|
|
"loss": 0.8564616441726685,
|
||
|
|
"loss/core": 0.8564616441726685,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.526529550552368,
|
||
|
|
"rewards/margins": 1.0289502143859863,
|
||
|
|
"rewards/rejected": 1.4975792169570923,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.20542219037382062,
|
||
|
|
"grad_norm": 10.8125,
|
||
|
|
"learning_rate": 4.716406730462153e-07,
|
||
|
|
"logits/chosen": -1.9449901580810547,
|
||
|
|
"logits/rejected": -1.9634069204330444,
|
||
|
|
"logps/chosen": -0.29736560583114624,
|
||
|
|
"logps/rejected": -0.2745818793773651,
|
||
|
|
"loss": 0.24143390357494354,
|
||
|
|
"loss/core": 0.24143390357494354,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.0462050437927246,
|
||
|
|
"rewards/margins": 1.0101492404937744,
|
||
|
|
"rewards/rejected": 1.0360558032989502,
|
||
|
|
"step": 215
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.21019945061507225,
|
||
|
|
"grad_norm": 3.0,
|
||
|
|
"learning_rate": 4.693563494886454e-07,
|
||
|
|
"logits/chosen": -2.025031089782715,
|
||
|
|
"logits/rejected": -2.0499961376190186,
|
||
|
|
"logps/chosen": -0.28308767080307007,
|
||
|
|
"logps/rejected": -0.27393776178359985,
|
||
|
|
"loss": 0.16485320031642914,
|
||
|
|
"loss/core": 0.16485320031642914,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.761115312576294,
|
||
|
|
"rewards/margins": 0.9114633798599243,
|
||
|
|
"rewards/rejected": 0.849652111530304,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2149767108563239,
|
||
|
|
"grad_norm": 15.3125,
|
||
|
|
"learning_rate": 4.6698953497905016e-07,
|
||
|
|
"logits/chosen": -2.0364017486572266,
|
||
|
|
"logits/rejected": -2.0577664375305176,
|
||
|
|
"logps/chosen": -0.31471285223960876,
|
||
|
|
"logps/rejected": -0.30929452180862427,
|
||
|
|
"loss": 1.460294485092163,
|
||
|
|
"loss/core": 1.460294485092163,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 3.0360703468322754,
|
||
|
|
"rewards/margins": 1.0577287673950195,
|
||
|
|
"rewards/rejected": 1.9783413410186768,
|
||
|
|
"step": 225
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.21975397109757555,
|
||
|
|
"grad_norm": 54.5,
|
||
|
|
"learning_rate": 4.645411195795709e-07,
|
||
|
|
"logits/chosen": -2.118978500366211,
|
||
|
|
"logits/rejected": -2.1093850135803223,
|
||
|
|
"logps/chosen": -0.27341052889823914,
|
||
|
|
"logps/rejected": -0.26687008142471313,
|
||
|
|
"loss": 0.740382194519043,
|
||
|
|
"loss/core": 0.740382194519043,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 3.039541006088257,
|
||
|
|
"rewards/margins": 1.364072561264038,
|
||
|
|
"rewards/rejected": 1.6754686832427979,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2245312313388272,
|
||
|
|
"grad_norm": 2.84375,
|
||
|
|
"learning_rate": 4.6201202403910643e-07,
|
||
|
|
"logits/chosen": -2.160707950592041,
|
||
|
|
"logits/rejected": -2.255814790725708,
|
||
|
|
"logps/chosen": -0.28612154722213745,
|
||
|
|
"logps/rejected": -0.28075844049453735,
|
||
|
|
"loss": 0.19919511675834656,
|
||
|
|
"loss/core": 0.19919511675834656,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 1.6981313228607178,
|
||
|
|
"rewards/margins": 1.023926854133606,
|
||
|
|
"rewards/rejected": 0.6742045879364014,
|
||
|
|
"step": 235
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.22930849158007882,
|
||
|
|
"grad_norm": 7.90625,
|
||
|
|
"learning_rate": 4.594031994470573e-07,
|
||
|
|
"logits/chosen": -2.0647361278533936,
|
||
|
|
"logits/rejected": -2.1183581352233887,
|
||
|
|
"logps/chosen": -0.2742571234703064,
|
||
|
|
"logps/rejected": -0.2913045287132263,
|
||
|
|
"loss": 1.243248701095581,
|
||
|
|
"loss/core": 1.243248701095581,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.7567174434661865,
|
||
|
|
"rewards/margins": 0.8206822276115417,
|
||
|
|
"rewards/rejected": 1.936035394668579,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.23408575182133046,
|
||
|
|
"grad_norm": 1112.0,
|
||
|
|
"learning_rate": 4.567156268756593e-07,
|
||
|
|
"logits/chosen": -2.120326519012451,
|
||
|
|
"logits/rejected": -2.2658631801605225,
|
||
|
|
"logps/chosen": -0.30770888924598694,
|
||
|
|
"logps/rejected": -0.292914479970932,
|
||
|
|
"loss": 0.6964072585105896,
|
||
|
|
"loss/core": 0.6964072585105896,
|
||
|
|
"rewards/accuracies": 0.9750000238418579,
|
||
|
|
"rewards/chosen": 2.1233694553375244,
|
||
|
|
"rewards/margins": 1.4673677682876587,
|
||
|
|
"rewards/rejected": 0.6560018062591553,
|
||
|
|
"step": 245
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.23886301206258212,
|
||
|
|
"grad_norm": 10.625,
|
||
|
|
"learning_rate": 4.5395031701104303e-07,
|
||
|
|
"logits/chosen": -1.9557708501815796,
|
||
|
|
"logits/rejected": -1.9600833654403687,
|
||
|
|
"logps/chosen": -0.30549654364585876,
|
||
|
|
"logps/rejected": -0.3048430383205414,
|
||
|
|
"loss": 1.1260910034179688,
|
||
|
|
"loss/core": 1.1260910034179688,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 3.3460769653320312,
|
||
|
|
"rewards/margins": 1.7832248210906982,
|
||
|
|
"rewards/rejected": 1.562852144241333,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.24364027230383375,
|
||
|
|
"grad_norm": 4.09375,
|
||
|
|
"learning_rate": 4.511083097731555e-07,
|
||
|
|
"logits/chosen": -2.1089088916778564,
|
||
|
|
"logits/rejected": -2.0778651237487793,
|
||
|
|
"logps/chosen": -0.287789523601532,
|
||
|
|
"logps/rejected": -0.2891516089439392,
|
||
|
|
"loss": 1.2482858896255493,
|
||
|
|
"loss/core": 1.2482858896255493,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.746084213256836,
|
||
|
|
"rewards/margins": 1.564002275466919,
|
||
|
|
"rewards/rejected": 1.1820822954177856,
|
||
|
|
"step": 255
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2484175325450854,
|
||
|
|
"grad_norm": 108.0,
|
||
|
|
"learning_rate": 4.481906739246894e-07,
|
||
|
|
"logits/chosen": -2.062340259552002,
|
||
|
|
"logits/rejected": -2.0389857292175293,
|
||
|
|
"logps/chosen": -0.3061924874782562,
|
||
|
|
"logps/rejected": -0.30993199348449707,
|
||
|
|
"loss": 0.4140895903110504,
|
||
|
|
"loss/core": 0.4140895903110504,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.9362844228744507,
|
||
|
|
"rewards/margins": 0.6997807025909424,
|
||
|
|
"rewards/rejected": 1.2365038394927979,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.25319479278633705,
|
||
|
|
"grad_norm": 2.8125,
|
||
|
|
"learning_rate": 4.451985066691648e-07,
|
||
|
|
"logits/chosen": -2.0385899543762207,
|
||
|
|
"logits/rejected": -2.0162370204925537,
|
||
|
|
"logps/chosen": -0.27738499641418457,
|
||
|
|
"logps/rejected": -0.28945407271385193,
|
||
|
|
"loss": 0.23780469596385956,
|
||
|
|
"loss/core": 0.23780469596385956,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.11801815032959,
|
||
|
|
"rewards/margins": 1.098084568977356,
|
||
|
|
"rewards/rejected": 1.0199334621429443,
|
||
|
|
"step": 265
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2579720530275887,
|
||
|
|
"grad_norm": 1.4375,
|
||
|
|
"learning_rate": 4.421329332383158e-07,
|
||
|
|
"logits/chosen": -2.0890111923217773,
|
||
|
|
"logits/rejected": -2.0384275913238525,
|
||
|
|
"logps/chosen": -0.30751222372055054,
|
||
|
|
"logps/rejected": -0.30553263425827026,
|
||
|
|
"loss": 0.3220127820968628,
|
||
|
|
"loss/core": 0.3220127820968628,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.6447738409042358,
|
||
|
|
"rewards/margins": 0.24662013351917267,
|
||
|
|
"rewards/rejected": 1.3981537818908691,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2627493132688403,
|
||
|
|
"grad_norm": 3.703125,
|
||
|
|
"learning_rate": 4.3899510646893696e-07,
|
||
|
|
"logits/chosen": -2.207843780517578,
|
||
|
|
"logits/rejected": -2.2784366607666016,
|
||
|
|
"logps/chosen": -0.2971876561641693,
|
||
|
|
"logps/rejected": -0.2956683933734894,
|
||
|
|
"loss": 0.05450323969125748,
|
||
|
|
"loss/core": 0.05450323969125748,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.0277817249298096,
|
||
|
|
"rewards/margins": 0.44771966338157654,
|
||
|
|
"rewards/rejected": 0.5800620317459106,
|
||
|
|
"step": 275
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.26752657351009196,
|
||
|
|
"grad_norm": 23.0,
|
||
|
|
"learning_rate": 4.357862063693485e-07,
|
||
|
|
"logits/chosen": -2.0210609436035156,
|
||
|
|
"logits/rejected": -1.9996906518936157,
|
||
|
|
"logps/chosen": -0.3016383945941925,
|
||
|
|
"logps/rejected": -0.3017963767051697,
|
||
|
|
"loss": 0.21795885264873505,
|
||
|
|
"loss/core": 0.21795885264873505,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.8103902339935303,
|
||
|
|
"rewards/margins": 0.9846366047859192,
|
||
|
|
"rewards/rejected": 0.825753390789032,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2723038337513436,
|
||
|
|
"grad_norm": 8.5625,
|
||
|
|
"learning_rate": 4.3250743967564364e-07,
|
||
|
|
"logits/chosen": -2.2049059867858887,
|
||
|
|
"logits/rejected": -2.1598610877990723,
|
||
|
|
"logps/chosen": -0.2876811623573303,
|
||
|
|
"logps/rejected": -0.2909473478794098,
|
||
|
|
"loss": 1.0883954763412476,
|
||
|
|
"loss/core": 1.0883954763412476,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.9615416526794434,
|
||
|
|
"rewards/margins": 1.0239375829696655,
|
||
|
|
"rewards/rejected": 1.9376041889190674,
|
||
|
|
"step": 285
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.27708109399259523,
|
||
|
|
"grad_norm": 1056.0,
|
||
|
|
"learning_rate": 4.29160039397884e-07,
|
||
|
|
"logits/chosen": -2.1312131881713867,
|
||
|
|
"logits/rejected": -2.1842284202575684,
|
||
|
|
"logps/chosen": -0.2860090136528015,
|
||
|
|
"logps/rejected": -0.2702611982822418,
|
||
|
|
"loss": 2.010714292526245,
|
||
|
|
"loss/core": 2.010714292526245,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 4.503174781799316,
|
||
|
|
"rewards/margins": 3.201117992401123,
|
||
|
|
"rewards/rejected": 1.3020570278167725,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.28185835423384686,
|
||
|
|
"grad_norm": 10.0625,
|
||
|
|
"learning_rate": 4.2574526435641546e-07,
|
||
|
|
"logits/chosen": -1.899706244468689,
|
||
|
|
"logits/rejected": -1.976220726966858,
|
||
|
|
"logps/chosen": -0.3136875629425049,
|
||
|
|
"logps/rejected": -0.302241712808609,
|
||
|
|
"loss": 1.1247197389602661,
|
||
|
|
"loss/core": 1.1247197389602661,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.7293989658355713,
|
||
|
|
"rewards/margins": 1.6034202575683594,
|
||
|
|
"rewards/rejected": 1.125978708267212,
|
||
|
|
"step": 295
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.28663561447509855,
|
||
|
|
"grad_norm": 5.375,
|
||
|
|
"learning_rate": 4.22264398708477e-07,
|
||
|
|
"logits/chosen": -2.0837273597717285,
|
||
|
|
"logits/rejected": -2.1485800743103027,
|
||
|
|
"logps/chosen": -0.28113582730293274,
|
||
|
|
"logps/rejected": -0.3067568838596344,
|
||
|
|
"loss": 2.029283046722412,
|
||
|
|
"loss/core": 2.029283046722412,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 3.4929652214050293,
|
||
|
|
"rewards/margins": 1.580517053604126,
|
||
|
|
"rewards/rejected": 1.9124481678009033,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2914128747163502,
|
||
|
|
"grad_norm": 45.25,
|
||
|
|
"learning_rate": 4.187187514652819e-07,
|
||
|
|
"logits/chosen": -1.9658424854278564,
|
||
|
|
"logits/rejected": -2.029053211212158,
|
||
|
|
"logps/chosen": -0.3088286519050598,
|
||
|
|
"logps/rejected": -0.30893000960350037,
|
||
|
|
"loss": 2.0511975288391113,
|
||
|
|
"loss/core": 2.0511975288391113,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.7278835773468018,
|
||
|
|
"rewards/margins": 1.2690778970718384,
|
||
|
|
"rewards/rejected": 2.458805561065674,
|
||
|
|
"step": 305
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.2961901349576018,
|
||
|
|
"grad_norm": 32.0,
|
||
|
|
"learning_rate": 4.151096559997519e-07,
|
||
|
|
"logits/chosen": -1.9535623788833618,
|
||
|
|
"logits/rejected": -1.9904283285140991,
|
||
|
|
"logps/chosen": -0.2780826687812805,
|
||
|
|
"logps/rejected": -0.2801373302936554,
|
||
|
|
"loss": 0.36180371046066284,
|
||
|
|
"loss/core": 0.36180371046066284,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.327801465988159,
|
||
|
|
"rewards/margins": 0.9045225977897644,
|
||
|
|
"rewards/rejected": 1.423278570175171,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.30096739519885346,
|
||
|
|
"grad_norm": 29.375,
|
||
|
|
"learning_rate": 4.114384695450905e-07,
|
||
|
|
"logits/chosen": -2.104870080947876,
|
||
|
|
"logits/rejected": -2.0882060527801514,
|
||
|
|
"logps/chosen": -0.3096676766872406,
|
||
|
|
"logps/rejected": -0.303961843252182,
|
||
|
|
"loss": 0.32151490449905396,
|
||
|
|
"loss/core": 0.32151490449905396,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.1817171573638916,
|
||
|
|
"rewards/margins": -0.025614654645323753,
|
||
|
|
"rewards/rejected": 1.207331895828247,
|
||
|
|
"step": 315
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3057446554401051,
|
||
|
|
"grad_norm": 812.0,
|
||
|
|
"learning_rate": 4.077065726843828e-07,
|
||
|
|
"logits/chosen": -1.9013550281524658,
|
||
|
|
"logits/rejected": -1.8669452667236328,
|
||
|
|
"logps/chosen": -0.3241393268108368,
|
||
|
|
"logps/rejected": -0.3211979269981384,
|
||
|
|
"loss": 1.0354502201080322,
|
||
|
|
"loss/core": 1.0354502201080322,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 3.091705799102783,
|
||
|
|
"rewards/margins": 1.3894522190093994,
|
||
|
|
"rewards/rejected": 1.7022536993026733,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.31052191568135673,
|
||
|
|
"grad_norm": 1.5234375,
|
||
|
|
"learning_rate": 4.039153688314145e-07,
|
||
|
|
"logits/chosen": -1.993546485900879,
|
||
|
|
"logits/rejected": -2.022549867630005,
|
||
|
|
"logps/chosen": -0.28880229592323303,
|
||
|
|
"logps/rejected": -0.2948676645755768,
|
||
|
|
"loss": 0.12079276889562607,
|
||
|
|
"loss/core": 0.12079276889562607,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 1.4271008968353271,
|
||
|
|
"rewards/margins": 0.6435669660568237,
|
||
|
|
"rewards/rejected": 0.7835339903831482,
|
||
|
|
"step": 325
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.31529917592260837,
|
||
|
|
"grad_norm": 836.0,
|
||
|
|
"learning_rate": 4.0006628370290613e-07,
|
||
|
|
"logits/chosen": -2.167954444885254,
|
||
|
|
"logits/rejected": -2.0930395126342773,
|
||
|
|
"logps/chosen": -0.3180980980396271,
|
||
|
|
"logps/rejected": -0.3477565050125122,
|
||
|
|
"loss": 0.3206109404563904,
|
||
|
|
"loss/core": 0.3206109404563904,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.3264482021331787,
|
||
|
|
"rewards/margins": 0.24738340079784393,
|
||
|
|
"rewards/rejected": 1.0790647268295288,
|
||
|
|
"step": 330
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.32007643616386,
|
||
|
|
"grad_norm": 3.53125,
|
||
|
|
"learning_rate": 3.9616076478235826e-07,
|
||
|
|
"logits/chosen": -2.2583556175231934,
|
||
|
|
"logits/rejected": -2.293337106704712,
|
||
|
|
"logps/chosen": -0.30557915568351746,
|
||
|
|
"logps/rejected": -0.30756452679634094,
|
||
|
|
"loss": 0.0511607900261879,
|
||
|
|
"loss/core": 0.0511607900261879,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.0766465663909912,
|
||
|
|
"rewards/margins": 0.5183663964271545,
|
||
|
|
"rewards/rejected": 0.5582801699638367,
|
||
|
|
"step": 335
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3248536964051117,
|
||
|
|
"grad_norm": 5.65625,
|
||
|
|
"learning_rate": 3.922002807757129e-07,
|
||
|
|
"logits/chosen": -2.2513046264648438,
|
||
|
|
"logits/rejected": -2.259812593460083,
|
||
|
|
"logps/chosen": -0.28524649143218994,
|
||
|
|
"logps/rejected": -0.283276230096817,
|
||
|
|
"loss": 0.631820797920227,
|
||
|
|
"loss/core": 0.631820797920227,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.1910934448242188,
|
||
|
|
"rewards/margins": 1.2970854043960571,
|
||
|
|
"rewards/rejected": 0.8940078616142273,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3296309566463633,
|
||
|
|
"grad_norm": 79.0,
|
||
|
|
"learning_rate": 3.8818632105903315e-07,
|
||
|
|
"logits/chosen": -2.1031250953674316,
|
||
|
|
"logits/rejected": -2.102226734161377,
|
||
|
|
"logps/chosen": -0.26754727959632874,
|
||
|
|
"logps/rejected": -0.28109580278396606,
|
||
|
|
"loss": 0.7250244617462158,
|
||
|
|
"loss/core": 0.7250244617462158,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.1810359954833984,
|
||
|
|
"rewards/margins": 0.5469441413879395,
|
||
|
|
"rewards/rejected": 1.6340919733047485,
|
||
|
|
"step": 345
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.33440821688761496,
|
||
|
|
"grad_norm": 11.6875,
|
||
|
|
"learning_rate": 3.841203951184094e-07,
|
||
|
|
"logits/chosen": -2.2044973373413086,
|
||
|
|
"logits/rejected": -2.2186226844787598,
|
||
|
|
"logps/chosen": -0.3000710904598236,
|
||
|
|
"logps/rejected": -0.30985429883003235,
|
||
|
|
"loss": 0.6811296939849854,
|
||
|
|
"loss/core": 0.6811296939849854,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.313596487045288,
|
||
|
|
"rewards/margins": 0.9291432499885559,
|
||
|
|
"rewards/rejected": 1.3844534158706665,
|
||
|
|
"step": 350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3391854771288666,
|
||
|
|
"grad_norm": 31.25,
|
||
|
|
"learning_rate": 3.800040319823038e-07,
|
||
|
|
"logits/chosen": -2.172675848007202,
|
||
|
|
"logits/rejected": -2.1876015663146973,
|
||
|
|
"logps/chosen": -0.291890025138855,
|
||
|
|
"logps/rejected": -0.2729564905166626,
|
||
|
|
"loss": 0.8517450094223022,
|
||
|
|
"loss/core": 0.8517450094223022,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 3.446463108062744,
|
||
|
|
"rewards/margins": 1.6567842960357666,
|
||
|
|
"rewards/rejected": 1.7896785736083984,
|
||
|
|
"step": 355
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.34396273737011823,
|
||
|
|
"grad_norm": 89.0,
|
||
|
|
"learning_rate": 3.75838779646545e-07,
|
||
|
|
"logits/chosen": -1.9489272832870483,
|
||
|
|
"logits/rejected": -1.9853594303131104,
|
||
|
|
"logps/chosen": -0.2767919600009918,
|
||
|
|
"logps/rejected": -0.279407799243927,
|
||
|
|
"loss": 0.7631909251213074,
|
||
|
|
"loss/core": 0.7631909251213074,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 3.232623338699341,
|
||
|
|
"rewards/margins": 1.9174954891204834,
|
||
|
|
"rewards/rejected": 1.3151280879974365,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.34873999761136987,
|
||
|
|
"grad_norm": 40.5,
|
||
|
|
"learning_rate": 3.7162620449218993e-07,
|
||
|
|
"logits/chosen": -2.235380172729492,
|
||
|
|
"logits/rejected": -2.306373119354248,
|
||
|
|
"logps/chosen": -0.3116162121295929,
|
||
|
|
"logps/rejected": -0.306151807308197,
|
||
|
|
"loss": 0.5377756357192993,
|
||
|
|
"loss/core": 0.5377756357192993,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.417541265487671,
|
||
|
|
"rewards/margins": 1.4117828607559204,
|
||
|
|
"rewards/rejected": 1.0057584047317505,
|
||
|
|
"step": 365
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3535172578526215,
|
||
|
|
"grad_norm": 22.0,
|
||
|
|
"learning_rate": 3.673678906964727e-07,
|
||
|
|
"logits/chosen": -2.117574691772461,
|
||
|
|
"logits/rejected": -2.0456650257110596,
|
||
|
|
"logps/chosen": -0.2936505675315857,
|
||
|
|
"logps/rejected": -0.30493101477622986,
|
||
|
|
"loss": 0.7472105622291565,
|
||
|
|
"loss/core": 0.7472105622291565,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.3911571502685547,
|
||
|
|
"rewards/margins": 0.7618893384933472,
|
||
|
|
"rewards/rejected": 1.629267692565918,
|
||
|
|
"step": 370
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.35829451809387314,
|
||
|
|
"grad_norm": 7.15625,
|
||
|
|
"learning_rate": 3.6306543963705936e-07,
|
||
|
|
"logits/chosen": -1.9923632144927979,
|
||
|
|
"logits/rejected": -1.9175786972045898,
|
||
|
|
"logps/chosen": -0.2919101119041443,
|
||
|
|
"logps/rejected": -0.28915977478027344,
|
||
|
|
"loss": 1.7186968326568604,
|
||
|
|
"loss/core": 1.7186968326568604,
|
||
|
|
"rewards/accuracies": 0.824999988079071,
|
||
|
|
"rewards/chosen": 3.748534679412842,
|
||
|
|
"rewards/margins": 2.381950855255127,
|
||
|
|
"rewards/rejected": 1.3665834665298462,
|
||
|
|
"step": 375
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.36307177833512483,
|
||
|
|
"grad_norm": 47.0,
|
||
|
|
"learning_rate": 3.5872046928983623e-07,
|
||
|
|
"logits/chosen": -1.9712073802947998,
|
||
|
|
"logits/rejected": -2.0564160346984863,
|
||
|
|
"logps/chosen": -0.28708744049072266,
|
||
|
|
"logps/rejected": -0.308341920375824,
|
||
|
|
"loss": 1.9703826904296875,
|
||
|
|
"loss/core": 1.9703826904296875,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 3.5351765155792236,
|
||
|
|
"rewards/margins": 1.9188482761383057,
|
||
|
|
"rewards/rejected": 1.616328239440918,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.36784903857637646,
|
||
|
|
"grad_norm": 286.0,
|
||
|
|
"learning_rate": 3.5433461362045447e-07,
|
||
|
|
"logits/chosen": -2.1837782859802246,
|
||
|
|
"logits/rejected": -2.109261989593506,
|
||
|
|
"logps/chosen": -0.3028949201107025,
|
||
|
|
"logps/rejected": -0.3316383957862854,
|
||
|
|
"loss": 1.2228304147720337,
|
||
|
|
"loss/core": 1.2228304147720337,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 3.1072325706481934,
|
||
|
|
"rewards/margins": 1.2266871929168701,
|
||
|
|
"rewards/rejected": 1.8805453777313232,
|
||
|
|
"step": 385
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3726262988176281,
|
||
|
|
"grad_norm": 2.671875,
|
||
|
|
"learning_rate": 3.4990952196986305e-07,
|
||
|
|
"logits/chosen": -2.0442094802856445,
|
||
|
|
"logits/rejected": -2.072903633117676,
|
||
|
|
"logps/chosen": -0.2997892498970032,
|
||
|
|
"logps/rejected": -0.30010247230529785,
|
||
|
|
"loss": 0.475435346364975,
|
||
|
|
"loss/core": 0.475435346364975,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 2.0823211669921875,
|
||
|
|
"rewards/margins": 1.195578694343567,
|
||
|
|
"rewards/rejected": 0.8867424726486206,
|
||
|
|
"step": 390
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.37740355905887973,
|
||
|
|
"grad_norm": 7.5625,
|
||
|
|
"learning_rate": 3.4544685843405875e-07,
|
||
|
|
"logits/chosen": -2.156062364578247,
|
||
|
|
"logits/rejected": -2.2315170764923096,
|
||
|
|
"logps/chosen": -0.2756079435348511,
|
||
|
|
"logps/rejected": -0.2708030045032501,
|
||
|
|
"loss": 0.1924670934677124,
|
||
|
|
"loss/core": 0.1924670934677124,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 2.018162965774536,
|
||
|
|
"rewards/margins": 1.0384140014648438,
|
||
|
|
"rewards/rejected": 0.9797487258911133,
|
||
|
|
"step": 395
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.38218081930013137,
|
||
|
|
"grad_norm": 25.25,
|
||
|
|
"learning_rate": 3.4094830123828786e-07,
|
||
|
|
"logits/chosen": -2.1258950233459473,
|
||
|
|
"logits/rejected": -2.064730405807495,
|
||
|
|
"logps/chosen": -0.27466386556625366,
|
||
|
|
"logps/rejected": -0.28442296385765076,
|
||
|
|
"loss": 0.35562631487846375,
|
||
|
|
"loss/core": 0.35562631487846375,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 2.0544662475585938,
|
||
|
|
"rewards/margins": 0.5271926522254944,
|
||
|
|
"rewards/rejected": 1.5272737741470337,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.386958079541383,
|
||
|
|
"grad_norm": 1208.0,
|
||
|
|
"learning_rate": 3.3641554210593414e-07,
|
||
|
|
"logits/chosen": -2.2992489337921143,
|
||
|
|
"logits/rejected": -2.3570144176483154,
|
||
|
|
"logps/chosen": -0.266804575920105,
|
||
|
|
"logps/rejected": -0.265764981508255,
|
||
|
|
"loss": 1.0862951278686523,
|
||
|
|
"loss/core": 1.0862951278686523,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.5454628467559814,
|
||
|
|
"rewards/margins": 1.311475157737732,
|
||
|
|
"rewards/rejected": 1.2339876890182495,
|
||
|
|
"step": 405
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.39173533978263464,
|
||
|
|
"grad_norm": 11.25,
|
||
|
|
"learning_rate": 3.3185028562233107e-07,
|
||
|
|
"logits/chosen": -2.073944568634033,
|
||
|
|
"logits/rejected": -2.1331162452697754,
|
||
|
|
"logps/chosen": -0.3112749457359314,
|
||
|
|
"logps/rejected": -0.3046428859233856,
|
||
|
|
"loss": 0.09161730855703354,
|
||
|
|
"loss/core": 0.09161730855703354,
|
||
|
|
"rewards/accuracies": 0.800000011920929,
|
||
|
|
"rewards/chosen": 1.2061810493469238,
|
||
|
|
"rewards/margins": 0.551508903503418,
|
||
|
|
"rewards/rejected": 0.6546720266342163,
|
||
|
|
"step": 410
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.3965126000238863,
|
||
|
|
"grad_norm": 4.96875,
|
||
|
|
"learning_rate": 3.272542485937368e-07,
|
||
|
|
"logits/chosen": -2.2970967292785645,
|
||
|
|
"logits/rejected": -2.295583724975586,
|
||
|
|
"logps/chosen": -0.29680368304252625,
|
||
|
|
"logps/rejected": -0.28816667199134827,
|
||
|
|
"loss": 0.5273622274398804,
|
||
|
|
"loss/core": 0.5273622274398804,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.4427194595336914,
|
||
|
|
"rewards/margins": 1.134141206741333,
|
||
|
|
"rewards/rejected": 1.3085782527923584,
|
||
|
|
"step": 415
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.40128986026513797,
|
||
|
|
"grad_norm": 508.0,
|
||
|
|
"learning_rate": 3.226291594017137e-07,
|
||
|
|
"logits/chosen": -2.098283290863037,
|
||
|
|
"logits/rejected": -2.067476749420166,
|
||
|
|
"logps/chosen": -0.3195353150367737,
|
||
|
|
"logps/rejected": -0.3052719235420227,
|
||
|
|
"loss": 1.6536753177642822,
|
||
|
|
"loss/core": 1.6536753177642822,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 3.3768184185028076,
|
||
|
|
"rewards/margins": 2.38930344581604,
|
||
|
|
"rewards/rejected": 0.9875147938728333,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4060671205063896,
|
||
|
|
"grad_norm": 23.625,
|
||
|
|
"learning_rate": 3.1797675735315454e-07,
|
||
|
|
"logits/chosen": -2.124053478240967,
|
||
|
|
"logits/rejected": -2.2029757499694824,
|
||
|
|
"logps/chosen": -0.31783485412597656,
|
||
|
|
"logps/rejected": -0.31542786955833435,
|
||
|
|
"loss": 1.095782995223999,
|
||
|
|
"loss/core": 1.095782995223999,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.0307421684265137,
|
||
|
|
"rewards/margins": 1.9506139755249023,
|
||
|
|
"rewards/rejected": 1.0801277160644531,
|
||
|
|
"step": 425
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.41084438074764124,
|
||
|
|
"grad_norm": 66.5,
|
||
|
|
"learning_rate": 3.1329879202620047e-07,
|
||
|
|
"logits/chosen": -2.1495888233184814,
|
||
|
|
"logits/rejected": -2.263394832611084,
|
||
|
|
"logps/chosen": -0.26696711778640747,
|
||
|
|
"logps/rejected": -0.2735012173652649,
|
||
|
|
"loss": 0.14888674020767212,
|
||
|
|
"loss/core": 0.14888674020767212,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.4852890968322754,
|
||
|
|
"rewards/margins": 0.5966399908065796,
|
||
|
|
"rewards/rejected": 0.8886489868164062,
|
||
|
|
"step": 430
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.41562164098889287,
|
||
|
|
"grad_norm": 1328.0,
|
||
|
|
"learning_rate": 3.0859702261229613e-07,
|
||
|
|
"logits/chosen": -1.9643999338150024,
|
||
|
|
"logits/rejected": -1.975927710533142,
|
||
|
|
"logps/chosen": -0.2912464141845703,
|
||
|
|
"logps/rejected": -0.2692493498325348,
|
||
|
|
"loss": 1.1893305778503418,
|
||
|
|
"loss/core": 1.1893305778503418,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 3.8585808277130127,
|
||
|
|
"rewards/margins": 2.5125555992126465,
|
||
|
|
"rewards/rejected": 1.3460257053375244,
|
||
|
|
"step": 435
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4203989012301445,
|
||
|
|
"grad_norm": 42.5,
|
||
|
|
"learning_rate": 3.0387321725463e-07,
|
||
|
|
"logits/chosen": -2.1609604358673096,
|
||
|
|
"logits/rejected": -2.1351842880249023,
|
||
|
|
"logps/chosen": -0.29672715067863464,
|
||
|
|
"logps/rejected": -0.2880227565765381,
|
||
|
|
"loss": 0.2775880694389343,
|
||
|
|
"loss/core": 0.2775880694389343,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.147207736968994,
|
||
|
|
"rewards/margins": 0.9227482080459595,
|
||
|
|
"rewards/rejected": 1.2244592905044556,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.42517616147139614,
|
||
|
|
"grad_norm": 97.0,
|
||
|
|
"learning_rate": 2.991291523832075e-07,
|
||
|
|
"logits/chosen": -2.0940496921539307,
|
||
|
|
"logits/rejected": -2.1732332706451416,
|
||
|
|
"logps/chosen": -0.285409152507782,
|
||
|
|
"logps/rejected": -0.27999910712242126,
|
||
|
|
"loss": 0.576115071773529,
|
||
|
|
"loss/core": 0.576115071773529,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.9870572090148926,
|
||
|
|
"rewards/margins": 1.8674147129058838,
|
||
|
|
"rewards/rejected": 1.1196422576904297,
|
||
|
|
"step": 445
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4299534217126478,
|
||
|
|
"grad_norm": 1600.0,
|
||
|
|
"learning_rate": 2.943666120468088e-07,
|
||
|
|
"logits/chosen": -2.1286191940307617,
|
||
|
|
"logits/rejected": -2.199714422225952,
|
||
|
|
"logps/chosen": -0.2662820518016815,
|
||
|
|
"logps/rejected": -0.30065736174583435,
|
||
|
|
"loss": 0.9491519927978516,
|
||
|
|
"loss/core": 0.9491519927978516,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.870041608810425,
|
||
|
|
"rewards/margins": 1.6734178066253662,
|
||
|
|
"rewards/rejected": 1.1966238021850586,
|
||
|
|
"step": 450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4347306819538994,
|
||
|
|
"grad_norm": 1.9921875,
|
||
|
|
"learning_rate": 2.8958738724208073e-07,
|
||
|
|
"logits/chosen": -1.8988832235336304,
|
||
|
|
"logits/rejected": -1.9345617294311523,
|
||
|
|
"logps/chosen": -0.30704838037490845,
|
||
|
|
"logps/rejected": -0.3010723292827606,
|
||
|
|
"loss": 0.18666288256645203,
|
||
|
|
"loss/core": 0.18666288256645203,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.7061916589736938,
|
||
|
|
"rewards/margins": 0.875575065612793,
|
||
|
|
"rewards/rejected": 0.8306165933609009,
|
||
|
|
"step": 455
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4395079421951511,
|
||
|
|
"grad_norm": 10.3125,
|
||
|
|
"learning_rate": 2.8479327524001633e-07,
|
||
|
|
"logits/chosen": -2.0395889282226562,
|
||
|
|
"logits/rejected": -2.1937108039855957,
|
||
|
|
"logps/chosen": -0.2862888276576996,
|
||
|
|
"logps/rejected": -0.28865212202072144,
|
||
|
|
"loss": 0.35019630193710327,
|
||
|
|
"loss/core": 0.35019630193710327,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.244964838027954,
|
||
|
|
"rewards/margins": 0.8104959726333618,
|
||
|
|
"rewards/rejected": 1.4344688653945923,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.44428520243640274,
|
||
|
|
"grad_norm": 32.5,
|
||
|
|
"learning_rate": 2.7998607891007493e-07,
|
||
|
|
"logits/chosen": -2.112020254135132,
|
||
|
|
"logits/rejected": -2.0445303916931152,
|
||
|
|
"logps/chosen": -0.2725178003311157,
|
||
|
|
"logps/rejected": -0.30414050817489624,
|
||
|
|
"loss": 1.0059139728546143,
|
||
|
|
"loss/core": 1.0059139728546143,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.83925199508667,
|
||
|
|
"rewards/margins": 0.9617875814437866,
|
||
|
|
"rewards/rejected": 1.8774642944335938,
|
||
|
|
"step": 465
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4490624626776544,
|
||
|
|
"grad_norm": 102.5,
|
||
|
|
"learning_rate": 2.7516760604219616e-07,
|
||
|
|
"logits/chosen": -2.1016361713409424,
|
||
|
|
"logits/rejected": -2.077967643737793,
|
||
|
|
"logps/chosen": -0.2712469696998596,
|
||
|
|
"logps/rejected": -0.29456794261932373,
|
||
|
|
"loss": 0.46677565574645996,
|
||
|
|
"loss/core": 0.46677565574645996,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.2126553058624268,
|
||
|
|
"rewards/margins": 0.6549152135848999,
|
||
|
|
"rewards/rejected": 1.557740330696106,
|
||
|
|
"step": 470
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.453839722918906,
|
||
|
|
"grad_norm": 11.6875,
|
||
|
|
"learning_rate": 2.703396686669646e-07,
|
||
|
|
"logits/chosen": -2.209343433380127,
|
||
|
|
"logits/rejected": -2.228372097015381,
|
||
|
|
"logps/chosen": -0.27892497181892395,
|
||
|
|
"logps/rejected": -0.2862241864204407,
|
||
|
|
"loss": 0.31267350912094116,
|
||
|
|
"loss/core": 0.31267350912094116,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.3414828777313232,
|
||
|
|
"rewards/margins": 0.8732233047485352,
|
||
|
|
"rewards/rejected": 1.468259572982788,
|
||
|
|
"step": 475
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.45861698316015764,
|
||
|
|
"grad_norm": 1168.0,
|
||
|
|
"learning_rate": 2.6550408237417884e-07,
|
||
|
|
"logits/chosen": -1.982518196105957,
|
||
|
|
"logits/rejected": -1.9973068237304688,
|
||
|
|
"logps/chosen": -0.28591465950012207,
|
||
|
|
"logps/rejected": -0.2720577120780945,
|
||
|
|
"loss": 2.5910723209381104,
|
||
|
|
"loss/core": 2.5910723209381104,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 5.177981376647949,
|
||
|
|
"rewards/margins": 3.9536633491516113,
|
||
|
|
"rewards/rejected": 1.2243181467056274,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4633942434014093,
|
||
|
|
"grad_norm": 11.4375,
|
||
|
|
"learning_rate": 2.6066266563008265e-07,
|
||
|
|
"logits/chosen": -2.1890599727630615,
|
||
|
|
"logits/rejected": -2.172191858291626,
|
||
|
|
"logps/chosen": -0.2838982939720154,
|
||
|
|
"logps/rejected": -0.30362966656684875,
|
||
|
|
"loss": 0.41411668062210083,
|
||
|
|
"loss/core": 0.41411668062210083,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.087114095687866,
|
||
|
|
"rewards/margins": 1.0661743879318237,
|
||
|
|
"rewards/rejected": 1.020939588546753,
|
||
|
|
"step": 485
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4681715036426609,
|
||
|
|
"grad_norm": 44.0,
|
||
|
|
"learning_rate": 2.5581723909351404e-07,
|
||
|
|
"logits/chosen": -2.1797804832458496,
|
||
|
|
"logits/rejected": -2.127960681915283,
|
||
|
|
"logps/chosen": -0.2894228994846344,
|
||
|
|
"logps/rejected": -0.3229179084300995,
|
||
|
|
"loss": 0.21129579842090607,
|
||
|
|
"loss/core": 0.21129579842090607,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.6542030572891235,
|
||
|
|
"rewards/margins": 0.47612959146499634,
|
||
|
|
"rewards/rejected": 1.1780736446380615,
|
||
|
|
"step": 490
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.47294876388391255,
|
||
|
|
"grad_norm": 1896.0,
|
||
|
|
"learning_rate": 2.509696249312301e-07,
|
||
|
|
"logits/chosen": -2.0821642875671387,
|
||
|
|
"logits/rejected": -2.087721347808838,
|
||
|
|
"logps/chosen": -0.3256458342075348,
|
||
|
|
"logps/rejected": -0.3126862943172455,
|
||
|
|
"loss": 1.8491243124008179,
|
||
|
|
"loss/core": 1.8491243124008179,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 3.5472946166992188,
|
||
|
|
"rewards/margins": 1.9033483266830444,
|
||
|
|
"rewards/rejected": 1.6439464092254639,
|
||
|
|
"step": 495
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.47772602412516424,
|
||
|
|
"grad_norm": 400.0,
|
||
|
|
"learning_rate": 2.461216461326642e-07,
|
||
|
|
"logits/chosen": -2.1551461219787598,
|
||
|
|
"logits/rejected": -2.116971015930176,
|
||
|
|
"logps/chosen": -0.279205858707428,
|
||
|
|
"logps/rejected": -0.27398478984832764,
|
||
|
|
"loss": 1.2034151554107666,
|
||
|
|
"loss/core": 1.2034151554107666,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 3.4934756755828857,
|
||
|
|
"rewards/margins": 2.316671371459961,
|
||
|
|
"rewards/rejected": 1.176804542541504,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4825032843664159,
|
||
|
|
"grad_norm": 7.875,
|
||
|
|
"learning_rate": 2.412751258243748e-07,
|
||
|
|
"logits/chosen": -1.734514832496643,
|
||
|
|
"logits/rejected": -1.755017638206482,
|
||
|
|
"logps/chosen": -0.3091053366661072,
|
||
|
|
"logps/rejected": -0.31018969416618347,
|
||
|
|
"loss": 0.7064386606216431,
|
||
|
|
"loss/core": 0.7064386606216431,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.180860996246338,
|
||
|
|
"rewards/margins": 1.8545881509780884,
|
||
|
|
"rewards/rejected": 1.32627272605896,
|
||
|
|
"step": 505
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4872805446076675,
|
||
|
|
"grad_norm": 502.0,
|
||
|
|
"learning_rate": 2.3643188658444158e-07,
|
||
|
|
"logits/chosen": -2.0605289936065674,
|
||
|
|
"logits/rejected": -2.1547398567199707,
|
||
|
|
"logps/chosen": -0.2701338529586792,
|
||
|
|
"logps/rejected": -0.2844063639640808,
|
||
|
|
"loss": 0.5944812297821045,
|
||
|
|
"loss/core": 0.5944812297821045,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 3.014646053314209,
|
||
|
|
"rewards/margins": 1.663463830947876,
|
||
|
|
"rewards/rejected": 1.351182222366333,
|
||
|
|
"step": 510
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.49205780484891914,
|
||
|
|
"grad_norm": 14.5,
|
||
|
|
"learning_rate": 2.315937497570688e-07,
|
||
|
|
"logits/chosen": -1.9274842739105225,
|
||
|
|
"logits/rejected": -2.0313713550567627,
|
||
|
|
"logps/chosen": -0.2993752360343933,
|
||
|
|
"logps/rejected": -0.3588908612728119,
|
||
|
|
"loss": 0.5572088956832886,
|
||
|
|
"loss/core": 0.5572088956832886,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.2762691974639893,
|
||
|
|
"rewards/margins": 1.200260043144226,
|
||
|
|
"rewards/rejected": 1.0760090351104736,
|
||
|
|
"step": 515
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.4968350650901708,
|
||
|
|
"grad_norm": 2.265625,
|
||
|
|
"learning_rate": 2.2676253476765194e-07,
|
||
|
|
"logits/chosen": -1.9769691228866577,
|
||
|
|
"logits/rejected": -2.1776771545410156,
|
||
|
|
"logps/chosen": -0.32004016637802124,
|
||
|
|
"logps/rejected": -0.2949730157852173,
|
||
|
|
"loss": 0.6923333406448364,
|
||
|
|
"loss/core": 0.6923333406448364,
|
||
|
|
"rewards/accuracies": 0.824999988079071,
|
||
|
|
"rewards/chosen": 2.3702404499053955,
|
||
|
|
"rewards/margins": 1.4442033767700195,
|
||
|
|
"rewards/rejected": 0.9260371923446655,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5016123253314224,
|
||
|
|
"grad_norm": 0.89453125,
|
||
|
|
"learning_rate": 2.2194005843856633e-07,
|
||
|
|
"logits/chosen": -2.003140687942505,
|
||
|
|
"logits/rejected": -2.0340893268585205,
|
||
|
|
"logps/chosen": -0.28318530321121216,
|
||
|
|
"logps/rejected": -0.281009316444397,
|
||
|
|
"loss": 0.7909449338912964,
|
||
|
|
"loss/core": 0.7909449338912964,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 3.3549766540527344,
|
||
|
|
"rewards/margins": 2.027585744857788,
|
||
|
|
"rewards/rejected": 1.3273913860321045,
|
||
|
|
"step": 525
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5063895855726741,
|
||
|
|
"grad_norm": 7.53125,
|
||
|
|
"learning_rate": 2.1712813430593434e-07,
|
||
|
|
"logits/chosen": -2.0626168251037598,
|
||
|
|
"logits/rejected": -2.1819634437561035,
|
||
|
|
"logps/chosen": -0.2981792986392975,
|
||
|
|
"logps/rejected": -0.297743558883667,
|
||
|
|
"loss": 0.17196998000144958,
|
||
|
|
"loss/core": 0.17196998000144958,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.795514464378357,
|
||
|
|
"rewards/margins": 0.9798916578292847,
|
||
|
|
"rewards/rejected": 0.8156226873397827,
|
||
|
|
"step": 530
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5111668458139257,
|
||
|
|
"grad_norm": 39.5,
|
||
|
|
"learning_rate": 2.123285719376292e-07,
|
||
|
|
"logits/chosen": -2.0366547107696533,
|
||
|
|
"logits/rejected": -2.0415263175964355,
|
||
|
|
"logps/chosen": -0.28957507014274597,
|
||
|
|
"logps/rejected": -0.28460192680358887,
|
||
|
|
"loss": 1.1312463283538818,
|
||
|
|
"loss/core": 1.1312463283538818,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 3.3058319091796875,
|
||
|
|
"rewards/margins": 1.5662612915039062,
|
||
|
|
"rewards/rejected": 1.7395708560943604,
|
||
|
|
"step": 535
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5159441060551774,
|
||
|
|
"grad_norm": 43.5,
|
||
|
|
"learning_rate": 2.0754317625276982e-07,
|
||
|
|
"logits/chosen": -2.0326008796691895,
|
||
|
|
"logits/rejected": -2.0860085487365723,
|
||
|
|
"logps/chosen": -0.2913965582847595,
|
||
|
|
"logps/rejected": -0.2849191427230835,
|
||
|
|
"loss": 0.28950196504592896,
|
||
|
|
"loss/core": 0.28950196504592896,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.971967101097107,
|
||
|
|
"rewards/margins": 1.0436313152313232,
|
||
|
|
"rewards/rejected": 0.9283358454704285,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.520721366296429,
|
||
|
|
"grad_norm": 208.0,
|
||
|
|
"learning_rate": 2.0277374684296498e-07,
|
||
|
|
"logits/chosen": -2.021231174468994,
|
||
|
|
"logits/rejected": -2.0813746452331543,
|
||
|
|
"logps/chosen": -0.2889094650745392,
|
||
|
|
"logps/rejected": -0.27322858572006226,
|
||
|
|
"loss": 0.8897081613540649,
|
||
|
|
"loss/core": 0.8897081613540649,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 3.6772232055664062,
|
||
|
|
"rewards/margins": 2.866091012954712,
|
||
|
|
"rewards/rejected": 0.8111323118209839,
|
||
|
|
"step": 545
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5254986265376806,
|
||
|
|
"grad_norm": 6.90625,
|
||
|
|
"learning_rate": 1.980220772955602e-07,
|
||
|
|
"logits/chosen": -1.9826974868774414,
|
||
|
|
"logits/rejected": -2.0089569091796875,
|
||
|
|
"logps/chosen": -0.2836936116218567,
|
||
|
|
"logps/rejected": -0.28125476837158203,
|
||
|
|
"loss": 0.17911246418952942,
|
||
|
|
"loss/core": 0.17911246418952942,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.2390589714050293,
|
||
|
|
"rewards/margins": 1.1507548093795776,
|
||
|
|
"rewards/rejected": 1.0883041620254517,
|
||
|
|
"step": 550
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5302758867789323,
|
||
|
|
"grad_norm": 15.25,
|
||
|
|
"learning_rate": 1.932899545191433e-07,
|
||
|
|
"logits/chosen": -2.034287452697754,
|
||
|
|
"logits/rejected": -2.1081697940826416,
|
||
|
|
"logps/chosen": -0.3206217288970947,
|
||
|
|
"logps/rejected": -0.30311864614486694,
|
||
|
|
"loss": 1.040518045425415,
|
||
|
|
"loss/core": 1.040518045425415,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 3.1932551860809326,
|
||
|
|
"rewards/margins": 1.7633758783340454,
|
||
|
|
"rewards/rejected": 1.429879069328308,
|
||
|
|
"step": 555
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5350531470201839,
|
||
|
|
"grad_norm": 14.6875,
|
||
|
|
"learning_rate": 1.885791580715609e-07,
|
||
|
|
"logits/chosen": -1.850868582725525,
|
||
|
|
"logits/rejected": -1.8559157848358154,
|
||
|
|
"logps/chosen": -0.3106183409690857,
|
||
|
|
"logps/rejected": -0.29663991928100586,
|
||
|
|
"loss": 0.6726303100585938,
|
||
|
|
"loss/core": 0.6726303100585938,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.4848763942718506,
|
||
|
|
"rewards/margins": 1.1928892135620117,
|
||
|
|
"rewards/rejected": 1.291987419128418,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5398304072614356,
|
||
|
|
"grad_norm": 45.25,
|
||
|
|
"learning_rate": 1.8389145949069951e-07,
|
||
|
|
"logits/chosen": -1.9359796047210693,
|
||
|
|
"logits/rejected": -2.0416994094848633,
|
||
|
|
"logps/chosen": -0.28728121519088745,
|
||
|
|
"logps/rejected": -0.2942139506340027,
|
||
|
|
"loss": 2.735677719116211,
|
||
|
|
"loss/core": 2.735677719116211,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 5.07828950881958,
|
||
|
|
"rewards/margins": 3.2359542846679688,
|
||
|
|
"rewards/rejected": 1.8423347473144531,
|
||
|
|
"step": 565
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5446076675026872,
|
||
|
|
"grad_norm": 91.5,
|
||
|
|
"learning_rate": 1.792286216282824e-07,
|
||
|
|
"logits/chosen": -2.077359437942505,
|
||
|
|
"logits/rejected": -2.0332412719726562,
|
||
|
|
"logps/chosen": -0.3251068890094757,
|
||
|
|
"logps/rejected": -0.31182152032852173,
|
||
|
|
"loss": 0.7743596434593201,
|
||
|
|
"loss/core": 0.7743596434593201,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 2.263739585876465,
|
||
|
|
"rewards/margins": 0.2986152172088623,
|
||
|
|
"rewards/rejected": 1.965124487876892,
|
||
|
|
"step": 570
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5493849277439389,
|
||
|
|
"grad_norm": 20.625,
|
||
|
|
"learning_rate": 1.745923979869336e-07,
|
||
|
|
"logits/chosen": -2.3297030925750732,
|
||
|
|
"logits/rejected": -2.3048276901245117,
|
||
|
|
"logps/chosen": -0.28046730160713196,
|
||
|
|
"logps/rejected": -0.2963675260543823,
|
||
|
|
"loss": 0.5231956839561462,
|
||
|
|
"loss/core": 0.5231956839561462,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.400116443634033,
|
||
|
|
"rewards/margins": 1.4617714881896973,
|
||
|
|
"rewards/rejected": 0.9383450746536255,
|
||
|
|
"step": 575
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5541621879851905,
|
||
|
|
"grad_norm": 8.8125,
|
||
|
|
"learning_rate": 1.6998453206075708e-07,
|
||
|
|
"logits/chosen": -2.00852632522583,
|
||
|
|
"logits/rejected": -2.128060817718506,
|
||
|
|
"logps/chosen": -0.30730748176574707,
|
||
|
|
"logps/rejected": -0.3067978024482727,
|
||
|
|
"loss": 0.4244763255119324,
|
||
|
|
"loss/core": 0.4244763255119324,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.4532878398895264,
|
||
|
|
"rewards/margins": 1.449927568435669,
|
||
|
|
"rewards/rejected": 1.0033600330352783,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5589394482264421,
|
||
|
|
"grad_norm": 234.0,
|
||
|
|
"learning_rate": 1.6540675667967973e-07,
|
||
|
|
"logits/chosen": -2.2257282733917236,
|
||
|
|
"logits/rejected": -2.239046096801758,
|
||
|
|
"logps/chosen": -0.2853415906429291,
|
||
|
|
"logps/rejected": -0.28488466143608093,
|
||
|
|
"loss": 0.8494731187820435,
|
||
|
|
"loss/core": 0.8494731187820435,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 2.5698318481445312,
|
||
|
|
"rewards/margins": 0.717768669128418,
|
||
|
|
"rewards/rejected": 1.8520629405975342,
|
||
|
|
"step": 585
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5637167084676937,
|
||
|
|
"grad_norm": 185.0,
|
||
|
|
"learning_rate": 1.60860793357805e-07,
|
||
|
|
"logits/chosen": -1.9403209686279297,
|
||
|
|
"logits/rejected": -1.9107745885849,
|
||
|
|
"logps/chosen": -0.30443838238716125,
|
||
|
|
"logps/rejected": -0.3014160990715027,
|
||
|
|
"loss": 0.7370032668113708,
|
||
|
|
"loss/core": 0.7370032668113708,
|
||
|
|
"rewards/accuracies": 0.824999988079071,
|
||
|
|
"rewards/chosen": 2.7498955726623535,
|
||
|
|
"rewards/margins": 1.003064513206482,
|
||
|
|
"rewards/rejected": 1.746830940246582,
|
||
|
|
"step": 590
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5684939687089454,
|
||
|
|
"grad_norm": 2.40625,
|
||
|
|
"learning_rate": 1.5634835164602196e-07,
|
||
|
|
"logits/chosen": -2.0419182777404785,
|
||
|
|
"logits/rejected": -1.969354271888733,
|
||
|
|
"logps/chosen": -0.2744390368461609,
|
||
|
|
"logps/rejected": -0.2946432828903198,
|
||
|
|
"loss": 0.6912421584129333,
|
||
|
|
"loss/core": 0.6912421584129333,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.8153032064437866,
|
||
|
|
"rewards/margins": -0.3735305368900299,
|
||
|
|
"rewards/rejected": 2.188833713531494,
|
||
|
|
"step": 595
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5732712289501971,
|
||
|
|
"grad_norm": 40.25,
|
||
|
|
"learning_rate": 1.518711284891132e-07,
|
||
|
|
"logits/chosen": -2.1388328075408936,
|
||
|
|
"logits/rejected": -2.0539865493774414,
|
||
|
|
"logps/chosen": -0.2972756028175354,
|
||
|
|
"logps/rejected": -0.31171417236328125,
|
||
|
|
"loss": 0.36328691244125366,
|
||
|
|
"loss/core": 0.36328691244125366,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.7741825580596924,
|
||
|
|
"rewards/margins": 0.6642401814460754,
|
||
|
|
"rewards/rejected": 1.1099423170089722,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5780484891914487,
|
||
|
|
"grad_norm": 28.5,
|
||
|
|
"learning_rate": 1.47430807587603e-07,
|
||
|
|
"logits/chosen": -2.2820992469787598,
|
||
|
|
"logits/rejected": -2.370114803314209,
|
||
|
|
"logps/chosen": -0.29316216707229614,
|
||
|
|
"logps/rejected": -0.3045360743999481,
|
||
|
|
"loss": 0.418804794549942,
|
||
|
|
"loss/core": 0.418804794549942,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.924261450767517,
|
||
|
|
"rewards/margins": 1.342273473739624,
|
||
|
|
"rewards/rejected": 0.5819882154464722,
|
||
|
|
"step": 605
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5828257494327004,
|
||
|
|
"grad_norm": 5.03125,
|
||
|
|
"learning_rate": 1.430290587645865e-07,
|
||
|
|
"logits/chosen": -2.0365357398986816,
|
||
|
|
"logits/rejected": -2.1053943634033203,
|
||
|
|
"logps/chosen": -0.30090898275375366,
|
||
|
|
"logps/rejected": -0.30962470173835754,
|
||
|
|
"loss": 0.7318702936172485,
|
||
|
|
"loss/core": 0.7318702936172485,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.31716251373291,
|
||
|
|
"rewards/margins": 0.7470144033432007,
|
||
|
|
"rewards/rejected": 1.5701478719711304,
|
||
|
|
"step": 610
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.587603009673952,
|
||
|
|
"grad_norm": 1.0390625,
|
||
|
|
"learning_rate": 1.3866753733777765e-07,
|
||
|
|
"logits/chosen": -2.0896778106689453,
|
||
|
|
"logits/rejected": -2.0692594051361084,
|
||
|
|
"logps/chosen": -0.3204464912414551,
|
||
|
|
"logps/rejected": -0.31083038449287415,
|
||
|
|
"loss": 0.15200874209403992,
|
||
|
|
"loss/core": 0.15200874209403992,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.405200481414795,
|
||
|
|
"rewards/margins": 0.6567575335502625,
|
||
|
|
"rewards/rejected": 0.7484430074691772,
|
||
|
|
"step": 615
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5923802699152036,
|
||
|
|
"grad_norm": 10.25,
|
||
|
|
"learning_rate": 1.343478834970121e-07,
|
||
|
|
"logits/chosen": -2.2016444206237793,
|
||
|
|
"logits/rejected": -2.162689447402954,
|
||
|
|
"logps/chosen": -0.2618758976459503,
|
||
|
|
"logps/rejected": -0.28418415784835815,
|
||
|
|
"loss": 0.2642122209072113,
|
||
|
|
"loss/core": 0.2642122209072113,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.8917810916900635,
|
||
|
|
"rewards/margins": 0.6098601818084717,
|
||
|
|
"rewards/rejected": 1.2819210290908813,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.5971575301564552,
|
||
|
|
"grad_norm": 13.5,
|
||
|
|
"learning_rate": 1.3007172168743852e-07,
|
||
|
|
"logits/chosen": -2.0167224407196045,
|
||
|
|
"logits/rejected": -1.9906463623046875,
|
||
|
|
"logps/chosen": -0.2946910262107849,
|
||
|
|
"logps/rejected": -0.29906517267227173,
|
||
|
|
"loss": 0.1281525194644928,
|
||
|
|
"loss/core": 0.1281525194644928,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 1.5117716789245605,
|
||
|
|
"rewards/margins": 0.6090092658996582,
|
||
|
|
"rewards/rejected": 0.9027624130249023,
|
||
|
|
"step": 625
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6019347903977069,
|
||
|
|
"grad_norm": 48.0,
|
||
|
|
"learning_rate": 1.25840659998631e-07,
|
||
|
|
"logits/chosen": -2.24929141998291,
|
||
|
|
"logits/rejected": -2.1729183197021484,
|
||
|
|
"logps/chosen": -0.2794455885887146,
|
||
|
|
"logps/rejected": -0.2963079810142517,
|
||
|
|
"loss": 0.09536317735910416,
|
||
|
|
"loss/core": 0.09536317735910416,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.2493692636489868,
|
||
|
|
"rewards/margins": 0.36890143156051636,
|
||
|
|
"rewards/rejected": 0.8804677724838257,
|
||
|
|
"step": 630
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6067120506389586,
|
||
|
|
"grad_norm": 0.60546875,
|
||
|
|
"learning_rate": 1.2165628955985313e-07,
|
||
|
|
"logits/chosen": -2.070937395095825,
|
||
|
|
"logits/rejected": -2.0991628170013428,
|
||
|
|
"logps/chosen": -0.30978572368621826,
|
||
|
|
"logps/rejected": -0.2937181890010834,
|
||
|
|
"loss": 1.2708914279937744,
|
||
|
|
"loss/core": 1.2708914279937744,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 3.1841537952423096,
|
||
|
|
"rewards/margins": 2.332937240600586,
|
||
|
|
"rewards/rejected": 0.851216197013855,
|
||
|
|
"step": 635
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6114893108802102,
|
||
|
|
"grad_norm": 255.0,
|
||
|
|
"learning_rate": 1.175201839416988e-07,
|
||
|
|
"logits/chosen": -1.9998000860214233,
|
||
|
|
"logits/rejected": -2.1376776695251465,
|
||
|
|
"logps/chosen": -0.2979816794395447,
|
||
|
|
"logps/rejected": -0.2850627899169922,
|
||
|
|
"loss": 0.48012876510620117,
|
||
|
|
"loss/core": 0.48012876510620117,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 2.3821167945861816,
|
||
|
|
"rewards/margins": 1.731583833694458,
|
||
|
|
"rewards/rejected": 0.6505327820777893,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6162665711214619,
|
||
|
|
"grad_norm": 13.0,
|
||
|
|
"learning_rate": 1.1343389856433658e-07,
|
||
|
|
"logits/chosen": -2.054605484008789,
|
||
|
|
"logits/rejected": -2.1557681560516357,
|
||
|
|
"logps/chosen": -0.2970808148384094,
|
||
|
|
"logps/rejected": -0.26274365186691284,
|
||
|
|
"loss": 2.3463776111602783,
|
||
|
|
"loss/core": 2.3463776111602783,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 4.414872169494629,
|
||
|
|
"rewards/margins": 2.85589337348938,
|
||
|
|
"rewards/rejected": 1.5589792728424072,
|
||
|
|
"step": 645
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6210438313627135,
|
||
|
|
"grad_norm": 7.71875,
|
||
|
|
"learning_rate": 1.0939897011258e-07,
|
||
|
|
"logits/chosen": -2.2726993560791016,
|
||
|
|
"logits/rejected": -2.288119077682495,
|
||
|
|
"logps/chosen": -0.3080156445503235,
|
||
|
|
"logps/rejected": -0.2879447340965271,
|
||
|
|
"loss": 1.3906691074371338,
|
||
|
|
"loss/core": 1.3906691074371338,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 3.284862995147705,
|
||
|
|
"rewards/margins": 2.400153875350952,
|
||
|
|
"rewards/rejected": 0.8847091794013977,
|
||
|
|
"step": 650
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6258210916039652,
|
||
|
|
"grad_norm": 17.75,
|
||
|
|
"learning_rate": 1.0541691595800336e-07,
|
||
|
|
"logits/chosen": -1.922623872756958,
|
||
|
|
"logits/rejected": -1.9154341220855713,
|
||
|
|
"logps/chosen": -0.28070202469825745,
|
||
|
|
"logps/rejected": -0.3093103766441345,
|
||
|
|
"loss": 0.5413604974746704,
|
||
|
|
"loss/core": 0.5413604974746704,
|
||
|
|
"rewards/accuracies": 0.9750000238418579,
|
||
|
|
"rewards/chosen": 2.844754457473755,
|
||
|
|
"rewards/margins": 1.1954736709594727,
|
||
|
|
"rewards/rejected": 1.6492809057235718,
|
||
|
|
"step": 655
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6305983518452167,
|
||
|
|
"grad_norm": 16.75,
|
||
|
|
"learning_rate": 1.0148923358832021e-07,
|
||
|
|
"logits/chosen": -2.2555432319641113,
|
||
|
|
"logits/rejected": -2.311061382293701,
|
||
|
|
"logps/chosen": -0.27462202310562134,
|
||
|
|
"logps/rejected": -0.2811489999294281,
|
||
|
|
"loss": 0.3550433814525604,
|
||
|
|
"loss/core": 0.3550433814525604,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.0413260459899902,
|
||
|
|
"rewards/margins": 0.9709696769714355,
|
||
|
|
"rewards/rejected": 1.0703566074371338,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6353756120864684,
|
||
|
|
"grad_norm": 26.875,
|
||
|
|
"learning_rate": 9.761740004423926e-08,
|
||
|
|
"logits/chosen": -2.12736439704895,
|
||
|
|
"logits/rejected": -2.2038140296936035,
|
||
|
|
"logps/chosen": -0.3001564145088196,
|
||
|
|
"logps/rejected": -0.32446950674057007,
|
||
|
|
"loss": 0.2872669994831085,
|
||
|
|
"loss/core": 0.2872669994831085,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.7709327936172485,
|
||
|
|
"rewards/margins": 0.7759586572647095,
|
||
|
|
"rewards/rejected": 0.9949741363525391,
|
||
|
|
"step": 665
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.64015287232772,
|
||
|
|
"grad_norm": 14.125,
|
||
|
|
"learning_rate": 9.380287136400999e-08,
|
||
|
|
"logits/chosen": -2.269469738006592,
|
||
|
|
"logits/rejected": -2.301842212677002,
|
||
|
|
"logps/chosen": -0.285754531621933,
|
||
|
|
"logps/rejected": -0.2945065200328827,
|
||
|
|
"loss": 0.39818090200424194,
|
||
|
|
"loss/core": 0.39818090200424194,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.9868361949920654,
|
||
|
|
"rewards/margins": 0.8125089406967163,
|
||
|
|
"rewards/rejected": 1.1743274927139282,
|
||
|
|
"step": 670
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6449301325689717,
|
||
|
|
"grad_norm": 23.375,
|
||
|
|
"learning_rate": 9.004708203586628e-08,
|
||
|
|
"logits/chosen": -2.080825090408325,
|
||
|
|
"logits/rejected": -2.2083404064178467,
|
||
|
|
"logps/chosen": -0.2972838282585144,
|
||
|
|
"logps/rejected": -0.28944870829582214,
|
||
|
|
"loss": 0.3145357370376587,
|
||
|
|
"loss/core": 0.3145357370376587,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.903146505355835,
|
||
|
|
"rewards/margins": 0.8571847677230835,
|
||
|
|
"rewards/rejected": 1.0459617376327515,
|
||
|
|
"step": 675
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6497073928102234,
|
||
|
|
"grad_norm": 704.0,
|
||
|
|
"learning_rate": 8.635144445857407e-08,
|
||
|
|
"logits/chosen": -2.1277308464050293,
|
||
|
|
"logits/rejected": -2.093435525894165,
|
||
|
|
"logps/chosen": -0.28518134355545044,
|
||
|
|
"logps/rejected": -0.2920211851596832,
|
||
|
|
"loss": 0.24321074783802032,
|
||
|
|
"loss/core": 0.24321074783802032,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.2132742404937744,
|
||
|
|
"rewards/margins": -0.060915298759937286,
|
||
|
|
"rewards/rejected": 1.2741894721984863,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.654484653051475,
|
||
|
|
"grad_norm": 392.0,
|
||
|
|
"learning_rate": 8.271734841028552e-08,
|
||
|
|
"logits/chosen": -2.1869044303894043,
|
||
|
|
"logits/rejected": -2.1886041164398193,
|
||
|
|
"logps/chosen": -0.2989751398563385,
|
||
|
|
"logps/rejected": -0.30234605073928833,
|
||
|
|
"loss": 0.5350499153137207,
|
||
|
|
"loss/core": 0.5350499153137207,
|
||
|
|
"rewards/accuracies": 0.824999988079071,
|
||
|
|
"rewards/chosen": 1.4281145334243774,
|
||
|
|
"rewards/margins": -0.04558032751083374,
|
||
|
|
"rewards/rejected": 1.4736945629119873,
|
||
|
|
"step": 685
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6592619132927267,
|
||
|
|
"grad_norm": 21.625,
|
||
|
|
"learning_rate": 7.91461605259007e-08,
|
||
|
|
"logits/chosen": -1.9535391330718994,
|
||
|
|
"logits/rejected": -1.9515883922576904,
|
||
|
|
"logps/chosen": -0.26703256368637085,
|
||
|
|
"logps/rejected": -0.28917747735977173,
|
||
|
|
"loss": 0.5689412355422974,
|
||
|
|
"loss/core": 0.5689412355422974,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 3.011037826538086,
|
||
|
|
"rewards/margins": 1.4389708042144775,
|
||
|
|
"rewards/rejected": 1.5720670223236084,
|
||
|
|
"step": 690
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6640391735339782,
|
||
|
|
"grad_norm": 6.96875,
|
||
|
|
"learning_rate": 7.563922378313217e-08,
|
||
|
|
"logits/chosen": -2.2548301219940186,
|
||
|
|
"logits/rejected": -2.1238934993743896,
|
||
|
|
"logps/chosen": -0.2951969504356384,
|
||
|
|
"logps/rejected": -0.3159661591053009,
|
||
|
|
"loss": 0.14107027649879456,
|
||
|
|
"loss/core": 0.14107027649879456,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.4432185888290405,
|
||
|
|
"rewards/margins": 0.7674707174301147,
|
||
|
|
"rewards/rejected": 0.6757478713989258,
|
||
|
|
"step": 695
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6688164337752299,
|
||
|
|
"grad_norm": 183.0,
|
||
|
|
"learning_rate": 7.219785699746572e-08,
|
||
|
|
"logits/chosen": -2.0966005325317383,
|
||
|
|
"logits/rejected": -2.1340537071228027,
|
||
|
|
"logps/chosen": -0.2893856465816498,
|
||
|
|
"logps/rejected": -0.2702532708644867,
|
||
|
|
"loss": 0.855477511882782,
|
||
|
|
"loss/core": 0.855477511882782,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 3.0240743160247803,
|
||
|
|
"rewards/margins": 1.6322101354599,
|
||
|
|
"rewards/rejected": 1.3918644189834595,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6735936940164815,
|
||
|
|
"grad_norm": 148.0,
|
||
|
|
"learning_rate": 6.882335432620779e-08,
|
||
|
|
"logits/chosen": -2.008580446243286,
|
||
|
|
"logits/rejected": -2.0907862186431885,
|
||
|
|
"logps/chosen": -0.2771678864955902,
|
||
|
|
"logps/rejected": -0.2853749692440033,
|
||
|
|
"loss": 0.19750399887561798,
|
||
|
|
"loss/core": 0.19750399887561798,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.4055383205413818,
|
||
|
|
"rewards/margins": 0.3061255216598511,
|
||
|
|
"rewards/rejected": 1.0994129180908203,
|
||
|
|
"step": 705
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6783709542577332,
|
||
|
|
"grad_norm": 5.15625,
|
||
|
|
"learning_rate": 6.551698478180589e-08,
|
||
|
|
"logits/chosen": -2.002742052078247,
|
||
|
|
"logits/rejected": -2.040894031524658,
|
||
|
|
"logps/chosen": -0.28560441732406616,
|
||
|
|
"logps/rejected": -0.2906520962715149,
|
||
|
|
"loss": 0.36187195777893066,
|
||
|
|
"loss/core": 0.36187195777893066,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.5490121841430664,
|
||
|
|
"rewards/margins": 1.2243046760559082,
|
||
|
|
"rewards/rejected": 1.3247077465057373,
|
||
|
|
"step": 710
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6831482144989849,
|
||
|
|
"grad_norm": 2.125,
|
||
|
|
"learning_rate": 6.22799917546252e-08,
|
||
|
|
"logits/chosen": -2.0940136909484863,
|
||
|
|
"logits/rejected": -2.1399710178375244,
|
||
|
|
"logps/chosen": -0.28695395588874817,
|
||
|
|
"logps/rejected": -0.2956109941005707,
|
||
|
|
"loss": 0.47707152366638184,
|
||
|
|
"loss/core": 0.47707152366638184,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.8282418251037598,
|
||
|
|
"rewards/margins": 1.4551374912261963,
|
||
|
|
"rewards/rejected": 1.3731043338775635,
|
||
|
|
"step": 715
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6879254747402365,
|
||
|
|
"grad_norm": 1.9140625,
|
||
|
|
"learning_rate": 5.9113592545359944e-08,
|
||
|
|
"logits/chosen": -2.2517735958099365,
|
||
|
|
"logits/rejected": -2.2257275581359863,
|
||
|
|
"logps/chosen": -0.3003559112548828,
|
||
|
|
"logps/rejected": -0.2943252921104431,
|
||
|
|
"loss": 0.47043830156326294,
|
||
|
|
"loss/core": 0.47043830156326294,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.214423418045044,
|
||
|
|
"rewards/margins": 1.1652237176895142,
|
||
|
|
"rewards/rejected": 1.0491998195648193,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6927027349814882,
|
||
|
|
"grad_norm": 3.765625,
|
||
|
|
"learning_rate": 5.601897790725643e-08,
|
||
|
|
"logits/chosen": -2.103532314300537,
|
||
|
|
"logits/rejected": -2.1393818855285645,
|
||
|
|
"logps/chosen": -0.31979304552078247,
|
||
|
|
"logps/rejected": -0.31625136733055115,
|
||
|
|
"loss": 0.08710397779941559,
|
||
|
|
"loss/core": 0.08710397779941559,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 1.1311936378479004,
|
||
|
|
"rewards/margins": 0.3907800316810608,
|
||
|
|
"rewards/rejected": 0.7404135465621948,
|
||
|
|
"step": 725
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.6974799952227397,
|
||
|
|
"grad_norm": 19.875,
|
||
|
|
"learning_rate": 5.299731159831952e-08,
|
||
|
|
"logits/chosen": -2.185694456100464,
|
||
|
|
"logits/rejected": -2.1851601600646973,
|
||
|
|
"logps/chosen": -0.29845187067985535,
|
||
|
|
"logps/rejected": -0.30084067583084106,
|
||
|
|
"loss": 0.5585907697677612,
|
||
|
|
"loss/core": 0.5585907697677612,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 2.5317673683166504,
|
||
|
|
"rewards/margins": 1.6783374547958374,
|
||
|
|
"rewards/rejected": 0.8534297943115234,
|
||
|
|
"step": 730
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7022572554639914,
|
||
|
|
"grad_norm": 7.65625,
|
||
|
|
"learning_rate": 5.0049729943671013e-08,
|
||
|
|
"logits/chosen": -2.2042665481567383,
|
||
|
|
"logits/rejected": -2.212019681930542,
|
||
|
|
"logps/chosen": -0.270427942276001,
|
||
|
|
"logps/rejected": -0.29099661111831665,
|
||
|
|
"loss": 0.2325257807970047,
|
||
|
|
"loss/core": 0.2325257807970047,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.025036573410034,
|
||
|
|
"rewards/margins": 0.7466187477111816,
|
||
|
|
"rewards/rejected": 1.2784178256988525,
|
||
|
|
"step": 735
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.707034515705243,
|
||
|
|
"grad_norm": 5.625,
|
||
|
|
"learning_rate": 4.7177341408223994e-08,
|
||
|
|
"logits/chosen": -2.023667812347412,
|
||
|
|
"logits/rejected": -2.055938243865967,
|
||
|
|
"logps/chosen": -0.267364501953125,
|
||
|
|
"logps/rejected": -0.284996896982193,
|
||
|
|
"loss": 0.3424833416938782,
|
||
|
|
"loss/core": 0.3424833416938782,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.194387912750244,
|
||
|
|
"rewards/margins": 1.0326035022735596,
|
||
|
|
"rewards/rejected": 1.1617845296859741,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7118117759464947,
|
||
|
|
"grad_norm": 62.25,
|
||
|
|
"learning_rate": 4.438122617983442e-08,
|
||
|
|
"logits/chosen": -1.9847673177719116,
|
||
|
|
"logits/rejected": -2.012366771697998,
|
||
|
|
"logps/chosen": -0.2840479910373688,
|
||
|
|
"logps/rejected": -0.27548593282699585,
|
||
|
|
"loss": 1.2490966320037842,
|
||
|
|
"loss/core": 1.2490966320037842,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 3.142892360687256,
|
||
|
|
"rewards/margins": 1.1425834894180298,
|
||
|
|
"rewards/rejected": 2.0003087520599365,
|
||
|
|
"step": 745
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7165890361877463,
|
||
|
|
"grad_norm": 10.9375,
|
||
|
|
"learning_rate": 4.1662435763087114e-08,
|
||
|
|
"logits/chosen": -2.047241687774658,
|
||
|
|
"logits/rejected": -2.079470634460449,
|
||
|
|
"logps/chosen": -0.2526155114173889,
|
||
|
|
"logps/rejected": -0.2926296293735504,
|
||
|
|
"loss": 0.3922721743583679,
|
||
|
|
"loss/core": 0.3922721743583679,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.4976963996887207,
|
||
|
|
"rewards/margins": 1.1639492511749268,
|
||
|
|
"rewards/rejected": 1.333747148513794,
|
||
|
|
"step": 750
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.721366296428998,
|
||
|
|
"grad_norm": 1.0078125,
|
||
|
|
"learning_rate": 3.902199258386732e-08,
|
||
|
|
"logits/chosen": -2.1882448196411133,
|
||
|
|
"logits/rejected": -2.1855483055114746,
|
||
|
|
"logps/chosen": -0.36757299304008484,
|
||
|
|
"logps/rejected": -0.29461368918418884,
|
||
|
|
"loss": 1.8846288919448853,
|
||
|
|
"loss/core": 1.8846288919448853,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.939770221710205,
|
||
|
|
"rewards/margins": 1.8737109899520874,
|
||
|
|
"rewards/rejected": 2.06605863571167,
|
||
|
|
"step": 755
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7261435566702497,
|
||
|
|
"grad_norm": 6.875,
|
||
|
|
"learning_rate": 3.646088960486862e-08,
|
||
|
|
"logits/chosen": -2.087844133377075,
|
||
|
|
"logits/rejected": -2.1318325996398926,
|
||
|
|
"logps/chosen": -0.2983017563819885,
|
||
|
|
"logps/rejected": -0.30041682720184326,
|
||
|
|
"loss": 0.2357165366411209,
|
||
|
|
"loss/core": 0.2357165366411209,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.57404363155365,
|
||
|
|
"rewards/margins": 0.8634489178657532,
|
||
|
|
"rewards/rejected": 0.7105947136878967,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7309208169115012,
|
||
|
|
"grad_norm": 4.3125,
|
||
|
|
"learning_rate": 3.398008995217988e-08,
|
||
|
|
"logits/chosen": -2.253822088241577,
|
||
|
|
"logits/rejected": -2.278644323348999,
|
||
|
|
"logps/chosen": -0.2737087607383728,
|
||
|
|
"logps/rejected": -0.2841106057167053,
|
||
|
|
"loss": 0.0988527461886406,
|
||
|
|
"loss/core": 0.0988527461886406,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.4775121212005615,
|
||
|
|
"rewards/margins": 0.5869768261909485,
|
||
|
|
"rewards/rejected": 0.8905353546142578,
|
||
|
|
"step": 765
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7356980771527529,
|
||
|
|
"grad_norm": 69.0,
|
||
|
|
"learning_rate": 3.1580526553093315e-08,
|
||
|
|
"logits/chosen": -2.1181020736694336,
|
||
|
|
"logits/rejected": -2.1448464393615723,
|
||
|
|
"logps/chosen": -0.30012568831443787,
|
||
|
|
"logps/rejected": -0.2899145185947418,
|
||
|
|
"loss": 0.3107560873031616,
|
||
|
|
"loss/core": 0.3107560873031616,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.013152599334717,
|
||
|
|
"rewards/margins": 0.9449566006660461,
|
||
|
|
"rewards/rejected": 1.0681959390640259,
|
||
|
|
"step": 770
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7404753373940045,
|
||
|
|
"grad_norm": 251.0,
|
||
|
|
"learning_rate": 2.9263101785268252e-08,
|
||
|
|
"logits/chosen": -2.248095750808716,
|
||
|
|
"logits/rejected": -2.2297964096069336,
|
||
|
|
"logps/chosen": -0.27153995633125305,
|
||
|
|
"logps/rejected": -0.27207639813423157,
|
||
|
|
"loss": 0.322858989238739,
|
||
|
|
"loss/core": 0.322858989238739,
|
||
|
|
"rewards/accuracies": 0.9750000238418579,
|
||
|
|
"rewards/chosen": 2.0420784950256348,
|
||
|
|
"rewards/margins": 1.176545262336731,
|
||
|
|
"rewards/rejected": 0.8655333518981934,
|
||
|
|
"step": 775
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7452525976352562,
|
||
|
|
"grad_norm": 20.625,
|
||
|
|
"learning_rate": 2.7028687137384264e-08,
|
||
|
|
"logits/chosen": -2.028242826461792,
|
||
|
|
"logits/rejected": -2.135806083679199,
|
||
|
|
"logps/chosen": -0.2820461392402649,
|
||
|
|
"logps/rejected": -0.2808576226234436,
|
||
|
|
"loss": 0.5030468702316284,
|
||
|
|
"loss/core": 0.5030468702316284,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.855433940887451,
|
||
|
|
"rewards/margins": 1.1535121202468872,
|
||
|
|
"rewards/rejected": 1.701921820640564,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7500298578765078,
|
||
|
|
"grad_norm": 14.875,
|
||
|
|
"learning_rate": 2.4878122881409447e-08,
|
||
|
|
"logits/chosen": -1.9230458736419678,
|
||
|
|
"logits/rejected": -1.8952453136444092,
|
||
|
|
"logps/chosen": -0.28769412636756897,
|
||
|
|
"logps/rejected": -0.2964576482772827,
|
||
|
|
"loss": 0.5485445261001587,
|
||
|
|
"loss/core": 0.5485445261001587,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.40129017829895,
|
||
|
|
"rewards/margins": 0.9055936932563782,
|
||
|
|
"rewards/rejected": 1.4956964254379272,
|
||
|
|
"step": 785
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7548071181177595,
|
||
|
|
"grad_norm": 3.84375,
|
||
|
|
"learning_rate": 2.2812217756608937e-08,
|
||
|
|
"logits/chosen": -2.113828420639038,
|
||
|
|
"logits/rejected": -2.184523344039917,
|
||
|
|
"logps/chosen": -0.30777841806411743,
|
||
|
|
"logps/rejected": -0.34668484330177307,
|
||
|
|
"loss": 0.178707093000412,
|
||
|
|
"loss/core": 0.178707093000412,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.7738380432128906,
|
||
|
|
"rewards/margins": 1.0590564012527466,
|
||
|
|
"rewards/rejected": 0.7147814631462097,
|
||
|
|
"step": 790
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7595843783590112,
|
||
|
|
"grad_norm": 57.25,
|
||
|
|
"learning_rate": 2.0831748665410763e-08,
|
||
|
|
"logits/chosen": -2.0712850093841553,
|
||
|
|
"logits/rejected": -2.047006368637085,
|
||
|
|
"logps/chosen": -0.2615426182746887,
|
||
|
|
"logps/rejected": -0.2844923138618469,
|
||
|
|
"loss": 0.5626412630081177,
|
||
|
|
"loss/core": 0.5626412630081177,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.4369277954101562,
|
||
|
|
"rewards/margins": 0.5682427883148193,
|
||
|
|
"rewards/rejected": 1.8686845302581787,
|
||
|
|
"step": 795
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7643616386002627,
|
||
|
|
"grad_norm": 9.0,
|
||
|
|
"learning_rate": 1.8937460381244967e-08,
|
||
|
|
"logits/chosen": -1.9848846197128296,
|
||
|
|
"logits/rejected": -2.050729751586914,
|
||
|
|
"logps/chosen": -0.3180027902126312,
|
||
|
|
"logps/rejected": -0.2833881378173828,
|
||
|
|
"loss": 1.3084042072296143,
|
||
|
|
"loss/core": 1.3084042072296143,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.4368815422058105,
|
||
|
|
"rewards/margins": 2.4918322563171387,
|
||
|
|
"rewards/rejected": 0.9450491666793823,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7691388988415144,
|
||
|
|
"grad_norm": 49.0,
|
||
|
|
"learning_rate": 1.713006526846439e-08,
|
||
|
|
"logits/chosen": -2.084895610809326,
|
||
|
|
"logits/rejected": -2.2298989295959473,
|
||
|
|
"logps/chosen": -0.2837081849575043,
|
||
|
|
"logps/rejected": -0.28118696808815,
|
||
|
|
"loss": 0.3433898687362671,
|
||
|
|
"loss/core": 0.3433898687362671,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.079929828643799,
|
||
|
|
"rewards/margins": 1.1563465595245361,
|
||
|
|
"rewards/rejected": 0.9235833287239075,
|
||
|
|
"step": 805
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.773916159082766,
|
||
|
|
"grad_norm": 10.9375,
|
||
|
|
"learning_rate": 1.541024301445404e-08,
|
||
|
|
"logits/chosen": -2.3721537590026855,
|
||
|
|
"logits/rejected": -2.3594298362731934,
|
||
|
|
"logps/chosen": -0.27915048599243164,
|
||
|
|
"logps/rejected": -0.2653045058250427,
|
||
|
|
"loss": 0.35306280851364136,
|
||
|
|
"loss/core": 0.35306280851364136,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.984630823135376,
|
||
|
|
"rewards/margins": 1.1519166231155396,
|
||
|
|
"rewards/rejected": 0.8327140808105469,
|
||
|
|
"step": 810
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7786934193240177,
|
||
|
|
"grad_norm": 45.5,
|
||
|
|
"learning_rate": 1.3778640374027983e-08,
|
||
|
|
"logits/chosen": -2.0560569763183594,
|
||
|
|
"logits/rejected": -2.0075690746307373,
|
||
|
|
"logps/chosen": -0.29053401947021484,
|
||
|
|
"logps/rejected": -0.3152034282684326,
|
||
|
|
"loss": 0.1969846785068512,
|
||
|
|
"loss/core": 0.1969846785068512,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.955904245376587,
|
||
|
|
"rewards/margins": 0.8895227313041687,
|
||
|
|
"rewards/rejected": 1.0663814544677734,
|
||
|
|
"step": 815
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7834706795652693,
|
||
|
|
"grad_norm": 30.75,
|
||
|
|
"learning_rate": 1.2235870926211616e-08,
|
||
|
|
"logits/chosen": -2.0074410438537598,
|
||
|
|
"logits/rejected": -1.998299241065979,
|
||
|
|
"logps/chosen": -0.2895830273628235,
|
||
|
|
"logps/rejected": -0.2900421619415283,
|
||
|
|
"loss": 0.6202723979949951,
|
||
|
|
"loss/core": 0.6202723979949951,
|
||
|
|
"rewards/accuracies": 0.9624999761581421,
|
||
|
|
"rewards/chosen": 2.5479187965393066,
|
||
|
|
"rewards/margins": 1.322956919670105,
|
||
|
|
"rewards/rejected": 1.2249616384506226,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.788247939806521,
|
||
|
|
"grad_norm": 1.15625,
|
||
|
|
"learning_rate": 1.0782514843499652e-08,
|
||
|
|
"logits/chosen": -2.06480073928833,
|
||
|
|
"logits/rejected": -2.2259457111358643,
|
||
|
|
"logps/chosen": -0.301548570394516,
|
||
|
|
"logps/rejected": -0.3503800928592682,
|
||
|
|
"loss": 0.5075355768203735,
|
||
|
|
"loss/core": 0.5075355768203735,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 2.2786221504211426,
|
||
|
|
"rewards/margins": 1.6066519021987915,
|
||
|
|
"rewards/rejected": 0.6719702482223511,
|
||
|
|
"step": 825
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7930252000477725,
|
||
|
|
"grad_norm": 55.0,
|
||
|
|
"learning_rate": 9.419118673676924e-09,
|
||
|
|
"logits/chosen": -2.1703529357910156,
|
||
|
|
"logits/rejected": -2.184058666229248,
|
||
|
|
"logps/chosen": -0.300544798374176,
|
||
|
|
"logps/rejected": -0.2891280949115753,
|
||
|
|
"loss": 0.2319040298461914,
|
||
|
|
"loss/core": 0.2319040298461914,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.8630399703979492,
|
||
|
|
"rewards/margins": 1.0912376642227173,
|
||
|
|
"rewards/rejected": 0.7718024849891663,
|
||
|
|
"step": 830
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.7978024602890242,
|
||
|
|
"grad_norm": 7.75,
|
||
|
|
"learning_rate": 8.14619513428405e-09,
|
||
|
|
"logits/chosen": -2.2036571502685547,
|
||
|
|
"logits/rejected": -2.212895631790161,
|
||
|
|
"logps/chosen": -0.2536885142326355,
|
||
|
|
"logps/rejected": -0.2815439701080322,
|
||
|
|
"loss": 0.3949839472770691,
|
||
|
|
"loss/core": 0.3949839472770691,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.112321138381958,
|
||
|
|
"rewards/margins": 1.3582528829574585,
|
||
|
|
"rewards/rejected": 0.7540683150291443,
|
||
|
|
"step": 835
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8025797205302759,
|
||
|
|
"grad_norm": 1256.0,
|
||
|
|
"learning_rate": 6.96422291980539e-09,
|
||
|
|
"logits/chosen": -2.035339832305908,
|
||
|
|
"logits/rejected": -2.0912997722625732,
|
||
|
|
"logps/chosen": -0.29334017634391785,
|
||
|
|
"logps/rejected": -0.29095450043678284,
|
||
|
|
"loss": 1.3885018825531006,
|
||
|
|
"loss/core": 1.3885018825531006,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 3.5305423736572266,
|
||
|
|
"rewards/margins": 2.387678861618042,
|
||
|
|
"rewards/rejected": 1.1428637504577637,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8073569807715275,
|
||
|
|
"grad_norm": 58.25,
|
||
|
|
"learning_rate": 5.8736465216517594e-09,
|
||
|
|
"logits/chosen": -1.8922698497772217,
|
||
|
|
"logits/rejected": -1.9340620040893555,
|
||
|
|
"logps/chosen": -0.30387482047080994,
|
||
|
|
"logps/rejected": -0.31475311517715454,
|
||
|
|
"loss": 1.267748236656189,
|
||
|
|
"loss/core": 1.267748236656189,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.673053503036499,
|
||
|
|
"rewards/margins": 0.4096958041191101,
|
||
|
|
"rewards/rejected": 2.263357639312744,
|
||
|
|
"step": 845
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8121342410127792,
|
||
|
|
"grad_norm": 19.0,
|
||
|
|
"learning_rate": 4.874876061005173e-09,
|
||
|
|
"logits/chosen": -1.9448999166488647,
|
||
|
|
"logits/rejected": -2.031862735748291,
|
||
|
|
"logps/chosen": -0.2876799702644348,
|
||
|
|
"logps/rejected": -0.29397642612457275,
|
||
|
|
"loss": 0.3203374445438385,
|
||
|
|
"loss/core": 0.3203374445438385,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 2.3861019611358643,
|
||
|
|
"rewards/margins": 1.2526428699493408,
|
||
|
|
"rewards/rejected": 1.1334589719772339,
|
||
|
|
"step": 850
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8169115012540308,
|
||
|
|
"grad_norm": 16.5,
|
||
|
|
"learning_rate": 3.968287134589188e-09,
|
||
|
|
"logits/chosen": -1.8941272497177124,
|
||
|
|
"logits/rejected": -1.8450911045074463,
|
||
|
|
"logps/chosen": -0.285461962223053,
|
||
|
|
"logps/rejected": -0.28394120931625366,
|
||
|
|
"loss": 0.4327033460140228,
|
||
|
|
"loss/core": 0.4327033460140228,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.151184558868408,
|
||
|
|
"rewards/margins": 0.9308632016181946,
|
||
|
|
"rewards/rejected": 1.220321536064148,
|
||
|
|
"step": 855
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8216887614952825,
|
||
|
|
"grad_norm": 5.65625,
|
||
|
|
"learning_rate": 3.154220673422192e-09,
|
||
|
|
"logits/chosen": -1.9777624607086182,
|
||
|
|
"logits/rejected": -2.0467350482940674,
|
||
|
|
"logps/chosen": -0.28861913084983826,
|
||
|
|
"logps/rejected": -0.3350909352302551,
|
||
|
|
"loss": 0.501311182975769,
|
||
|
|
"loss/core": 0.501311182975769,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.9023014307022095,
|
||
|
|
"rewards/margins": 0.7318955659866333,
|
||
|
|
"rewards/rejected": 1.1704059839248657,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.826466021736534,
|
||
|
|
"grad_norm": 105.5,
|
||
|
|
"learning_rate": 2.4329828146074096e-09,
|
||
|
|
"logits/chosen": -1.8944776058197021,
|
||
|
|
"logits/rejected": -1.953665018081665,
|
||
|
|
"logps/chosen": -0.2609250247478485,
|
||
|
|
"logps/rejected": -0.29491904377937317,
|
||
|
|
"loss": 0.4003438353538513,
|
||
|
|
"loss/core": 0.4003438353538513,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.2690205574035645,
|
||
|
|
"rewards/margins": 1.027840256690979,
|
||
|
|
"rewards/rejected": 1.241180181503296,
|
||
|
|
"step": 865
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8312432819777857,
|
||
|
|
"grad_norm": 924.0,
|
||
|
|
"learning_rate": 1.8048447862070714e-09,
|
||
|
|
"logits/chosen": -2.322716236114502,
|
||
|
|
"logits/rejected": -2.29233980178833,
|
||
|
|
"logps/chosen": -0.3064175248146057,
|
||
|
|
"logps/rejected": -0.29991593956947327,
|
||
|
|
"loss": 1.024073839187622,
|
||
|
|
"loss/core": 1.024073839187622,
|
||
|
|
"rewards/accuracies": 0.824999988079071,
|
||
|
|
"rewards/chosen": 2.370023012161255,
|
||
|
|
"rewards/margins": 1.0268133878707886,
|
||
|
|
"rewards/rejected": 1.343209981918335,
|
||
|
|
"step": 870
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8360205422190374,
|
||
|
|
"grad_norm": 35.5,
|
||
|
|
"learning_rate": 1.2700428052447033e-09,
|
||
|
|
"logits/chosen": -2.17930006980896,
|
||
|
|
"logits/rejected": -2.034334897994995,
|
||
|
|
"logps/chosen": -0.27654317021369934,
|
||
|
|
"logps/rejected": -0.310249388217926,
|
||
|
|
"loss": 0.34868985414505005,
|
||
|
|
"loss/core": 0.34868985414505005,
|
||
|
|
"rewards/accuracies": 0.8125,
|
||
|
|
"rewards/chosen": 1.6901212930679321,
|
||
|
|
"rewards/margins": -0.11936825513839722,
|
||
|
|
"rewards/rejected": 1.8094894886016846,
|
||
|
|
"step": 875
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.840797802460289,
|
||
|
|
"grad_norm": 12.25,
|
||
|
|
"learning_rate": 8.287779888734858e-10,
|
||
|
|
"logits/chosen": -2.0425121784210205,
|
||
|
|
"logits/rejected": -2.0631158351898193,
|
||
|
|
"logps/chosen": -0.31236395239830017,
|
||
|
|
"logps/rejected": -0.32643812894821167,
|
||
|
|
"loss": 0.7430340647697449,
|
||
|
|
"loss/core": 0.7430340647697449,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.8382582664489746,
|
||
|
|
"rewards/margins": 1.6181519031524658,
|
||
|
|
"rewards/rejected": 1.2201061248779297,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8455750627015407,
|
||
|
|
"grad_norm": 9.375,
|
||
|
|
"learning_rate": 4.812162787445062e-10,
|
||
|
|
"logits/chosen": -2.1633739471435547,
|
||
|
|
"logits/rejected": -2.2033333778381348,
|
||
|
|
"logps/chosen": -0.309976726770401,
|
||
|
|
"logps/rejected": -0.3008884787559509,
|
||
|
|
"loss": 0.05537397414445877,
|
||
|
|
"loss/core": 0.05537397414445877,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.1624144315719604,
|
||
|
|
"rewards/margins": 0.6319789290428162,
|
||
|
|
"rewards/rejected": 0.5304354429244995,
|
||
|
|
"step": 885
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8503523229427923,
|
||
|
|
"grad_norm": 201.0,
|
||
|
|
"learning_rate": 2.2748837860270265e-10,
|
||
|
|
"logits/chosen": -2.046548366546631,
|
||
|
|
"logits/rejected": -2.1142578125,
|
||
|
|
"logps/chosen": -0.33651334047317505,
|
||
|
|
"logps/rejected": -0.32960960268974304,
|
||
|
|
"loss": 0.38854843378067017,
|
||
|
|
"loss/core": 0.38854843378067017,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.0308008193969727,
|
||
|
|
"rewards/margins": 1.3045727014541626,
|
||
|
|
"rewards/rejected": 0.7262283563613892,
|
||
|
|
"step": 890
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.855129583184044,
|
||
|
|
"grad_norm": 6.53125,
|
||
|
|
"learning_rate": 6.768970513457151e-11,
|
||
|
|
"logits/chosen": -2.0591623783111572,
|
||
|
|
"logits/rejected": -2.0446290969848633,
|
||
|
|
"logps/chosen": -0.2869151532649994,
|
||
|
|
"logps/rejected": -0.2770235538482666,
|
||
|
|
"loss": 0.31208789348602295,
|
||
|
|
"loss/core": 0.31208789348602295,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.1741526126861572,
|
||
|
|
"rewards/margins": 1.1323846578598022,
|
||
|
|
"rewards/rejected": 1.0417680740356445,
|
||
|
|
"step": 895
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8599068434252956,
|
||
|
|
"grad_norm": 117.5,
|
||
|
|
"learning_rate": 1.8803520859811406e-12,
|
||
|
|
"logits/chosen": -2.1362462043762207,
|
||
|
|
"logits/rejected": -2.0990567207336426,
|
||
|
|
"logps/chosen": -0.3250855803489685,
|
||
|
|
"logps/rejected": -0.332465261220932,
|
||
|
|
"loss": 1.7968904972076416,
|
||
|
|
"loss/core": 1.7968904972076416,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 4.322796821594238,
|
||
|
|
"rewards/margins": 2.9066953659057617,
|
||
|
|
"rewards/rejected": 1.4161012172698975,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8599068434252956,
|
||
|
|
"step": 900,
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_loss": 0.6663506550259061,
|
||
|
|
"train_runtime": 7058.8291,
|
||
|
|
"train_samples_per_second": 2.04,
|
||
|
|
"train_steps_per_second": 0.127
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 5,
|
||
|
|
"max_steps": 900,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 1,
|
||
|
|
"save_steps": 900,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": true
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 1,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|