3644 lines
134 KiB
JSON
3644 lines
134 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.8599068434252956,
|
|
"eval_steps": 500,
|
|
"global_step": 900,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"drift/chosen_abs": 0.22064828872680664,
|
|
"drift/rejected_abs": 0.06916916370391846,
|
|
"epoch": 0.004777260241251642,
|
|
"grad_norm": 756.0,
|
|
"learning_rate": 1.111111111111111e-08,
|
|
"logits/chosen": -2.3183224201202393,
|
|
"logits/rejected": -2.3036041259765625,
|
|
"logps/chosen": -0.3117493987083435,
|
|
"logps/rejected": -0.2904360890388489,
|
|
"loss": 4424.53369140625,
|
|
"loss/core": 4424.529296875,
|
|
"loss/preference_sft": 0.3117493987083435,
|
|
"loss/reference_anchor": 0.1971215307712555,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.2059829235076904,
|
|
"rewards/margins": 1.5144869089126587,
|
|
"rewards/rejected": 0.6914961338043213,
|
|
"step": 5
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.30023816227912903,
|
|
"drift/rejected_abs": 0.26469460129737854,
|
|
"epoch": 0.009554520482503284,
|
|
"grad_norm": 2288.0,
|
|
"learning_rate": 2.5e-08,
|
|
"logits/chosen": -2.06440806388855,
|
|
"logits/rejected": -2.044948101043701,
|
|
"logps/chosen": -0.2865406572818756,
|
|
"logps/rejected": -0.3020057678222656,
|
|
"loss": 4440.205078125,
|
|
"loss/core": 4440.19384765625,
|
|
"loss/preference_sft": 0.2865406572818756,
|
|
"loss/reference_anchor": 0.5595167279243469,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.0006680488586426,
|
|
"rewards/margins": 0.35411542654037476,
|
|
"rewards/rejected": 2.646552324295044,
|
|
"step": 10
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17687278985977173,
|
|
"drift/rejected_abs": 0.10483698546886444,
|
|
"epoch": 0.014331780723754926,
|
|
"grad_norm": 1672.0,
|
|
"learning_rate": 3.888888888888889e-08,
|
|
"logits/chosen": -2.229069709777832,
|
|
"logits/rejected": -2.2289915084838867,
|
|
"logps/chosen": -0.2580825984477997,
|
|
"logps/rejected": -0.27230164408683777,
|
|
"loss": 4434.91748046875,
|
|
"loss/core": 4434.9140625,
|
|
"loss/preference_sft": 0.2580825984477997,
|
|
"loss/reference_anchor": 0.13444273173809052,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.7682386636734009,
|
|
"rewards/margins": 0.7218483090400696,
|
|
"rewards/rejected": 1.046390414237976,
|
|
"step": 15
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3651048541069031,
|
|
"drift/rejected_abs": 0.11754413694143295,
|
|
"epoch": 0.01910904096500657,
|
|
"grad_norm": 680.0,
|
|
"learning_rate": 5.2777777777777776e-08,
|
|
"logits/chosen": -2.1665427684783936,
|
|
"logits/rejected": -2.18465518951416,
|
|
"logps/chosen": -0.272116094827652,
|
|
"logps/rejected": -0.27083370089530945,
|
|
"loss": 4412.21240234375,
|
|
"loss/core": 4412.20166015625,
|
|
"loss/preference_sft": 0.272116094827652,
|
|
"loss/reference_anchor": 0.5144023895263672,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.6496434211730957,
|
|
"rewards/margins": 2.475973606109619,
|
|
"rewards/rejected": 1.1736698150634766,
|
|
"step": 20
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2144162654876709,
|
|
"drift/rejected_abs": 0.09806983172893524,
|
|
"epoch": 0.02388630120625821,
|
|
"grad_norm": 960.0,
|
|
"learning_rate": 6.666666666666667e-08,
|
|
"logits/chosen": -2.008526563644409,
|
|
"logits/rejected": -2.124323844909668,
|
|
"logps/chosen": -0.27961307764053345,
|
|
"logps/rejected": -0.27083954215049744,
|
|
"loss": 4428.9775390625,
|
|
"loss/core": 4428.97509765625,
|
|
"loss/preference_sft": 0.27961307764053345,
|
|
"loss/reference_anchor": 0.08990418910980225,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 2.143292188644409,
|
|
"rewards/margins": 1.1632330417633057,
|
|
"rewards/rejected": 0.9800591468811035,
|
|
"step": 25
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2801198363304138,
|
|
"drift/rejected_abs": 0.1081269234418869,
|
|
"epoch": 0.028663561447509853,
|
|
"grad_norm": 688.0,
|
|
"learning_rate": 8.055555555555555e-08,
|
|
"logits/chosen": -2.133108615875244,
|
|
"logits/rejected": -2.1352686882019043,
|
|
"logps/chosen": -0.2811334729194641,
|
|
"logps/rejected": -0.2618844211101532,
|
|
"loss": 4421.5927734375,
|
|
"loss/core": 4421.58642578125,
|
|
"loss/preference_sft": 0.2811334729194641,
|
|
"loss/reference_anchor": 0.2856946587562561,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.8003880977630615,
|
|
"rewards/margins": 1.7426341772079468,
|
|
"rewards/rejected": 1.0577540397644043,
|
|
"step": 30
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.252098023891449,
|
|
"drift/rejected_abs": 0.07856006920337677,
|
|
"epoch": 0.033440821688761495,
|
|
"grad_norm": 442.0,
|
|
"learning_rate": 9.444444444444444e-08,
|
|
"logits/chosen": -2.2716076374053955,
|
|
"logits/rejected": -2.2810473442077637,
|
|
"logps/chosen": -0.2923678755760193,
|
|
"logps/rejected": -0.2761971056461334,
|
|
"loss": 4421.9375,
|
|
"loss/core": 4421.9287109375,
|
|
"loss/preference_sft": 0.2923678755760193,
|
|
"loss/reference_anchor": 0.4119436740875244,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.5209803581237793,
|
|
"rewards/margins": 1.7362782955169678,
|
|
"rewards/rejected": 0.784701943397522,
|
|
"step": 35
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2930517792701721,
|
|
"drift/rejected_abs": 0.09838008880615234,
|
|
"epoch": 0.03821808193001314,
|
|
"grad_norm": 1432.0,
|
|
"learning_rate": 1.0833333333333334e-07,
|
|
"logits/chosen": -2.2542996406555176,
|
|
"logits/rejected": -2.35125994682312,
|
|
"logps/chosen": -0.2655903697013855,
|
|
"logps/rejected": -0.2916432023048401,
|
|
"loss": 4415.17236328125,
|
|
"loss/core": 4415.16552734375,
|
|
"loss/preference_sft": 0.2655903697013855,
|
|
"loss/reference_anchor": 0.33125585317611694,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.9291985034942627,
|
|
"rewards/margins": 2.2448573112487793,
|
|
"rewards/rejected": 0.6843410730361938,
|
|
"step": 40
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.30555135011672974,
|
|
"drift/rejected_abs": 0.1278877556324005,
|
|
"epoch": 0.04299534217126478,
|
|
"grad_norm": 872.0,
|
|
"learning_rate": 1.2222222222222222e-07,
|
|
"logits/chosen": -2.0905721187591553,
|
|
"logits/rejected": -2.145535945892334,
|
|
"logps/chosen": -0.2696756422519684,
|
|
"logps/rejected": -0.27680477499961853,
|
|
"loss": 4421.12060546875,
|
|
"loss/core": 4421.1123046875,
|
|
"loss/preference_sft": 0.2696756422519684,
|
|
"loss/reference_anchor": 0.4080870747566223,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 3.0551581382751465,
|
|
"rewards/margins": 1.7777941226959229,
|
|
"rewards/rejected": 1.277363896369934,
|
|
"step": 45
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1658797562122345,
|
|
"drift/rejected_abs": 0.12210337072610855,
|
|
"epoch": 0.04777260241251642,
|
|
"grad_norm": 2928.0,
|
|
"learning_rate": 1.3611111111111108e-07,
|
|
"logits/chosen": -2.1889004707336426,
|
|
"logits/rejected": -2.2628321647644043,
|
|
"logps/chosen": -0.2878100872039795,
|
|
"logps/rejected": -0.2691812217235565,
|
|
"loss": 4438.6357421875,
|
|
"loss/core": 4438.63232421875,
|
|
"loss/preference_sft": 0.2878100872039795,
|
|
"loss/reference_anchor": 0.10800884664058685,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.6577980518341064,
|
|
"rewards/margins": 0.4397946894168854,
|
|
"rewards/rejected": 1.218003273010254,
|
|
"step": 50
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23436591029167175,
|
|
"drift/rejected_abs": 0.07058925181627274,
|
|
"epoch": 0.05254986265376806,
|
|
"grad_norm": 3184.0,
|
|
"learning_rate": 1.5e-07,
|
|
"logits/chosen": -2.076425552368164,
|
|
"logits/rejected": -2.139699697494507,
|
|
"logps/chosen": -0.28844743967056274,
|
|
"logps/rejected": -0.28838062286376953,
|
|
"loss": 4422.97802734375,
|
|
"loss/core": 4422.97216796875,
|
|
"loss/preference_sft": 0.28844743967056274,
|
|
"loss/reference_anchor": 0.25497233867645264,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.3433704376220703,
|
|
"rewards/margins": 1.6407816410064697,
|
|
"rewards/rejected": 0.7025889158248901,
|
|
"step": 55
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3793022632598877,
|
|
"drift/rejected_abs": 0.1703735888004303,
|
|
"epoch": 0.057327122895019705,
|
|
"grad_norm": 2256.0,
|
|
"learning_rate": 1.6388888888888888e-07,
|
|
"logits/chosen": -2.1773691177368164,
|
|
"logits/rejected": -2.124330520629883,
|
|
"logps/chosen": -0.2736962139606476,
|
|
"logps/rejected": -0.2769095301628113,
|
|
"loss": 4417.6767578125,
|
|
"loss/core": 4417.66015625,
|
|
"loss/preference_sft": 0.2736962139606476,
|
|
"loss/reference_anchor": 0.7732974886894226,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.7925236225128174,
|
|
"rewards/margins": 2.0935792922973633,
|
|
"rewards/rejected": 1.698943853378296,
|
|
"step": 60
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.32665160298347473,
|
|
"drift/rejected_abs": 0.11409376561641693,
|
|
"epoch": 0.06210438313627135,
|
|
"grad_norm": 3920.0,
|
|
"learning_rate": 1.7777777777777776e-07,
|
|
"logits/chosen": -2.108382225036621,
|
|
"logits/rejected": -2.1682333946228027,
|
|
"logps/chosen": -0.29594534635543823,
|
|
"logps/rejected": -0.2858942151069641,
|
|
"loss": 4416.6337890625,
|
|
"loss/core": 4416.6220703125,
|
|
"loss/preference_sft": 0.29594534635543823,
|
|
"loss/reference_anchor": 0.552937388420105,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.2631747722625732,
|
|
"rewards/margins": 2.1304943561553955,
|
|
"rewards/rejected": 1.1326806545257568,
|
|
"step": 65
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2492091953754425,
|
|
"drift/rejected_abs": 0.1481395810842514,
|
|
"epoch": 0.06688164337752299,
|
|
"grad_norm": 1208.0,
|
|
"learning_rate": 1.9166666666666668e-07,
|
|
"logits/chosen": -2.1356945037841797,
|
|
"logits/rejected": -2.046206474304199,
|
|
"logps/chosen": -0.27981576323509216,
|
|
"logps/rejected": -0.2755919098854065,
|
|
"loss": 4431.11767578125,
|
|
"loss/core": 4431.11328125,
|
|
"loss/preference_sft": 0.27981576323509216,
|
|
"loss/reference_anchor": 0.20977923274040222,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.4919538497924805,
|
|
"rewards/margins": 1.011785864830017,
|
|
"rewards/rejected": 1.4801679849624634,
|
|
"step": 70
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22614233195781708,
|
|
"drift/rejected_abs": 0.11433468014001846,
|
|
"epoch": 0.07165890361877464,
|
|
"grad_norm": 584.0,
|
|
"learning_rate": 2.0555555555555553e-07,
|
|
"logits/chosen": -2.0366768836975098,
|
|
"logits/rejected": -2.0689096450805664,
|
|
"logps/chosen": -0.30070480704307556,
|
|
"logps/rejected": -0.3065111041069031,
|
|
"loss": 4428.748046875,
|
|
"loss/core": 4428.7421875,
|
|
"loss/preference_sft": 0.30070480704307556,
|
|
"loss/reference_anchor": 0.25870731472969055,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.259551525115967,
|
|
"rewards/margins": 1.1922428607940674,
|
|
"rewards/rejected": 1.067308783531189,
|
|
"step": 75
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.13649195432662964,
|
|
"drift/rejected_abs": 0.0969230979681015,
|
|
"epoch": 0.07643616386002627,
|
|
"grad_norm": 392.0,
|
|
"learning_rate": 2.1944444444444442e-07,
|
|
"logits/chosen": -2.2698612213134766,
|
|
"logits/rejected": -2.2682509422302246,
|
|
"logps/chosen": -0.2945755422115326,
|
|
"logps/rejected": -0.28585803508758545,
|
|
"loss": 4439.20751953125,
|
|
"loss/core": 4439.205078125,
|
|
"loss/preference_sft": 0.2945755422115326,
|
|
"loss/reference_anchor": 0.09729237854480743,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.3633534908294678,
|
|
"rewards/margins": 0.39599722623825073,
|
|
"rewards/rejected": 0.9673563241958618,
|
|
"step": 80
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1682918667793274,
|
|
"drift/rejected_abs": 0.16686959564685822,
|
|
"epoch": 0.08121342410127792,
|
|
"grad_norm": 9024.0,
|
|
"learning_rate": 2.3333333333333333e-07,
|
|
"logits/chosen": -2.2202513217926025,
|
|
"logits/rejected": -2.233330011367798,
|
|
"logps/chosen": -0.2740824520587921,
|
|
"logps/rejected": -0.2770374119281769,
|
|
"loss": 4444.6669921875,
|
|
"loss/core": 4444.66064453125,
|
|
"loss/preference_sft": 0.2740824520587921,
|
|
"loss/reference_anchor": 0.2943022847175598,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.6820385456085205,
|
|
"rewards/margins": 0.015296387486159801,
|
|
"rewards/rejected": 1.6667420864105225,
|
|
"step": 85
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25606733560562134,
|
|
"drift/rejected_abs": 0.16503527760505676,
|
|
"epoch": 0.08599068434252956,
|
|
"grad_norm": 1632.0,
|
|
"learning_rate": 2.4722222222222224e-07,
|
|
"logits/chosen": -2.130919933319092,
|
|
"logits/rejected": -2.2292962074279785,
|
|
"logps/chosen": -0.2705443799495697,
|
|
"logps/rejected": -0.3077170252799988,
|
|
"loss": 4431.96630859375,
|
|
"loss/core": 4431.95703125,
|
|
"loss/preference_sft": 0.2705443799495697,
|
|
"loss/reference_anchor": 0.43878602981567383,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.5599331855773926,
|
|
"rewards/margins": 0.9912513494491577,
|
|
"rewards/rejected": 1.5686819553375244,
|
|
"step": 90
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22851324081420898,
|
|
"drift/rejected_abs": 0.10233010351657867,
|
|
"epoch": 0.09076794458378121,
|
|
"grad_norm": 2288.0,
|
|
"learning_rate": 2.4998495746616845e-07,
|
|
"logits/chosen": -2.1402933597564697,
|
|
"logits/rejected": -2.06827974319458,
|
|
"logps/chosen": -0.28999412059783936,
|
|
"logps/rejected": -0.2968214154243469,
|
|
"loss": 4427.84716796875,
|
|
"loss/core": 4427.84228515625,
|
|
"loss/preference_sft": 0.28999412059783936,
|
|
"loss/reference_anchor": 0.24312031269073486,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.2837419509887695,
|
|
"rewards/margins": 1.2656266689300537,
|
|
"rewards/rejected": 1.0181152820587158,
|
|
"step": 95
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.37163451313972473,
|
|
"drift/rejected_abs": 0.1659991592168808,
|
|
"epoch": 0.09554520482503284,
|
|
"grad_norm": 836.0,
|
|
"learning_rate": 2.4992385337738696e-07,
|
|
"logits/chosen": -2.1171348094940186,
|
|
"logits/rejected": -2.160855531692505,
|
|
"logps/chosen": -0.268626868724823,
|
|
"logps/rejected": -0.28261902928352356,
|
|
"loss": 4417.1259765625,
|
|
"loss/core": 4417.11474609375,
|
|
"loss/preference_sft": 0.268626868724823,
|
|
"loss/reference_anchor": 0.558134138584137,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.716301679611206,
|
|
"rewards/margins": 2.0977909564971924,
|
|
"rewards/rejected": 1.6185109615325928,
|
|
"step": 100
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3358571529388428,
|
|
"drift/rejected_abs": 0.08821113407611847,
|
|
"epoch": 0.10032246506628449,
|
|
"grad_norm": 1464.0,
|
|
"learning_rate": 2.4981577053636144e-07,
|
|
"logits/chosen": -2.0699539184570312,
|
|
"logits/rejected": -2.0407674312591553,
|
|
"logps/chosen": -0.31734392046928406,
|
|
"logps/rejected": -0.30329352617263794,
|
|
"loss": 4411.2470703125,
|
|
"loss/core": 4411.2373046875,
|
|
"loss/preference_sft": 0.31734392046928406,
|
|
"loss/reference_anchor": 0.4697941243648529,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.355891466140747,
|
|
"rewards/margins": 2.551638126373291,
|
|
"rewards/rejected": 0.8042529821395874,
|
|
"step": 105
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16569358110427856,
|
|
"drift/rejected_abs": 0.0958634614944458,
|
|
"epoch": 0.10509972530753613,
|
|
"grad_norm": 584.0,
|
|
"learning_rate": 2.496607495886281e-07,
|
|
"logits/chosen": -2.129267930984497,
|
|
"logits/rejected": -2.1310436725616455,
|
|
"logps/chosen": -0.29805296659469604,
|
|
"logps/rejected": -0.3116980493068695,
|
|
"loss": 4435.1533203125,
|
|
"loss/core": 4435.1513671875,
|
|
"loss/preference_sft": 0.29805296659469604,
|
|
"loss/reference_anchor": 0.10324454307556152,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.6569359302520752,
|
|
"rewards/margins": 0.7000143527984619,
|
|
"rewards/rejected": 0.956921398639679,
|
|
"step": 110
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.29079359769821167,
|
|
"drift/rejected_abs": 0.1145281046628952,
|
|
"epoch": 0.10987698554878778,
|
|
"grad_norm": 1192.0,
|
|
"learning_rate": 2.4945884883122553e-07,
|
|
"logits/chosen": -2.0063235759735107,
|
|
"logits/rejected": -2.1220145225524902,
|
|
"logps/chosen": -0.3002547025680542,
|
|
"logps/rejected": -0.2964302897453308,
|
|
"loss": 4421.4990234375,
|
|
"loss/core": 4421.486328125,
|
|
"loss/preference_sft": 0.3002547025680542,
|
|
"loss/reference_anchor": 0.597941517829895,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.9075615406036377,
|
|
"rewards/margins": 1.7633129358291626,
|
|
"rewards/rejected": 1.1442487239837646,
|
|
"step": 115
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2360951155424118,
|
|
"drift/rejected_abs": 0.12227632850408554,
|
|
"epoch": 0.11465424579003941,
|
|
"grad_norm": 2736.0,
|
|
"learning_rate": 2.492101441907714e-07,
|
|
"logits/chosen": -2.2496066093444824,
|
|
"logits/rejected": -2.261533498764038,
|
|
"logps/chosen": -0.304383784532547,
|
|
"logps/rejected": -0.30693715810775757,
|
|
"loss": 4428.9970703125,
|
|
"loss/core": 4428.9921875,
|
|
"loss/preference_sft": 0.304383784532547,
|
|
"loss/reference_anchor": 0.24186281859874725,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.359388828277588,
|
|
"rewards/margins": 1.1752779483795166,
|
|
"rewards/rejected": 1.1841111183166504,
|
|
"step": 120
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1892295777797699,
|
|
"drift/rejected_abs": 0.22446322441101074,
|
|
"epoch": 0.11943150603129106,
|
|
"grad_norm": 808.0,
|
|
"learning_rate": 2.4891472919490977e-07,
|
|
"logits/chosen": -2.0349624156951904,
|
|
"logits/rejected": -1.9459272623062134,
|
|
"logps/chosen": -0.292393296957016,
|
|
"logps/rejected": -0.2992958724498749,
|
|
"loss": 4449.83154296875,
|
|
"loss/core": 4449.8212890625,
|
|
"loss/preference_sft": 0.292393296957016,
|
|
"loss/reference_anchor": 0.48200732469558716,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.8902618885040283,
|
|
"rewards/margins": -0.35318875312805176,
|
|
"rewards/rejected": 2.24345064163208,
|
|
"step": 125
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22298288345336914,
|
|
"drift/rejected_abs": 0.20003148913383484,
|
|
"epoch": 0.1242087662725427,
|
|
"grad_norm": 2336.0,
|
|
"learning_rate": 2.4857271493713894e-07,
|
|
"logits/chosen": -2.3568854331970215,
|
|
"logits/rejected": -2.2751965522766113,
|
|
"logps/chosen": -0.2751834988594055,
|
|
"logps/rejected": -0.298502653837204,
|
|
"loss": 4441.8662109375,
|
|
"loss/core": 4441.8583984375,
|
|
"loss/preference_sft": 0.2751834988594055,
|
|
"loss/reference_anchor": 0.33170682191848755,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.2295897006988525,
|
|
"rewards/margins": 0.2301691472530365,
|
|
"rewards/rejected": 1.9994205236434937,
|
|
"step": 130
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27084535360336304,
|
|
"drift/rejected_abs": 0.1473771631717682,
|
|
"epoch": 0.12898602651379434,
|
|
"grad_norm": 1432.0,
|
|
"learning_rate": 2.481842300350339e-07,
|
|
"logits/chosen": -2.086169958114624,
|
|
"logits/rejected": -2.182788848876953,
|
|
"logps/chosen": -0.34438925981521606,
|
|
"logps/rejected": -0.3138183355331421,
|
|
"loss": 4428.39990234375,
|
|
"loss/core": 4428.39208984375,
|
|
"loss/preference_sft": 0.34438925981521606,
|
|
"loss/reference_anchor": 0.338693231344223,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.7076194286346436,
|
|
"rewards/margins": 1.236877679824829,
|
|
"rewards/rejected": 1.4707417488098145,
|
|
"step": 135
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3097183108329773,
|
|
"drift/rejected_abs": 0.24102067947387695,
|
|
"epoch": 0.13376328675504598,
|
|
"grad_norm": 1904.0,
|
|
"learning_rate": 2.4774942058187854e-07,
|
|
"logits/chosen": -1.950994849205017,
|
|
"logits/rejected": -1.937705397605896,
|
|
"logps/chosen": -0.3324321210384369,
|
|
"logps/rejected": -0.33997035026550293,
|
|
"loss": 4436.19677734375,
|
|
"loss/core": 4436.18359375,
|
|
"loss/preference_sft": 0.3324321210384369,
|
|
"loss/reference_anchor": 0.619308590888977,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.096101760864258,
|
|
"rewards/margins": 0.6867878437042236,
|
|
"rewards/rejected": 2.409313678741455,
|
|
"step": 140
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17987477779388428,
|
|
"drift/rejected_abs": 0.08481304347515106,
|
|
"epoch": 0.13854054699629761,
|
|
"grad_norm": 1048.0,
|
|
"learning_rate": 2.472684500917257e-07,
|
|
"logits/chosen": -2.043088912963867,
|
|
"logits/rejected": -2.053077220916748,
|
|
"logps/chosen": -0.300198495388031,
|
|
"logps/rejected": -0.2859780192375183,
|
|
"loss": 4431.86328125,
|
|
"loss/core": 4431.8603515625,
|
|
"loss/preference_sft": 0.300198495388031,
|
|
"loss/reference_anchor": 0.10386216640472412,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.7945579290390015,
|
|
"rewards/margins": 0.9506572484970093,
|
|
"rewards/rejected": 0.8439006805419922,
|
|
"step": 145
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2635027766227722,
|
|
"drift/rejected_abs": 0.12567314505577087,
|
|
"epoch": 0.14331780723754928,
|
|
"grad_norm": 3248.0,
|
|
"learning_rate": 2.467414994379065e-07,
|
|
"logits/chosen": -2.0115456581115723,
|
|
"logits/rejected": -2.035735845565796,
|
|
"logps/chosen": -0.28563544154167175,
|
|
"logps/rejected": -0.2719977796077728,
|
|
"loss": 4426.4873046875,
|
|
"loss/core": 4426.47998046875,
|
|
"loss/preference_sft": 0.28563544154167175,
|
|
"loss/reference_anchor": 0.3351196348667145,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.6349034309387207,
|
|
"rewards/margins": 1.3796746730804443,
|
|
"rewards/rejected": 1.2552287578582764,
|
|
"step": 150
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26199209690093994,
|
|
"drift/rejected_abs": 0.14406320452690125,
|
|
"epoch": 0.1480950674788009,
|
|
"grad_norm": 1020.0,
|
|
"learning_rate": 2.4616876678501114e-07,
|
|
"logits/chosen": -1.9780941009521484,
|
|
"logits/rejected": -2.0333588123321533,
|
|
"logps/chosen": -0.24971485137939453,
|
|
"logps/rejected": -0.26529937982559204,
|
|
"loss": 4428.74267578125,
|
|
"loss/core": 4428.73779296875,
|
|
"loss/preference_sft": 0.24971485137939453,
|
|
"loss/reference_anchor": 0.19584576785564423,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.6188275814056396,
|
|
"rewards/margins": 1.1842237710952759,
|
|
"rewards/rejected": 1.4346040487289429,
|
|
"step": 155
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2649013102054596,
|
|
"drift/rejected_abs": 0.11616607755422592,
|
|
"epoch": 0.15287232772005255,
|
|
"grad_norm": 1984.0,
|
|
"learning_rate": 2.4555046751436735e-07,
|
|
"logits/chosen": -2.1708004474639893,
|
|
"logits/rejected": -2.1445536613464355,
|
|
"logps/chosen": -0.2795725464820862,
|
|
"logps/rejected": -0.2928757071495056,
|
|
"loss": 4425.0234375,
|
|
"loss/core": 4425.0166015625,
|
|
"loss/preference_sft": 0.2795725464820862,
|
|
"loss/reference_anchor": 0.29672375321388245,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.647858142852783,
|
|
"rewards/margins": 1.4861972332000732,
|
|
"rewards/rejected": 1.1616607904434204,
|
|
"step": 160
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17845085263252258,
|
|
"drift/rejected_abs": 0.09973239153623581,
|
|
"epoch": 0.15764958796130418,
|
|
"grad_norm": 848.0,
|
|
"learning_rate": 2.4488683414304425e-07,
|
|
"logits/chosen": -2.1181225776672363,
|
|
"logits/rejected": -2.0922858715057373,
|
|
"logps/chosen": -0.3071829080581665,
|
|
"logps/rejected": -0.30448269844055176,
|
|
"loss": 4434.00048828125,
|
|
"loss/core": 4433.998046875,
|
|
"loss/preference_sft": 0.3071829080581665,
|
|
"loss/reference_anchor": 0.09884973615407944,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.7831709384918213,
|
|
"rewards/margins": 0.7861735820770264,
|
|
"rewards/rejected": 0.9969974756240845,
|
|
"step": 165
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18412050604820251,
|
|
"drift/rejected_abs": 0.0624062642455101,
|
|
"epoch": 0.16242684820255585,
|
|
"grad_norm": 450.0,
|
|
"learning_rate": 2.4417811623641203e-07,
|
|
"logits/chosen": -2.074202537536621,
|
|
"logits/rejected": -2.0019283294677734,
|
|
"logps/chosen": -0.30903059244155884,
|
|
"logps/rejected": -0.3260036110877991,
|
|
"loss": 4428.12548828125,
|
|
"loss/core": 4428.123046875,
|
|
"loss/preference_sft": 0.30903059244155884,
|
|
"loss/reference_anchor": 0.10582767426967621,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.8374769687652588,
|
|
"rewards/margins": 1.235172152519226,
|
|
"rewards/rejected": 0.6023045182228088,
|
|
"step": 170
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17877274751663208,
|
|
"drift/rejected_abs": 0.07043075561523438,
|
|
"epoch": 0.16720410844380748,
|
|
"grad_norm": 2608.0,
|
|
"learning_rate": 2.4342458031429113e-07,
|
|
"logits/chosen": -2.2448172569274902,
|
|
"logits/rejected": -2.2653393745422363,
|
|
"logps/chosen": -0.2739713490009308,
|
|
"logps/rejected": -0.28165867924690247,
|
|
"loss": 4430.08447265625,
|
|
"loss/core": 4430.08154296875,
|
|
"loss/preference_sft": 0.2739713490009308,
|
|
"loss/reference_anchor": 0.09590987861156464,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.787602424621582,
|
|
"rewards/margins": 1.0849480628967285,
|
|
"rewards/rejected": 0.702654242515564,
|
|
"step": 175
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2464297115802765,
|
|
"drift/rejected_abs": 0.13828793168067932,
|
|
"epoch": 0.17198136868505912,
|
|
"grad_norm": 1020.0,
|
|
"learning_rate": 2.4262650975072444e-07,
|
|
"logits/chosen": -2.204326629638672,
|
|
"logits/rejected": -2.1553001403808594,
|
|
"logps/chosen": -0.2618137300014496,
|
|
"logps/rejected": -0.28036749362945557,
|
|
"loss": 4430.1103515625,
|
|
"loss/core": 4430.10302734375,
|
|
"loss/preference_sft": 0.2618137300014496,
|
|
"loss/reference_anchor": 0.3428693115711212,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.463837146759033,
|
|
"rewards/margins": 1.0816283226013184,
|
|
"rewards/rejected": 1.3822085857391357,
|
|
"step": 180
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19091588258743286,
|
|
"drift/rejected_abs": 0.15579600632190704,
|
|
"epoch": 0.17675862892631075,
|
|
"grad_norm": 6464.0,
|
|
"learning_rate": 2.417842046674122e-07,
|
|
"logits/chosen": -2.212608814239502,
|
|
"logits/rejected": -2.253899574279785,
|
|
"logps/chosen": -0.30273327231407166,
|
|
"logps/rejected": -0.302903950214386,
|
|
"loss": 4440.3125,
|
|
"loss/core": 4440.30517578125,
|
|
"loss/preference_sft": 0.30273327231407166,
|
|
"loss/reference_anchor": 0.3576907217502594,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.9086692333221436,
|
|
"rewards/margins": 0.3515912890434265,
|
|
"rewards/rejected": 1.5570778846740723,
|
|
"step": 185
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2642178237438202,
|
|
"drift/rejected_abs": 0.11951635032892227,
|
|
"epoch": 0.18153588916756241,
|
|
"grad_norm": 7584.0,
|
|
"learning_rate": 2.4089798182084843e-07,
|
|
"logits/chosen": -2.1424639225006104,
|
|
"logits/rejected": -2.1239447593688965,
|
|
"logps/chosen": -0.28714102506637573,
|
|
"logps/rejected": -0.270624577999115,
|
|
"loss": 4425.54296875,
|
|
"loss/core": 4425.5361328125,
|
|
"loss/preference_sft": 0.28714102506637573,
|
|
"loss/reference_anchor": 0.32669633626937866,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.6386032104492188,
|
|
"rewards/margins": 1.4514687061309814,
|
|
"rewards/rejected": 1.1871345043182373,
|
|
"step": 190
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1681932508945465,
|
|
"drift/rejected_abs": 0.13734407722949982,
|
|
"epoch": 0.18631314940881405,
|
|
"grad_norm": 752.0,
|
|
"learning_rate": 2.3996817448320195e-07,
|
|
"logits/chosen": -1.964268445968628,
|
|
"logits/rejected": -1.9507757425308228,
|
|
"logps/chosen": -0.28487104177474976,
|
|
"logps/rejected": -0.29054194688796997,
|
|
"loss": 4440.4189453125,
|
|
"loss/core": 4440.416015625,
|
|
"loss/preference_sft": 0.28487104177474976,
|
|
"loss/reference_anchor": 0.1267441213130951,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.6816270351409912,
|
|
"rewards/margins": 0.30894654989242554,
|
|
"rewards/rejected": 1.3726806640625,
|
|
"step": 195
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25091999769210815,
|
|
"drift/rejected_abs": 0.14358346164226532,
|
|
"epoch": 0.19109040965006568,
|
|
"grad_norm": 1408.0,
|
|
"learning_rate": 2.3899513231698607e-07,
|
|
"logits/chosen": -2.1187353134155273,
|
|
"logits/rejected": -2.150465488433838,
|
|
"logps/chosen": -0.2739163041114807,
|
|
"logps/rejected": -0.27358219027519226,
|
|
"loss": 4429.14306640625,
|
|
"loss/core": 4429.1376953125,
|
|
"loss/preference_sft": 0.2739163041114807,
|
|
"loss/reference_anchor": 0.23309831321239471,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.507127523422241,
|
|
"rewards/margins": 1.161219835281372,
|
|
"rewards/rejected": 1.3459078073501587,
|
|
"step": 200
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.13701114058494568,
|
|
"drift/rejected_abs": 0.06575393676757812,
|
|
"epoch": 0.19586766989131732,
|
|
"grad_norm": 716.0,
|
|
"learning_rate": 2.3797922124356506e-07,
|
|
"logits/chosen": -2.0870461463928223,
|
|
"logits/rejected": -2.1504645347595215,
|
|
"logps/chosen": -0.2821192443370819,
|
|
"logps/rejected": -0.28834599256515503,
|
|
"loss": 4434.98291015625,
|
|
"loss/core": 4434.9814453125,
|
|
"loss/preference_sft": 0.2821192443370819,
|
|
"loss/reference_anchor": 0.043852418661117554,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.3701115846633911,
|
|
"rewards/margins": 0.7137712240219116,
|
|
"rewards/rejected": 0.6563402414321899,
|
|
"step": 205
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25299134850502014,
|
|
"drift/rejected_abs": 0.1516466736793518,
|
|
"epoch": 0.20064493013256898,
|
|
"grad_norm": 1152.0,
|
|
"learning_rate": 2.3692082330554585e-07,
|
|
"logits/chosen": -2.03848934173584,
|
|
"logits/rejected": -2.016936779022217,
|
|
"logps/chosen": -0.28784194588661194,
|
|
"logps/rejected": -0.28754788637161255,
|
|
"loss": 4430.9228515625,
|
|
"loss/core": 4430.91357421875,
|
|
"loss/preference_sft": 0.28784194588661194,
|
|
"loss/reference_anchor": 0.4292024075984955,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.5295217037200928,
|
|
"rewards/margins": 1.0197639465332031,
|
|
"rewards/rejected": 1.5097577571868896,
|
|
"step": 210
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20603258907794952,
|
|
"drift/rejected_abs": 0.10440722852945328,
|
|
"epoch": 0.20542219037382062,
|
|
"grad_norm": 760.0,
|
|
"learning_rate": 2.3582033652310765e-07,
|
|
"logits/chosen": -1.955307960510254,
|
|
"logits/rejected": -1.973205804824829,
|
|
"logps/chosen": -0.2961030900478363,
|
|
"logps/rejected": -0.27407750487327576,
|
|
"loss": 4430.9453125,
|
|
"loss/core": 4430.9423828125,
|
|
"loss/preference_sft": 0.2961030900478363,
|
|
"loss/reference_anchor": 0.12213130295276642,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.0588302612304688,
|
|
"rewards/margins": 1.0177305936813354,
|
|
"rewards/rejected": 1.0410997867584229,
|
|
"step": 215
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1778123527765274,
|
|
"drift/rejected_abs": 0.08586692065000534,
|
|
"epoch": 0.21019945061507225,
|
|
"grad_norm": 656.0,
|
|
"learning_rate": 2.346781747443227e-07,
|
|
"logits/chosen": -2.037010669708252,
|
|
"logits/rejected": -2.0615968704223633,
|
|
"logps/chosen": -0.28160199522972107,
|
|
"logps/rejected": -0.2732716202735901,
|
|
"loss": 4432.251953125,
|
|
"loss/core": 4432.24951171875,
|
|
"loss/preference_sft": 0.28160199522972107,
|
|
"loss/reference_anchor": 0.08426757156848907,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.7759720087051392,
|
|
"rewards/margins": 0.9196584820747375,
|
|
"rewards/rejected": 0.8563135266304016,
|
|
"step": 220
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3073180913925171,
|
|
"drift/rejected_abs": 0.20542486011981964,
|
|
"epoch": 0.2149767108563239,
|
|
"grad_norm": 1480.0,
|
|
"learning_rate": 2.3349476748952508e-07,
|
|
"logits/chosen": -2.0461509227752686,
|
|
"logits/rejected": -2.0668578147888184,
|
|
"logps/chosen": -0.3112167716026306,
|
|
"logps/rejected": -0.3018384873867035,
|
|
"loss": 4431.3017578125,
|
|
"loss/core": 4431.28515625,
|
|
"loss/preference_sft": 0.3112167716026306,
|
|
"loss/reference_anchor": 0.7777162194252014,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.071030855178833,
|
|
"rewards/margins": 1.018129825592041,
|
|
"rewards/rejected": 2.052901268005371,
|
|
"step": 225
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3042348027229309,
|
|
"drift/rejected_abs": 0.16846933960914612,
|
|
"epoch": 0.21975397109757555,
|
|
"grad_norm": 1632.0,
|
|
"learning_rate": 2.3227055978978545e-07,
|
|
"logits/chosen": -2.130049467086792,
|
|
"logits/rejected": -2.120723247528076,
|
|
"logps/chosen": -0.27312982082366943,
|
|
"logps/rejected": -0.26595622301101685,
|
|
"loss": 4426.55078125,
|
|
"loss/core": 4426.54296875,
|
|
"loss/preference_sft": 0.27312982082366943,
|
|
"loss/reference_anchor": 0.3708053231239319,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.0423481464385986,
|
|
"rewards/margins": 1.3577412366867065,
|
|
"rewards/rejected": 1.6846065521240234,
|
|
"step": 230
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17078641057014465,
|
|
"drift/rejected_abs": 0.06932120025157928,
|
|
"epoch": 0.2245312313388272,
|
|
"grad_norm": 544.0,
|
|
"learning_rate": 2.3100601201955321e-07,
|
|
"logits/chosen": -2.171781063079834,
|
|
"logits/rejected": -2.267810106277466,
|
|
"logps/chosen": -0.2851482927799225,
|
|
"logps/rejected": -0.279243528842926,
|
|
"loss": 4430.9599609375,
|
|
"loss/core": 4430.9580078125,
|
|
"loss/preference_sft": 0.2851482927799225,
|
|
"loss/reference_anchor": 0.10230743885040283,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.7078640460968018,
|
|
"rewards/margins": 1.018509864807129,
|
|
"rewards/rejected": 0.6893541216850281,
|
|
"step": 235
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27906155586242676,
|
|
"drift/rejected_abs": 0.19857071340084076,
|
|
"epoch": 0.22930849158007882,
|
|
"grad_norm": 1012.0,
|
|
"learning_rate": 2.2970159972352864e-07,
|
|
"logits/chosen": -2.0736007690429688,
|
|
"logits/rejected": -2.1250550746917725,
|
|
"logps/chosen": -0.2709374725818634,
|
|
"logps/rejected": -0.28649747371673584,
|
|
"loss": 4434.0234375,
|
|
"loss/core": 4434.009765625,
|
|
"loss/preference_sft": 0.2709374725818634,
|
|
"loss/reference_anchor": 0.656843900680542,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.78991436958313,
|
|
"rewards/margins": 0.8058086633682251,
|
|
"rewards/rejected": 1.9841057062149048,
|
|
"step": 240
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21625050902366638,
|
|
"drift/rejected_abs": 0.06605993956327438,
|
|
"epoch": 0.23408575182133046,
|
|
"grad_norm": 6624.0,
|
|
"learning_rate": 2.2835781343782966e-07,
|
|
"logits/chosen": -2.1297452449798584,
|
|
"logits/rejected": -2.2750277519226074,
|
|
"logps/chosen": -0.3038528859615326,
|
|
"logps/rejected": -0.29280510544776917,
|
|
"loss": 4424.94091796875,
|
|
"loss/core": 4424.9326171875,
|
|
"loss/preference_sft": 0.3038528859615326,
|
|
"loss/reference_anchor": 0.3740764260292053,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.1619296073913574,
|
|
"rewards/margins": 1.504833698272705,
|
|
"rewards/rejected": 0.6570958495140076,
|
|
"step": 245
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.33135589957237244,
|
|
"drift/rejected_abs": 0.16284796595573425,
|
|
"epoch": 0.23886301206258212,
|
|
"grad_norm": 916.0,
|
|
"learning_rate": 2.2697515850552152e-07,
|
|
"logits/chosen": -1.9698188304901123,
|
|
"logits/rejected": -1.9743677377700806,
|
|
"logps/chosen": -0.30887797474861145,
|
|
"logps/rejected": -0.29849907755851746,
|
|
"loss": 4422.46337890625,
|
|
"loss/core": 4422.45166015625,
|
|
"loss/preference_sft": 0.30887797474861145,
|
|
"loss/reference_anchor": 0.5555719137191772,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.312263011932373,
|
|
"rewards/margins": 1.6859710216522217,
|
|
"rewards/rejected": 1.6262918710708618,
|
|
"step": 250
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27730509638786316,
|
|
"drift/rejected_abs": 0.11791691929101944,
|
|
"epoch": 0.24364027230383375,
|
|
"grad_norm": 600.0,
|
|
"learning_rate": 2.2555415488657775e-07,
|
|
"logits/chosen": -2.120657444000244,
|
|
"logits/rejected": -2.0884969234466553,
|
|
"logps/chosen": -0.28511613607406616,
|
|
"logps/rejected": -0.2897268831729889,
|
|
"loss": 4423.853515625,
|
|
"loss/core": 4423.83984375,
|
|
"loss/preference_sft": 0.28511613607406616,
|
|
"loss/reference_anchor": 0.6375155448913574,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.772818088531494,
|
|
"rewards/margins": 1.5964891910552979,
|
|
"rewards/rejected": 1.1763291358947754,
|
|
"step": 255
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19447073340415955,
|
|
"drift/rejected_abs": 0.12607555091381073,
|
|
"epoch": 0.2484175325450854,
|
|
"grad_norm": 2720.0,
|
|
"learning_rate": 2.240953369623447e-07,
|
|
"logits/chosen": -2.07661771774292,
|
|
"logits/rejected": -2.0507664680480957,
|
|
"logps/chosen": -0.3053693175315857,
|
|
"logps/rejected": -0.307594358921051,
|
|
"loss": 4435.50927734375,
|
|
"loss/core": 4435.5048828125,
|
|
"loss/preference_sft": 0.3053693175315857,
|
|
"loss/reference_anchor": 0.21331624686717987,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.9445161819458008,
|
|
"rewards/margins": 0.6846358180046082,
|
|
"rewards/rejected": 1.2598803043365479,
|
|
"step": 260
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.213128000497818,
|
|
"drift/rejected_abs": 0.10632689297199249,
|
|
"epoch": 0.25319479278633705,
|
|
"grad_norm": 724.0,
|
|
"learning_rate": 2.225992533345824e-07,
|
|
"logits/chosen": -2.0516610145568848,
|
|
"logits/rejected": -2.0290393829345703,
|
|
"logps/chosen": -0.2761135995388031,
|
|
"logps/rejected": -0.28632479906082153,
|
|
"loss": 4430.1591796875,
|
|
"loss/core": 4430.15673828125,
|
|
"loss/preference_sft": 0.2761135995388031,
|
|
"loss/reference_anchor": 0.1218862310051918,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.1307318210601807,
|
|
"rewards/margins": 1.079505205154419,
|
|
"rewards/rejected": 1.0512263774871826,
|
|
"step": 265
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16613757610321045,
|
|
"drift/rejected_abs": 0.1417321413755417,
|
|
"epoch": 0.2579720530275887,
|
|
"grad_norm": 474.0,
|
|
"learning_rate": 2.210664666191579e-07,
|
|
"logits/chosen": -2.102410316467285,
|
|
"logits/rejected": -2.0517630577087402,
|
|
"logps/chosen": -0.30586427450180054,
|
|
"logps/rejected": -0.3037346303462982,
|
|
"loss": 4441.33203125,
|
|
"loss/core": 4441.328125,
|
|
"loss/preference_sft": 0.30586427450180054,
|
|
"loss/reference_anchor": 0.16727973520755768,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.6612533330917358,
|
|
"rewards/margins": 0.24511925876140594,
|
|
"rewards/rejected": 1.416133999824524,
|
|
"step": 270
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.10398060083389282,
|
|
"drift/rejected_abs": 0.05893680453300476,
|
|
"epoch": 0.2627493132688403,
|
|
"grad_norm": 820.0,
|
|
"learning_rate": 2.1949755323446848e-07,
|
|
"logits/chosen": -2.2223498821258545,
|
|
"logits/rejected": -2.292362928390503,
|
|
"logps/chosen": -0.29613521695137024,
|
|
"logps/rejected": -0.2959131598472595,
|
|
"loss": 4438.3203125,
|
|
"loss/core": 4438.3193359375,
|
|
"loss/preference_sft": 0.29613521695137024,
|
|
"loss/reference_anchor": 0.0267405416816473,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.0383059978485107,
|
|
"rewards/margins": 0.46069198846817017,
|
|
"rewards/rejected": 0.5776140093803406,
|
|
"step": 275
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18298956751823425,
|
|
"drift/rejected_abs": 0.08578231185674667,
|
|
"epoch": 0.26752657351009196,
|
|
"grad_norm": 1576.0,
|
|
"learning_rate": 2.1789310318467426e-07,
|
|
"logits/chosen": -2.0358364582061768,
|
|
"logits/rejected": -2.016176700592041,
|
|
"logps/chosen": -0.2997116446495056,
|
|
"logps/rejected": -0.29861459136009216,
|
|
"loss": 4431.60595703125,
|
|
"loss/core": 4431.60302734375,
|
|
"loss/preference_sft": 0.2997116446495056,
|
|
"loss/reference_anchor": 0.1133713498711586,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.8296573162078857,
|
|
"rewards/margins": 0.9720863103866577,
|
|
"rewards/rejected": 0.857571005821228,
|
|
"step": 280
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.29716697335243225,
|
|
"drift/rejected_abs": 0.20067524909973145,
|
|
"epoch": 0.2723038337513436,
|
|
"grad_norm": 972.0,
|
|
"learning_rate": 2.1625371983782182e-07,
|
|
"logits/chosen": -2.219372034072876,
|
|
"logits/rejected": -2.1742899417877197,
|
|
"logps/chosen": -0.28680914640426636,
|
|
"logps/rejected": -0.2844160497188568,
|
|
"loss": 4431.8857421875,
|
|
"loss/core": 4431.8740234375,
|
|
"loss/preference_sft": 0.28680914640426636,
|
|
"loss/reference_anchor": 0.5631648302078247,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.970261573791504,
|
|
"rewards/margins": 0.9673450589179993,
|
|
"rewards/rejected": 2.0029163360595703,
|
|
"step": 285
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.45473068952560425,
|
|
"drift/rejected_abs": 0.13007131218910217,
|
|
"epoch": 0.27708109399259523,
|
|
"grad_norm": 9920.0,
|
|
"learning_rate": 2.14580019698942e-07,
|
|
"logits/chosen": -2.151202440261841,
|
|
"logits/rejected": -2.2022032737731934,
|
|
"logps/chosen": -0.2817015051841736,
|
|
"logps/rejected": -0.27055931091308594,
|
|
"loss": 4402.37890625,
|
|
"loss/core": 4402.357421875,
|
|
"loss/preference_sft": 0.2817015051841736,
|
|
"loss/reference_anchor": 1.024298906326294,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.546250343322754,
|
|
"rewards/margins": 3.2471745014190674,
|
|
"rewards/rejected": 1.299075961112976,
|
|
"step": 290
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2755797505378723,
|
|
"drift/rejected_abs": 0.1132490411400795,
|
|
"epoch": 0.28185835423384686,
|
|
"grad_norm": 1424.0,
|
|
"learning_rate": 2.1287263217820773e-07,
|
|
"logits/chosen": -1.9132686853408813,
|
|
"logits/rejected": -1.9894206523895264,
|
|
"logps/chosen": -0.31108710169792175,
|
|
"logps/rejected": -0.3019738793373108,
|
|
"loss": 4423.05029296875,
|
|
"loss/core": 4423.0380859375,
|
|
"loss/preference_sft": 0.31108710169792175,
|
|
"loss/reference_anchor": 0.5742016434669495,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.755403518676758,
|
|
"rewards/margins": 1.6267465353012085,
|
|
"rewards/rejected": 1.1286571025848389,
|
|
"step": 295
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3510657846927643,
|
|
"drift/rejected_abs": 0.1981009691953659,
|
|
"epoch": 0.28663561447509855,
|
|
"grad_norm": 600.0,
|
|
"learning_rate": 2.111321993542385e-07,
|
|
"logits/chosen": -2.1004462242126465,
|
|
"logits/rejected": -2.165738582611084,
|
|
"logps/chosen": -0.2795126140117645,
|
|
"logps/rejected": -0.30045658349990845,
|
|
"loss": 4424.57080078125,
|
|
"loss/core": 4424.5498046875,
|
|
"loss/preference_sft": 0.2795126140117645,
|
|
"loss/reference_anchor": 1.045248031616211,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.509197235107422,
|
|
"rewards/margins": 1.5337461233139038,
|
|
"rewards/rejected": 1.9754512310028076,
|
|
"step": 300
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3817795217037201,
|
|
"drift/rejected_abs": 0.25070327520370483,
|
|
"epoch": 0.2914128747163502,
|
|
"grad_norm": 4544.0,
|
|
"learning_rate": 2.0935937573264096e-07,
|
|
"logits/chosen": -1.98002028465271,
|
|
"logits/rejected": -2.0424513816833496,
|
|
"logps/chosen": -0.29999712109565735,
|
|
"logps/rejected": -0.30416327714920044,
|
|
"loss": 4427.5712890625,
|
|
"loss/core": 4427.548828125,
|
|
"loss/preference_sft": 0.29999712109565735,
|
|
"loss/reference_anchor": 1.0998529195785522,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.8161988258361816,
|
|
"rewards/margins": 1.3097256422042847,
|
|
"rewards/rejected": 2.5064730644226074,
|
|
"step": 305
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23365557193756104,
|
|
"drift/rejected_abs": 0.14666993916034698,
|
|
"epoch": 0.2961901349576018,
|
|
"grad_norm": 2416.0,
|
|
"learning_rate": 2.0755482799987596e-07,
|
|
"logits/chosen": -1.9693409204483032,
|
|
"logits/rejected": -2.007472038269043,
|
|
"logps/chosen": -0.27722686529159546,
|
|
"logps/rejected": -0.27590957283973694,
|
|
"loss": 4433.0087890625,
|
|
"loss/core": 4433.00439453125,
|
|
"loss/preference_sft": 0.27722686529159546,
|
|
"loss/reference_anchor": 0.190281942486763,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.3363592624664307,
|
|
"rewards/margins": 0.8708029985427856,
|
|
"rewards/rejected": 1.4655563831329346,
|
|
"step": 310
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.11892326921224594,
|
|
"drift/rejected_abs": 0.12483175098896027,
|
|
"epoch": 0.30096739519885346,
|
|
"grad_norm": 1304.0,
|
|
"learning_rate": 2.0571923477254526e-07,
|
|
"logits/chosen": -2.1190714836120605,
|
|
"logits/rejected": -2.103590250015259,
|
|
"logps/chosen": -0.30928635597229004,
|
|
"logps/rejected": -0.30010607838630676,
|
|
"loss": 4445.4541015625,
|
|
"loss/core": 4445.4501953125,
|
|
"loss/preference_sft": 0.30928635597229004,
|
|
"loss/reference_anchor": 0.1733807623386383,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.1855305433273315,
|
|
"rewards/margins": -0.060359179973602295,
|
|
"rewards/rejected": 1.2458897829055786,
|
|
"step": 315
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3176988959312439,
|
|
"drift/rejected_abs": 0.17362681031227112,
|
|
"epoch": 0.3057446554401051,
|
|
"grad_norm": 12032.0,
|
|
"learning_rate": 2.038532863421914e-07,
|
|
"logits/chosen": -1.9140937328338623,
|
|
"logits/rejected": -1.8793401718139648,
|
|
"logps/chosen": -0.315841943025589,
|
|
"logps/rejected": -0.31801360845565796,
|
|
"loss": 4425.9814453125,
|
|
"loss/core": 4425.97021484375,
|
|
"loss/preference_sft": 0.315841943025589,
|
|
"loss/reference_anchor": 0.5636979937553406,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.17467999458313,
|
|
"rewards/margins": 1.440582513809204,
|
|
"rewards/rejected": 1.7340972423553467,
|
|
"step": 320
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1431046724319458,
|
|
"drift/rejected_abs": 0.0807650089263916,
|
|
"epoch": 0.31052191568135673,
|
|
"grad_norm": 510.0,
|
|
"learning_rate": 2.0195768441570726e-07,
|
|
"logits/chosen": -2.013561487197876,
|
|
"logits/rejected": -2.0425143241882324,
|
|
"logps/chosen": -0.2884654402732849,
|
|
"logps/rejected": -0.29259175062179565,
|
|
"loss": 4436.1591796875,
|
|
"loss/core": 4436.1572265625,
|
|
"loss/preference_sft": 0.2884654402732849,
|
|
"loss/reference_anchor": 0.061391688883304596,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.430469274520874,
|
|
"rewards/margins": 0.6241756677627563,
|
|
"rewards/rejected": 0.8062933683395386,
|
|
"step": 325
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.13373948633670807,
|
|
"drift/rejected_abs": 0.122031569480896,
|
|
"epoch": 0.31529917592260837,
|
|
"grad_norm": 11648.0,
|
|
"learning_rate": 2.0003314185145307e-07,
|
|
"logits/chosen": -2.183828830718994,
|
|
"logits/rejected": -2.1095242500305176,
|
|
"logps/chosen": -0.3171173930168152,
|
|
"logps/rejected": -0.3337666392326355,
|
|
"loss": 4443.197265625,
|
|
"loss/core": 4443.1923828125,
|
|
"loss/preference_sft": 0.3171173930168152,
|
|
"loss/reference_anchor": 0.21503393352031708,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.336255431175232,
|
|
"rewards/margins": 0.11729172617197037,
|
|
"rewards/rejected": 1.2189635038375854,
|
|
"step": 330
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.10823869705200195,
|
|
"drift/rejected_abs": 0.056395940482616425,
|
|
"epoch": 0.32007643616386,
|
|
"grad_norm": 812.0,
|
|
"learning_rate": 1.9808038239117913e-07,
|
|
"logits/chosen": -2.2766873836517334,
|
|
"logits/rejected": -2.3121063709259033,
|
|
"logps/chosen": -0.30512499809265137,
|
|
"logps/rejected": -0.306996613740921,
|
|
"loss": 4437.56005859375,
|
|
"loss/core": 4437.5595703125,
|
|
"loss/preference_sft": 0.30512499809265137,
|
|
"loss/reference_anchor": 0.025697598233819008,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.0811882019042969,
|
|
"rewards/margins": 0.5172288417816162,
|
|
"rewards/rejected": 0.5639593601226807,
|
|
"step": 335
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22527499496936798,
|
|
"drift/rejected_abs": 0.09023512899875641,
|
|
"epoch": 0.3248536964051117,
|
|
"grad_norm": 764.0,
|
|
"learning_rate": 1.9610014038785646e-07,
|
|
"logits/chosen": -2.272291660308838,
|
|
"logits/rejected": -2.280327320098877,
|
|
"logps/chosen": -0.27914947271347046,
|
|
"logps/rejected": -0.28271421790122986,
|
|
"loss": 4426.8134765625,
|
|
"loss/core": 4426.8056640625,
|
|
"loss/preference_sft": 0.27914947271347046,
|
|
"loss/reference_anchor": 0.3431945741176605,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.2520639896392822,
|
|
"rewards/margins": 1.3524361848831177,
|
|
"rewards/rejected": 0.8996278047561646,
|
|
"step": 340
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2175404578447342,
|
|
"drift/rejected_abs": 0.1755029261112213,
|
|
"epoch": 0.3296309566463633,
|
|
"grad_norm": 2672.0,
|
|
"learning_rate": 1.9409316052951657e-07,
|
|
"logits/chosen": -2.121755838394165,
|
|
"logits/rejected": -2.120198965072632,
|
|
"logps/chosen": -0.26820969581604004,
|
|
"logps/rejected": -0.2698619067668915,
|
|
"loss": 4439.0205078125,
|
|
"loss/core": 4439.01171875,
|
|
"loss/preference_sft": 0.26820969581604004,
|
|
"loss/reference_anchor": 0.3845630884170532,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.1744120121002197,
|
|
"rewards/margins": 0.42798107862472534,
|
|
"rewards/rejected": 1.7464309930801392,
|
|
"step": 345
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23404574394226074,
|
|
"drift/rejected_abs": 0.14923684298992157,
|
|
"epoch": 0.33440821688761496,
|
|
"grad_norm": 992.0,
|
|
"learning_rate": 1.920601975592047e-07,
|
|
"logits/chosen": -2.22644305229187,
|
|
"logits/rejected": -2.2413718700408936,
|
|
"logps/chosen": -0.29749247431755066,
|
|
"logps/rejected": -0.3060537874698639,
|
|
"loss": 4432.6982421875,
|
|
"loss/core": 4432.6904296875,
|
|
"loss/preference_sft": 0.29749247431755066,
|
|
"loss/reference_anchor": 0.35366910696029663,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.3393826484680176,
|
|
"rewards/margins": 0.9169244766235352,
|
|
"rewards/rejected": 1.4224579334259033,
|
|
"step": 350
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3508952260017395,
|
|
"drift/rejected_abs": 0.18243159353733063,
|
|
"epoch": 0.3391854771288666,
|
|
"grad_norm": 1576.0,
|
|
"learning_rate": 1.900020159911519e-07,
|
|
"logits/chosen": -2.1946253776550293,
|
|
"logits/rejected": -2.2092864513397217,
|
|
"logps/chosen": -0.285781592130661,
|
|
"logps/rejected": -0.26965421438217163,
|
|
"loss": 4422.45849609375,
|
|
"loss/core": 4422.4482421875,
|
|
"loss/preference_sft": 0.285781592130661,
|
|
"loss/reference_anchor": 0.4602643549442291,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.507547378540039,
|
|
"rewards/margins": 1.6848455667495728,
|
|
"rewards/rejected": 1.8227014541625977,
|
|
"step": 355
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3240725100040436,
|
|
"drift/rejected_abs": 0.14371727406978607,
|
|
"epoch": 0.34396273737011823,
|
|
"grad_norm": 4448.0,
|
|
"learning_rate": 1.879193898232725e-07,
|
|
"logits/chosen": -1.9690821170806885,
|
|
"logits/rejected": -2.004427433013916,
|
|
"logps/chosen": -0.27598175406455994,
|
|
"logps/rejected": -0.2723207473754883,
|
|
"loss": 4420.2294921875,
|
|
"loss/core": 4420.2216796875,
|
|
"loss/preference_sft": 0.27598175406455994,
|
|
"loss/reference_anchor": 0.38521429896354675,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.24072527885437,
|
|
"rewards/margins": 1.8547264337539673,
|
|
"rewards/rejected": 1.3859989643096924,
|
|
"step": 360
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.250040739774704,
|
|
"drift/rejected_abs": 0.10423221439123154,
|
|
"epoch": 0.34873999761136987,
|
|
"grad_norm": 2208.0,
|
|
"learning_rate": 1.8581310224609496e-07,
|
|
"logits/chosen": -2.2583580017089844,
|
|
"logits/rejected": -2.328725576400757,
|
|
"logps/chosen": -0.3033362925052643,
|
|
"logps/rejected": -0.30249541997909546,
|
|
"loss": 4425.35693359375,
|
|
"loss/core": 4425.35009765625,
|
|
"loss/preference_sft": 0.3033362925052643,
|
|
"loss/reference_anchor": 0.2949478030204773,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.500340461730957,
|
|
"rewards/margins": 1.4580180644989014,
|
|
"rewards/rejected": 1.0423221588134766,
|
|
"step": 365
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2433367520570755,
|
|
"drift/rejected_abs": 0.16708707809448242,
|
|
"epoch": 0.3535172578526215,
|
|
"grad_norm": 1004.0,
|
|
"learning_rate": 1.8368394534823635e-07,
|
|
"logits/chosen": -2.1375997066497803,
|
|
"logits/rejected": -2.0652060508728027,
|
|
"logps/chosen": -0.2894711196422577,
|
|
"logps/rejected": -0.3007884919643402,
|
|
"loss": 4434.6865234375,
|
|
"loss/core": 4434.677734375,
|
|
"loss/preference_sft": 0.2894711196422577,
|
|
"loss/reference_anchor": 0.397393137216568,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.4329514503479004,
|
|
"rewards/margins": 0.7622579336166382,
|
|
"rewards/rejected": 1.6706933975219727,
|
|
"step": 370
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3843259811401367,
|
|
"drift/rejected_abs": 0.14180049300193787,
|
|
"epoch": 0.35829451809387314,
|
|
"grad_norm": 844.0,
|
|
"learning_rate": 1.8153271981852968e-07,
|
|
"logits/chosen": -2.014108419418335,
|
|
"logits/rejected": -1.938706398010254,
|
|
"logps/chosen": -0.28277072310447693,
|
|
"logps/rejected": -0.28435999155044556,
|
|
"loss": 4413.47509765625,
|
|
"loss/core": 4413.4560546875,
|
|
"loss/preference_sft": 0.28277072310447693,
|
|
"loss/reference_anchor": 0.9182009696960449,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 3.8399288654327393,
|
|
"rewards/margins": 2.425347089767456,
|
|
"rewards/rejected": 1.4145814180374146,
|
|
"step": 375
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.35180220007896423,
|
|
"drift/rejected_abs": 0.16935007274150848,
|
|
"epoch": 0.36307177833512483,
|
|
"grad_norm": 3824.0,
|
|
"learning_rate": 1.7936023464491812e-07,
|
|
"logits/chosen": -1.9925806522369385,
|
|
"logits/rejected": -2.077805995941162,
|
|
"logps/chosen": -0.2889358401298523,
|
|
"logps/rejected": -0.30222803354263306,
|
|
"loss": 4420.29736328125,
|
|
"loss/core": 4420.27685546875,
|
|
"loss/preference_sft": 0.2889358401298523,
|
|
"loss/reference_anchor": 0.9768502116203308,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.5166919231414795,
|
|
"rewards/margins": 1.8392245769500732,
|
|
"rewards/rejected": 1.6774673461914062,
|
|
"step": 380
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.311501145362854,
|
|
"drift/rejected_abs": 0.20911741256713867,
|
|
"epoch": 0.36784903857637646,
|
|
"grad_norm": 8704.0,
|
|
"learning_rate": 1.7716730681022723e-07,
|
|
"logits/chosen": -2.202134609222412,
|
|
"logits/rejected": -2.128066062927246,
|
|
"logps/chosen": -0.3021170496940613,
|
|
"logps/rejected": -0.3110412359237671,
|
|
"loss": 4431.4599609375,
|
|
"loss/core": 4431.4462890625,
|
|
"loss/preference_sft": 0.3021170496940613,
|
|
"loss/reference_anchor": 0.6745834350585938,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.115011692047119,
|
|
"rewards/margins": 1.0284936428070068,
|
|
"rewards/rejected": 2.086517810821533,
|
|
"step": 385
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2097025215625763,
|
|
"drift/rejected_abs": 0.08994968235492706,
|
|
"epoch": 0.3726262988176281,
|
|
"grad_norm": 708.0,
|
|
"learning_rate": 1.7495476098493152e-07,
|
|
"logits/chosen": -2.065886974334717,
|
|
"logits/rejected": -2.094799757003784,
|
|
"logps/chosen": -0.29858630895614624,
|
|
"logps/rejected": -0.29905611276626587,
|
|
"loss": 4428.68310546875,
|
|
"loss/core": 4428.6787109375,
|
|
"loss/preference_sft": 0.29858630895614624,
|
|
"loss/reference_anchor": 0.24309007823467255,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.094350814819336,
|
|
"rewards/margins": 1.1971451044082642,
|
|
"rewards/rejected": 0.897205650806427,
|
|
"step": 390
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20265540480613708,
|
|
"drift/rejected_abs": 0.09964494407176971,
|
|
"epoch": 0.37740355905887973,
|
|
"grad_norm": 824.0,
|
|
"learning_rate": 1.7272342921702937e-07,
|
|
"logits/chosen": -2.179865598678589,
|
|
"logits/rejected": -2.2556076049804688,
|
|
"logps/chosen": -0.2747688293457031,
|
|
"logps/rejected": -0.2691434919834137,
|
|
"loss": 4430.7685546875,
|
|
"loss/core": 4430.76611328125,
|
|
"loss/preference_sft": 0.2747688293457031,
|
|
"loss/reference_anchor": 0.09649205207824707,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.0265541076660156,
|
|
"rewards/margins": 1.0302097797393799,
|
|
"rewards/rejected": 0.9963444471359253,
|
|
"step": 395
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20592239499092102,
|
|
"drift/rejected_abs": 0.16747477650642395,
|
|
"epoch": 0.38218081930013137,
|
|
"grad_norm": 1896.0,
|
|
"learning_rate": 1.7047415061914393e-07,
|
|
"logits/chosen": -2.1442654132843018,
|
|
"logits/rejected": -2.081714153289795,
|
|
"logps/chosen": -0.2744106352329254,
|
|
"logps/rejected": -0.2705535292625427,
|
|
"loss": 4439.44970703125,
|
|
"loss/core": 4439.44482421875,
|
|
"loss/preference_sft": 0.2744106352329254,
|
|
"loss/reference_anchor": 0.20020189881324768,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.0569989681243896,
|
|
"rewards/margins": 0.39103108644485474,
|
|
"rewards/rejected": 1.6659677028656006,
|
|
"step": 400
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25720465183258057,
|
|
"drift/rejected_abs": 0.12404946982860565,
|
|
"epoch": 0.386958079541383,
|
|
"grad_norm": 7136.0,
|
|
"learning_rate": 1.6820777105296707e-07,
|
|
"logits/chosen": -2.3219733238220215,
|
|
"logits/rejected": -2.3814594745635986,
|
|
"logps/chosen": -0.2641887664794922,
|
|
"logps/rejected": -0.26516467332839966,
|
|
"loss": 4426.92333984375,
|
|
"loss/core": 4426.91162109375,
|
|
"loss/preference_sft": 0.2641887664794922,
|
|
"loss/reference_anchor": 0.5534130334854126,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.5716207027435303,
|
|
"rewards/margins": 1.331629753112793,
|
|
"rewards/rejected": 1.2399909496307373,
|
|
"step": 405
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.12248623371124268,
|
|
"drift/rejected_abs": 0.06730367243289948,
|
|
"epoch": 0.39173533978263464,
|
|
"grad_norm": 1352.0,
|
|
"learning_rate": 1.6592514281116553e-07,
|
|
"logits/chosen": -2.0958163738250732,
|
|
"logits/rejected": -2.1545522212982178,
|
|
"logps/chosen": -0.30971580743789673,
|
|
"logps/rejected": -0.3031691908836365,
|
|
"loss": 4437.1142578125,
|
|
"loss/core": 4437.1123046875,
|
|
"loss/preference_sft": 0.30971580743789673,
|
|
"loss/reference_anchor": 0.046301212161779404,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 1.2217724323272705,
|
|
"rewards/margins": 0.5523636937141418,
|
|
"rewards/rejected": 0.6694086790084839,
|
|
"step": 410
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2457958459854126,
|
|
"drift/rejected_abs": 0.1320219784975052,
|
|
"epoch": 0.3965126000238863,
|
|
"grad_norm": 700.0,
|
|
"learning_rate": 1.636271242968684e-07,
|
|
"logits/chosen": -2.319655418395996,
|
|
"logits/rejected": -2.3182573318481445,
|
|
"logps/chosen": -0.2952980399131775,
|
|
"logps/rejected": -0.2870740294456482,
|
|
"loss": 4429.59912109375,
|
|
"loss/core": 4429.59375,
|
|
"loss/preference_sft": 0.2952980399131775,
|
|
"loss/reference_anchor": 0.270760178565979,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.457775831222534,
|
|
"rewards/margins": 1.1382719278335571,
|
|
"rewards/rejected": 1.3195040225982666,
|
|
"step": 415
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3312036991119385,
|
|
"drift/rejected_abs": 0.10126861184835434,
|
|
"epoch": 0.40128986026513797,
|
|
"grad_norm": 7872.0,
|
|
"learning_rate": 1.6131457970085684e-07,
|
|
"logits/chosen": -2.119199752807617,
|
|
"logits/rejected": -2.087686061859131,
|
|
"logps/chosen": -0.32637646794319153,
|
|
"logps/rejected": -0.3032102882862091,
|
|
"loss": 4414.77880859375,
|
|
"loss/core": 4414.76318359375,
|
|
"loss/preference_sft": 0.32637646794319153,
|
|
"loss/reference_anchor": 0.7741028070449829,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.3084073066711426,
|
|
"rewards/margins": 2.3002755641937256,
|
|
"rewards/rejected": 1.0081310272216797,
|
|
"step": 420
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3094877302646637,
|
|
"drift/rejected_abs": 0.10990305244922638,
|
|
"epoch": 0.4060671205063896,
|
|
"grad_norm": 1688.0,
|
|
"learning_rate": 1.5898837867657727e-07,
|
|
"logits/chosen": -2.1425862312316895,
|
|
"logits/rejected": -2.222062587738037,
|
|
"logps/chosen": -0.31150728464126587,
|
|
"logps/rejected": -0.31376224756240845,
|
|
"loss": 4418.671875,
|
|
"loss/core": 4418.65966796875,
|
|
"loss/preference_sft": 0.31150728464126587,
|
|
"loss/reference_anchor": 0.5772914886474609,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.094017744064331,
|
|
"rewards/margins": 1.9972339868545532,
|
|
"rewards/rejected": 1.0967837572097778,
|
|
"step": 425
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14985668659210205,
|
|
"drift/rejected_abs": 0.0947737768292427,
|
|
"epoch": 0.41084438074764124,
|
|
"grad_norm": 3520.0,
|
|
"learning_rate": 1.5664939601310023e-07,
|
|
"logits/chosen": -2.170436143875122,
|
|
"logits/rejected": -2.2830162048339844,
|
|
"logps/chosen": -0.2657105326652527,
|
|
"logps/rejected": -0.2679697275161743,
|
|
"loss": 4437.12255859375,
|
|
"loss/core": 4437.1201171875,
|
|
"loss/preference_sft": 0.2657105326652527,
|
|
"loss/reference_anchor": 0.0813552588224411,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.4978549480438232,
|
|
"rewards/margins": 0.5538910627365112,
|
|
"rewards/rejected": 0.943963885307312,
|
|
"step": 430
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.39538705348968506,
|
|
"drift/rejected_abs": 0.13529789447784424,
|
|
"epoch": 0.41562164098889287,
|
|
"grad_norm": 11008.0,
|
|
"learning_rate": 1.5429851130614807e-07,
|
|
"logits/chosen": -1.9859920740127563,
|
|
"logits/rejected": -1.9977947473526,
|
|
"logps/chosen": -0.2817636728286743,
|
|
"logps/rejected": -0.2686322033405304,
|
|
"loss": 4410.4755859375,
|
|
"loss/core": 4410.4619140625,
|
|
"loss/preference_sft": 0.2817636728286743,
|
|
"loss/reference_anchor": 0.6513532400131226,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.953408718109131,
|
|
"rewards/margins": 2.6012117862701416,
|
|
"rewards/rejected": 1.3521969318389893,
|
|
"step": 435
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21620866656303406,
|
|
"drift/rejected_abs": 0.12473054230213165,
|
|
"epoch": 0.4203989012301445,
|
|
"grad_norm": 1888.0,
|
|
"learning_rate": 1.51936608627315e-07,
|
|
"logits/chosen": -2.182278633117676,
|
|
"logits/rejected": -2.1586408615112305,
|
|
"logps/chosen": -0.2953268885612488,
|
|
"logps/rejected": -0.2857879102230072,
|
|
"loss": 4432.2978515625,
|
|
"loss/core": 4432.294921875,
|
|
"loss/preference_sft": 0.2953268885612488,
|
|
"loss/reference_anchor": 0.13938388228416443,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.161210060119629,
|
|
"rewards/margins": 0.9144028425216675,
|
|
"rewards/rejected": 1.2468074560165405,
|
|
"step": 440
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.30145135521888733,
|
|
"drift/rejected_abs": 0.1143740639090538,
|
|
"epoch": 0.42517616147139614,
|
|
"grad_norm": 3456.0,
|
|
"learning_rate": 1.4956457619160375e-07,
|
|
"logits/chosen": -2.11724853515625,
|
|
"logits/rejected": -2.1948139667510986,
|
|
"logps/chosen": -0.28283238410949707,
|
|
"logps/rejected": -0.27826812863349915,
|
|
"loss": 4419.79443359375,
|
|
"loss/core": 4419.7880859375,
|
|
"loss/preference_sft": 0.28283238410949707,
|
|
"loss/reference_anchor": 0.2919161915779114,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.012824296951294,
|
|
"rewards/margins": 1.8758718967437744,
|
|
"rewards/rejected": 1.1369524002075195,
|
|
"step": 445
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2928563058376312,
|
|
"drift/rejected_abs": 0.1479603499174118,
|
|
"epoch": 0.4299534217126478,
|
|
"grad_norm": 15680.0,
|
|
"learning_rate": 1.471833060234044e-07,
|
|
"logits/chosen": -2.1485331058502197,
|
|
"logits/rejected": -2.2221622467041016,
|
|
"logps/chosen": -0.2605140507221222,
|
|
"logps/rejected": -0.27261584997177124,
|
|
"loss": 4425.9091796875,
|
|
"loss/core": 4425.8974609375,
|
|
"loss/preference_sft": 0.2605140507221222,
|
|
"loss/reference_anchor": 0.5446182489395142,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.9277219772338867,
|
|
"rewards/margins": 1.450683355331421,
|
|
"rewards/rejected": 1.4770386219024658,
|
|
"step": 450
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17106688022613525,
|
|
"drift/rejected_abs": 0.08423759788274765,
|
|
"epoch": 0.4347306819538994,
|
|
"grad_norm": 552.0,
|
|
"learning_rate": 1.4479369362104037e-07,
|
|
"logits/chosen": -1.9175952672958374,
|
|
"logits/rejected": -1.9540889263153076,
|
|
"logps/chosen": -0.3066188097000122,
|
|
"logps/rejected": -0.30025577545166016,
|
|
"loss": 4432.8935546875,
|
|
"loss/core": 4432.89013671875,
|
|
"loss/preference_sft": 0.3066188097000122,
|
|
"loss/reference_anchor": 0.09201753884553909,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.7104876041412354,
|
|
"rewards/margins": 0.8717054128646851,
|
|
"rewards/rejected": 0.838782012462616,
|
|
"step": 455
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22610139846801758,
|
|
"drift/rejected_abs": 0.1578260362148285,
|
|
"epoch": 0.4395079421951511,
|
|
"grad_norm": 1088.0,
|
|
"learning_rate": 1.4239663762000817e-07,
|
|
"logits/chosen": -2.0635833740234375,
|
|
"logits/rejected": -2.215299606323242,
|
|
"logps/chosen": -0.2846943736076355,
|
|
"logps/rejected": -0.2744455933570862,
|
|
"loss": 4435.564453125,
|
|
"loss/core": 4435.56005859375,
|
|
"loss/preference_sft": 0.2846943736076355,
|
|
"loss/reference_anchor": 0.2083575427532196,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.26090931892395,
|
|
"rewards/margins": 0.6843754053115845,
|
|
"rewards/rejected": 1.5765340328216553,
|
|
"step": 460
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2833055555820465,
|
|
"drift/rejected_abs": 0.21140813827514648,
|
|
"epoch": 0.44428520243640274,
|
|
"grad_norm": 980.0,
|
|
"learning_rate": 1.3999303945503747e-07,
|
|
"logits/chosen": -2.1378350257873535,
|
|
"logits/rejected": -2.0690975189208984,
|
|
"logps/chosen": -0.27313750982284546,
|
|
"logps/rejected": -0.2805202007293701,
|
|
"loss": 4435.7646484375,
|
|
"loss/core": 4435.7529296875,
|
|
"loss/preference_sft": 0.27313750982284546,
|
|
"loss/reference_anchor": 0.5566142797470093,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.833055257797241,
|
|
"rewards/margins": 0.7193877100944519,
|
|
"rewards/rejected": 2.1136677265167236,
|
|
"step": 465
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22237162292003632,
|
|
"drift/rejected_abs": 0.1690370738506317,
|
|
"epoch": 0.4490624626776544,
|
|
"grad_norm": 2528.0,
|
|
"learning_rate": 1.3758380302109808e-07,
|
|
"logits/chosen": -2.125699520111084,
|
|
"logits/rejected": -2.099730968475342,
|
|
"logps/chosen": -0.2701879143714905,
|
|
"logps/rejected": -0.28192803263664246,
|
|
"loss": 4437.58447265625,
|
|
"loss/core": 4437.5791015625,
|
|
"loss/preference_sft": 0.2701879143714905,
|
|
"loss/reference_anchor": 0.24922780692577362,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.223245859146118,
|
|
"rewards/margins": 0.5391066670417786,
|
|
"rewards/rejected": 1.6841392517089844,
|
|
"step": 470
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23621515929698944,
|
|
"drift/rejected_abs": 0.16340488195419312,
|
|
"epoch": 0.453839722918906,
|
|
"grad_norm": 1424.0,
|
|
"learning_rate": 1.351698343334823e-07,
|
|
"logits/chosen": -2.234205961227417,
|
|
"logits/rejected": -2.2512259483337402,
|
|
"logps/chosen": -0.2768619954586029,
|
|
"logps/rejected": -0.2696452736854553,
|
|
"loss": 4434.79052734375,
|
|
"loss/core": 4434.787109375,
|
|
"loss/preference_sft": 0.2768619954586029,
|
|
"loss/reference_anchor": 0.17205794155597687,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.3621127605438232,
|
|
"rewards/margins": 0.7280641198158264,
|
|
"rewards/rejected": 1.6340488195419312,
|
|
"step": 475
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.530329704284668,
|
|
"drift/rejected_abs": 0.12030331045389175,
|
|
"epoch": 0.45861698316015764,
|
|
"grad_norm": 2736.0,
|
|
"learning_rate": 1.3275204118708942e-07,
|
|
"logits/chosen": -2.001190662384033,
|
|
"logits/rejected": -2.016195774078369,
|
|
"logps/chosen": -0.2734314799308777,
|
|
"logps/rejected": -0.27441081404685974,
|
|
"loss": 4392.0009765625,
|
|
"loss/core": 4391.97265625,
|
|
"loss/preference_sft": 0.2734314799308777,
|
|
"loss/reference_anchor": 1.3795697689056396,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 5.302812099456787,
|
|
"rewards/margins": 4.102025032043457,
|
|
"rewards/rejected": 1.2007869482040405,
|
|
"step": 480
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2102138102054596,
|
|
"drift/rejected_abs": 0.11363495886325836,
|
|
"epoch": 0.4633942434014093,
|
|
"grad_norm": 2040.0,
|
|
"learning_rate": 1.3033133281504132e-07,
|
|
"logits/chosen": -2.2091434001922607,
|
|
"logits/rejected": -2.1927781105041504,
|
|
"logps/chosen": -0.2823958992958069,
|
|
"logps/rejected": -0.3023380637168884,
|
|
"loss": 4430.564453125,
|
|
"loss/core": 4430.5595703125,
|
|
"loss/preference_sft": 0.2823958992958069,
|
|
"loss/reference_anchor": 0.20998165011405945,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.102138042449951,
|
|
"rewards/margins": 1.0682824850082397,
|
|
"rewards/rejected": 1.033855676651001,
|
|
"step": 485
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16725513339042664,
|
|
"drift/rejected_abs": 0.143391415476799,
|
|
"epoch": 0.4681715036426609,
|
|
"grad_norm": 6176.0,
|
|
"learning_rate": 1.2790861954675702e-07,
|
|
"logits/chosen": -2.198439598083496,
|
|
"logits/rejected": -2.145232677459717,
|
|
"logps/chosen": -0.2879284620285034,
|
|
"logps/rejected": -0.2975713610649109,
|
|
"loss": 4441.4736328125,
|
|
"loss/core": 4441.4697265625,
|
|
"loss/preference_sft": 0.2879284620285034,
|
|
"loss/reference_anchor": 0.16039147973060608,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.6691477298736572,
|
|
"rewards/margins": 0.2376086413860321,
|
|
"rewards/rejected": 1.4315390586853027,
|
|
"step": 490
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3689550459384918,
|
|
"drift/rejected_abs": 0.17689982056617737,
|
|
"epoch": 0.47294876388391255,
|
|
"grad_norm": 12736.0,
|
|
"learning_rate": 1.2548481246561504e-07,
|
|
"logits/chosen": -2.104423761367798,
|
|
"logits/rejected": -2.1106228828430176,
|
|
"logps/chosen": -0.31167513132095337,
|
|
"logps/rejected": -0.30056482553482056,
|
|
"loss": 4420.04833984375,
|
|
"loss/core": 4420.02685546875,
|
|
"loss/preference_sft": 0.31167513132095337,
|
|
"loss/reference_anchor": 1.0426156520843506,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 3.6870017051696777,
|
|
"rewards/margins": 1.9218406677246094,
|
|
"rewards/rejected": 1.7651607990264893,
|
|
"step": 495
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3468535840511322,
|
|
"drift/rejected_abs": 0.12408767640590668,
|
|
"epoch": 0.47772602412516424,
|
|
"grad_norm": 11392.0,
|
|
"learning_rate": 1.230608230663321e-07,
|
|
"logits/chosen": -2.1764144897460938,
|
|
"logits/rejected": -2.1404600143432617,
|
|
"logps/chosen": -0.28169992566108704,
|
|
"logps/rejected": -0.26774054765701294,
|
|
"loss": 4415.38818359375,
|
|
"loss/core": 4415.37548828125,
|
|
"loss/preference_sft": 0.28169992566108704,
|
|
"loss/reference_anchor": 0.5941214561462402,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.4685356616973877,
|
|
"rewards/margins": 2.2292885780334473,
|
|
"rewards/rejected": 1.2392469644546509,
|
|
"step": 500
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.32149267196655273,
|
|
"drift/rejected_abs": 0.13741762936115265,
|
|
"epoch": 0.4825032843664159,
|
|
"grad_norm": 1152.0,
|
|
"learning_rate": 1.206375629121874e-07,
|
|
"logits/chosen": -1.7544605731964111,
|
|
"logits/rejected": -1.77561354637146,
|
|
"logps/chosen": -0.30578163266181946,
|
|
"logps/rejected": -0.3066912889480591,
|
|
"loss": 4420.0146484375,
|
|
"loss/core": 4420.0068359375,
|
|
"loss/preference_sft": 0.30578163266181946,
|
|
"loss/reference_anchor": 0.36515378952026367,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.2140979766845703,
|
|
"rewards/margins": 1.852840781211853,
|
|
"rewards/rejected": 1.3612569570541382,
|
|
"step": 505
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3091573417186737,
|
|
"drift/rejected_abs": 0.1424328237771988,
|
|
"epoch": 0.4872805446076675,
|
|
"grad_norm": 3552.0,
|
|
"learning_rate": 1.1821594329222079e-07,
|
|
"logits/chosen": -2.080624580383301,
|
|
"logits/rejected": -2.1759705543518066,
|
|
"logps/chosen": -0.2625710964202881,
|
|
"logps/rejected": -0.27721071243286133,
|
|
"loss": 4422.59716796875,
|
|
"loss/core": 4422.5908203125,
|
|
"loss/preference_sft": 0.2625710964202881,
|
|
"loss/reference_anchor": 0.3314869999885559,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.09027361869812,
|
|
"rewards/margins": 1.6671348810195923,
|
|
"rewards/rejected": 1.423138976097107,
|
|
"step": 510
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22843725979328156,
|
|
"drift/rejected_abs": 0.13884204626083374,
|
|
"epoch": 0.49205780484891914,
|
|
"grad_norm": 912.0,
|
|
"learning_rate": 1.157968748785344e-07,
|
|
"logits/chosen": -1.9473901987075806,
|
|
"logits/rejected": -2.0520355701446533,
|
|
"logps/chosen": -0.2986038327217102,
|
|
"logps/rejected": -0.3280816972255707,
|
|
"loss": 4432.8408203125,
|
|
"loss/core": 4432.8349609375,
|
|
"loss/preference_sft": 0.2986038327217102,
|
|
"loss/reference_anchor": 0.27390003204345703,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.283982753753662,
|
|
"rewards/margins": 0.8998821973800659,
|
|
"rewards/rejected": 1.3841006755828857,
|
|
"step": 515
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25142940878868103,
|
|
"drift/rejected_abs": 0.0938950628042221,
|
|
"epoch": 0.4968350650901708,
|
|
"grad_norm": 474.0,
|
|
"learning_rate": 1.1338126738382597e-07,
|
|
"logits/chosen": -2.0015084743499756,
|
|
"logits/rejected": -2.200745105743408,
|
|
"logps/chosen": -0.3057096600532532,
|
|
"logps/rejected": -0.29396018385887146,
|
|
"loss": 4423.95947265625,
|
|
"loss/core": 4423.9501953125,
|
|
"loss/preference_sft": 0.3057096600532532,
|
|
"loss/reference_anchor": 0.41009026765823364,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.513545513153076,
|
|
"rewards/margins": 1.5773802995681763,
|
|
"rewards/rejected": 0.9361652135848999,
|
|
"step": 520
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3425474762916565,
|
|
"drift/rejected_abs": 0.14658579230308533,
|
|
"epoch": 0.5016123253314224,
|
|
"grad_norm": 328.0,
|
|
"learning_rate": 1.1097002921928316e-07,
|
|
"logits/chosen": -2.023900270462036,
|
|
"logits/rejected": -2.0547852516174316,
|
|
"logps/chosen": -0.2761421203613281,
|
|
"logps/rejected": -0.2680854797363281,
|
|
"loss": 4418.85009765625,
|
|
"loss/core": 4418.8408203125,
|
|
"loss/preference_sft": 0.2761421203613281,
|
|
"loss/reference_anchor": 0.4391368329524994,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.4254088401794434,
|
|
"rewards/margins": 1.96877920627594,
|
|
"rewards/rejected": 1.456629753112793,
|
|
"step": 525
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1811850368976593,
|
|
"drift/rejected_abs": 0.08488638699054718,
|
|
"epoch": 0.5063895855726741,
|
|
"grad_norm": 684.0,
|
|
"learning_rate": 1.0856406715296717e-07,
|
|
"logits/chosen": -2.0869510173797607,
|
|
"logits/rejected": -2.206756830215454,
|
|
"logps/chosen": -0.29684141278266907,
|
|
"logps/rejected": -0.29523271322250366,
|
|
"loss": 4431.6162109375,
|
|
"loss/core": 4431.61376953125,
|
|
"loss/preference_sft": 0.29684141278266907,
|
|
"loss/reference_anchor": 0.08795829117298126,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.8088932037353516,
|
|
"rewards/margins": 0.9681621789932251,
|
|
"rewards/rejected": 0.8407310247421265,
|
|
"step": 530
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3354306221008301,
|
|
"drift/rejected_abs": 0.177082821726799,
|
|
"epoch": 0.5111668458139257,
|
|
"grad_norm": 1024.0,
|
|
"learning_rate": 1.061642859688146e-07,
|
|
"logits/chosen": -2.060147762298584,
|
|
"logits/rejected": -2.065338373184204,
|
|
"logps/chosen": -0.2848697900772095,
|
|
"logps/rejected": -0.2815316915512085,
|
|
"loss": 4423.5537109375,
|
|
"loss/core": 4423.54052734375,
|
|
"loss/preference_sft": 0.2848697900772095,
|
|
"loss/reference_anchor": 0.5895854234695435,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.3528850078582764,
|
|
"rewards/margins": 1.5826117992401123,
|
|
"rewards/rejected": 1.770272970199585,
|
|
"step": 535
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19800987839698792,
|
|
"drift/rejected_abs": 0.09570334106683731,
|
|
"epoch": 0.5159441060551774,
|
|
"grad_norm": 2848.0,
|
|
"learning_rate": 1.0377158812638491e-07,
|
|
"logits/chosen": -2.050477981567383,
|
|
"logits/rejected": -2.1051759719848633,
|
|
"logps/chosen": -0.29067665338516235,
|
|
"logps/rejected": -0.28211501240730286,
|
|
"loss": 4430.95361328125,
|
|
"loss/core": 4430.9501953125,
|
|
"loss/preference_sft": 0.29067665338516235,
|
|
"loss/reference_anchor": 0.14536850154399872,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.9791667461395264,
|
|
"rewards/margins": 1.0227887630462646,
|
|
"rewards/rejected": 0.9563776254653931,
|
|
"step": 540
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.37276729941368103,
|
|
"drift/rejected_abs": 0.08400270342826843,
|
|
"epoch": 0.520721366296429,
|
|
"grad_norm": 3744.0,
|
|
"learning_rate": 1.0138687342148249e-07,
|
|
"logits/chosen": -2.044623374938965,
|
|
"logits/rejected": -2.1050045490264893,
|
|
"logps/chosen": -0.28390389680862427,
|
|
"logps/rejected": -0.2720825672149658,
|
|
"loss": 4406.427734375,
|
|
"loss/core": 4406.41748046875,
|
|
"loss/preference_sft": 0.28390389680862427,
|
|
"loss/reference_anchor": 0.45570674538612366,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.7272789478302,
|
|
"rewards/margins": 2.9046859741210938,
|
|
"rewards/rejected": 0.8225927352905273,
|
|
"step": 545
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2248731106519699,
|
|
"drift/rejected_abs": 0.11237964779138565,
|
|
"epoch": 0.5254986265376806,
|
|
"grad_norm": 1240.0,
|
|
"learning_rate": 9.90110386477801e-08,
|
|
"logits/chosen": -2.00352144241333,
|
|
"logits/rejected": -2.0287065505981445,
|
|
"logps/chosen": -0.28295984864234924,
|
|
"logps/rejected": -0.27802377939224243,
|
|
"loss": 4429.490234375,
|
|
"loss/core": 4429.48779296875,
|
|
"loss/preference_sft": 0.28295984864234924,
|
|
"loss/reference_anchor": 0.09104995429515839,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.246397018432617,
|
|
"rewards/margins": 1.1257827281951904,
|
|
"rewards/rejected": 1.1206141710281372,
|
|
"step": 550
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3234367370605469,
|
|
"drift/rejected_abs": 0.15718965232372284,
|
|
"epoch": 0.5302758867789323,
|
|
"grad_norm": 1104.0,
|
|
"learning_rate": 9.664497725957164e-08,
|
|
"logits/chosen": -2.059795618057251,
|
|
"logits/rejected": -2.1331515312194824,
|
|
"logps/chosen": -0.31671643257141113,
|
|
"logps/rejected": -0.2889899015426636,
|
|
"loss": 4422.66943359375,
|
|
"loss/core": 4422.658203125,
|
|
"loss/preference_sft": 0.31671643257141113,
|
|
"loss/reference_anchor": 0.5569087266921997,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.2323079109191895,
|
|
"rewards/margins": 1.6611411571502686,
|
|
"rewards/rejected": 1.5711662769317627,
|
|
"step": 555
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2606988251209259,
|
|
"drift/rejected_abs": 0.13210263848304749,
|
|
"epoch": 0.5350531470201839,
|
|
"grad_norm": 1032.0,
|
|
"learning_rate": 9.428957903578045e-08,
|
|
"logits/chosen": -1.871110200881958,
|
|
"logits/rejected": -1.8772865533828735,
|
|
"logps/chosen": -0.2984418272972107,
|
|
"logps/rejected": -0.2938275635242462,
|
|
"loss": 4427.8583984375,
|
|
"loss/core": 4427.85009765625,
|
|
"loss/preference_sft": 0.2984418272972107,
|
|
"loss/reference_anchor": 0.41203561425209045,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.606642007827759,
|
|
"rewards/margins": 1.286530613899231,
|
|
"rewards/rejected": 1.3201110363006592,
|
|
"step": 560
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.5179003477096558,
|
|
"drift/rejected_abs": 0.19176138937473297,
|
|
"epoch": 0.5398304072614356,
|
|
"grad_norm": 2688.0,
|
|
"learning_rate": 9.194572974534976e-08,
|
|
"logits/chosen": -1.9553916454315186,
|
|
"logits/rejected": -2.062887191772461,
|
|
"logps/chosen": -0.2773086130619049,
|
|
"logps/rejected": -0.2870504558086395,
|
|
"loss": 4402.384765625,
|
|
"loss/core": 4402.35546875,
|
|
"loss/preference_sft": 0.2773086130619049,
|
|
"loss/reference_anchor": 1.4593416452407837,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 5.178015232086182,
|
|
"rewards/margins": 3.264045000076294,
|
|
"rewards/rejected": 1.9139697551727295,
|
|
"step": 565
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23666679859161377,
|
|
"drift/rejected_abs": 0.21036815643310547,
|
|
"epoch": 0.5446076675026872,
|
|
"grad_norm": 3984.0,
|
|
"learning_rate": 8.96143108141412e-08,
|
|
"logits/chosen": -2.098996639251709,
|
|
"logits/rejected": -2.055161952972412,
|
|
"logps/chosen": -0.31491559743881226,
|
|
"logps/rejected": -0.29813891649246216,
|
|
"loss": 4441.4765625,
|
|
"loss/core": 4441.46630859375,
|
|
"loss/preference_sft": 0.31491559743881226,
|
|
"loss/reference_anchor": 0.47641339898109436,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.3656527996063232,
|
|
"rewards/margins": 0.26370275020599365,
|
|
"rewards/rejected": 2.101950168609619,
|
|
"step": 570
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24016256630420685,
|
|
"drift/rejected_abs": 0.10366284847259521,
|
|
"epoch": 0.5493849277439389,
|
|
"grad_norm": 1968.0,
|
|
"learning_rate": 8.72961989934668e-08,
|
|
"logits/chosen": -2.352053165435791,
|
|
"logits/rejected": -2.3277459144592285,
|
|
"logps/chosen": -0.28055113554000854,
|
|
"logps/rejected": -0.2906273305416107,
|
|
"loss": 4426.044921875,
|
|
"loss/core": 4426.0390625,
|
|
"loss/preference_sft": 0.28055113554000854,
|
|
"loss/reference_anchor": 0.2594422698020935,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.399277925491333,
|
|
"rewards/margins": 1.4035309553146362,
|
|
"rewards/rejected": 0.9957469701766968,
|
|
"step": 575
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24877488613128662,
|
|
"drift/rejected_abs": 0.10169482231140137,
|
|
"epoch": 0.5541621879851905,
|
|
"grad_norm": 908.0,
|
|
"learning_rate": 8.499226603037854e-08,
|
|
"logits/chosen": -2.032081127166748,
|
|
"logits/rejected": -2.1548638343811035,
|
|
"logps/chosen": -0.30404120683670044,
|
|
"logps/rejected": -0.3054927885532379,
|
|
"loss": 4425.0419921875,
|
|
"loss/core": 4425.03759765625,
|
|
"loss/preference_sft": 0.30404120683670044,
|
|
"loss/reference_anchor": 0.2175559103488922,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.485950469970703,
|
|
"rewards/margins": 1.4695405960083008,
|
|
"rewards/rejected": 1.016409993171692,
|
|
"step": 580
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26507624983787537,
|
|
"drift/rejected_abs": 0.19033315777778625,
|
|
"epoch": 0.5589394482264421,
|
|
"grad_norm": 2144.0,
|
|
"learning_rate": 8.270337833983987e-08,
|
|
"logits/chosen": -2.246429681777954,
|
|
"logits/rejected": -2.2616257667541504,
|
|
"logps/chosen": -0.27734872698783875,
|
|
"logps/rejected": -0.2799379229545593,
|
|
"loss": 4435.09521484375,
|
|
"loss/core": 4435.0849609375,
|
|
"loss/preference_sft": 0.27734872698783875,
|
|
"loss/reference_anchor": 0.45706719160079956,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.6497602462768555,
|
|
"rewards/margins": 0.7482293248176575,
|
|
"rewards/rejected": 1.9015309810638428,
|
|
"step": 585
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27788597345352173,
|
|
"drift/rejected_abs": 0.17827074229717255,
|
|
"epoch": 0.5637167084676937,
|
|
"grad_norm": 3984.0,
|
|
"learning_rate": 8.04303966789025e-08,
|
|
"logits/chosen": -1.9625133275985718,
|
|
"logits/rejected": -1.9321056604385376,
|
|
"logps/chosen": -0.30193933844566345,
|
|
"logps/rejected": -0.29800906777381897,
|
|
"loss": 4431.5615234375,
|
|
"loss/core": 4431.5537109375,
|
|
"loss/preference_sft": 0.30193933844566345,
|
|
"loss/reference_anchor": 0.3806990385055542,
|
|
"rewards/accuracies": 0.7875000238418579,
|
|
"rewards/chosen": 2.774885892868042,
|
|
"rewards/margins": 0.9939848780632019,
|
|
"rewards/rejected": 1.7809009552001953,
|
|
"step": 590
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1820889711380005,
|
|
"drift/rejected_abs": 0.23861515522003174,
|
|
"epoch": 0.5684939687089454,
|
|
"grad_norm": 728.0,
|
|
"learning_rate": 7.817417582301098e-08,
|
|
"logits/chosen": -2.0625860691070557,
|
|
"logits/rejected": -1.9918296337127686,
|
|
"logps/chosen": -0.27388039231300354,
|
|
"logps/rejected": -0.2751568555831909,
|
|
"loss": 4452.5673828125,
|
|
"loss/core": 4452.55859375,
|
|
"loss/preference_sft": 0.27388039231300354,
|
|
"loss/reference_anchor": 0.40798282623291016,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.8208898305892944,
|
|
"rewards/margins": -0.5628082752227783,
|
|
"rewards/rejected": 2.383697986602783,
|
|
"step": 595
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17881783843040466,
|
|
"drift/rejected_abs": 0.11638796329498291,
|
|
"epoch": 0.5732712289501971,
|
|
"grad_norm": 2560.0,
|
|
"learning_rate": 7.59355642445566e-08,
|
|
"logits/chosen": -2.1608338356018066,
|
|
"logits/rejected": -2.074814558029175,
|
|
"logps/chosen": -0.2959844470024109,
|
|
"logps/rejected": -0.30641934275627136,
|
|
"loss": 4436.38525390625,
|
|
"loss/core": 4436.38037109375,
|
|
"loss/preference_sft": 0.2959844470024109,
|
|
"loss/reference_anchor": 0.1849345564842224,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.7870938777923584,
|
|
"rewards/margins": 0.6242033839225769,
|
|
"rewards/rejected": 1.1628906726837158,
|
|
"step": 600
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19429127871990204,
|
|
"drift/rejected_abs": 0.06751377135515213,
|
|
"epoch": 0.5780484891914487,
|
|
"grad_norm": 1664.0,
|
|
"learning_rate": 7.37154037938015e-08,
|
|
"logits/chosen": -2.3095786571502686,
|
|
"logits/rejected": -2.3983094692230225,
|
|
"logps/chosen": -0.29155880212783813,
|
|
"logps/rejected": -0.3032078146934509,
|
|
"loss": 4426.8818359375,
|
|
"loss/core": 4426.87744140625,
|
|
"loss/preference_sft": 0.29155880212783813,
|
|
"loss/reference_anchor": 0.20911045372486115,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.9402952194213867,
|
|
"rewards/margins": 1.3450243473052979,
|
|
"rewards/rejected": 0.5952709913253784,
|
|
"step": 605
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.231297105550766,
|
|
"drift/rejected_abs": 0.15627989172935486,
|
|
"epoch": 0.5828257494327004,
|
|
"grad_norm": 700.0,
|
|
"learning_rate": 7.151452938229325e-08,
|
|
"logits/chosen": -2.0597474575042725,
|
|
"logits/rejected": -2.12882661819458,
|
|
"logps/chosen": -0.30136600136756897,
|
|
"logps/rejected": -0.310443252325058,
|
|
"loss": 4434.5126953125,
|
|
"loss/core": 4434.5048828125,
|
|
"loss/preference_sft": 0.30136600136756897,
|
|
"loss/reference_anchor": 0.3506472408771515,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.3125925064086914,
|
|
"rewards/margins": 0.750630259513855,
|
|
"rewards/rejected": 1.5619620084762573,
|
|
"step": 610
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14156286418437958,
|
|
"drift/rejected_abs": 0.07604067772626877,
|
|
"epoch": 0.587603009673952,
|
|
"grad_norm": 394.0,
|
|
"learning_rate": 6.933376866888883e-08,
|
|
"logits/chosen": -2.1115915775299072,
|
|
"logits/rejected": -2.091139316558838,
|
|
"logps/chosen": -0.3194340765476227,
|
|
"logps/rejected": -0.3096611499786377,
|
|
"loss": 4435.76904296875,
|
|
"loss/core": 4435.7666015625,
|
|
"loss/preference_sft": 0.3194340765476227,
|
|
"loss/reference_anchor": 0.07679594308137894,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.4153246879577637,
|
|
"rewards/margins": 0.655189037322998,
|
|
"rewards/rejected": 0.7601357102394104,
|
|
"step": 615
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1898563951253891,
|
|
"drift/rejected_abs": 0.13657966256141663,
|
|
"epoch": 0.5923802699152036,
|
|
"grad_norm": 908.0,
|
|
"learning_rate": 6.717394174850605e-08,
|
|
"logits/chosen": -2.2214183807373047,
|
|
"logits/rejected": -2.182659387588501,
|
|
"logps/chosen": -0.2612680494785309,
|
|
"logps/rejected": -0.27601683139801025,
|
|
"loss": 4437.4326171875,
|
|
"loss/core": 4437.42919921875,
|
|
"loss/preference_sft": 0.2612680494785309,
|
|
"loss/reference_anchor": 0.1394243687391281,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.8978593349456787,
|
|
"rewards/margins": 0.5342652797698975,
|
|
"rewards/rejected": 1.3635940551757812,
|
|
"step": 620
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15214236080646515,
|
|
"drift/rejected_abs": 0.09313346445560455,
|
|
"epoch": 0.5971575301564552,
|
|
"grad_norm": 1688.0,
|
|
"learning_rate": 6.503586084371926e-08,
|
|
"logits/chosen": -2.0383543968200684,
|
|
"logits/rejected": -2.0106043815612793,
|
|
"logps/chosen": -0.29384806752204895,
|
|
"logps/rejected": -0.2968134880065918,
|
|
"loss": 4436.52734375,
|
|
"loss/core": 4436.52490234375,
|
|
"loss/preference_sft": 0.29384806752204895,
|
|
"loss/reference_anchor": 0.0648588091135025,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.5202014446258545,
|
|
"rewards/margins": 0.5949220061302185,
|
|
"rewards/rejected": 0.9252792596817017,
|
|
"step": 625
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1255757361650467,
|
|
"drift/rejected_abs": 0.09434159100055695,
|
|
"epoch": 0.6019347903977069,
|
|
"grad_norm": 1464.0,
|
|
"learning_rate": 6.29203299993155e-08,
|
|
"logits/chosen": -2.274817705154419,
|
|
"logits/rejected": -2.197274684906006,
|
|
"logps/chosen": -0.2789786458015442,
|
|
"logps/rejected": -0.29018187522888184,
|
|
"loss": 4440.3251953125,
|
|
"loss/core": 4440.3232421875,
|
|
"loss/preference_sft": 0.2789786458015442,
|
|
"loss/reference_anchor": 0.05480783060193062,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.25403892993927,
|
|
"rewards/margins": 0.3123100697994232,
|
|
"rewards/rejected": 0.9417287707328796,
|
|
"step": 630
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.32034021615982056,
|
|
"drift/rejected_abs": 0.08672090619802475,
|
|
"epoch": 0.6067120506389586,
|
|
"grad_norm": 322.0,
|
|
"learning_rate": 6.082814477992656e-08,
|
|
"logits/chosen": -2.096656560897827,
|
|
"logits/rejected": -2.1229050159454346,
|
|
"logps/chosen": -0.30800944566726685,
|
|
"logps/rejected": -0.2923958897590637,
|
|
"loss": 4414.3359375,
|
|
"loss/core": 4414.32177734375,
|
|
"loss/preference_sft": 0.30800944566726685,
|
|
"loss/reference_anchor": 0.643897533416748,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.201916217803955,
|
|
"rewards/margins": 2.337477207183838,
|
|
"rewards/rejected": 0.8644390106201172,
|
|
"step": 635
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24005039036273956,
|
|
"drift/rejected_abs": 0.06744923442602158,
|
|
"epoch": 0.6114893108802102,
|
|
"grad_norm": 4000.0,
|
|
"learning_rate": 5.87600919708494e-08,
|
|
"logits/chosen": -2.0245227813720703,
|
|
"logits/rejected": -2.164236545562744,
|
|
"logps/chosen": -0.2961650490760803,
|
|
"logps/rejected": -0.28276434540748596,
|
|
"loss": 4421.81494140625,
|
|
"loss/core": 4421.8095703125,
|
|
"loss/preference_sft": 0.2961650490760803,
|
|
"loss/reference_anchor": 0.24256105720996857,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.400282621383667,
|
|
"rewards/margins": 1.7267650365829468,
|
|
"rewards/rejected": 0.673517644405365,
|
|
"step": 640
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4588896632194519,
|
|
"drift/rejected_abs": 0.15720048546791077,
|
|
"epoch": 0.6162665711214619,
|
|
"grad_norm": 1512.0,
|
|
"learning_rate": 5.671694928216829e-08,
|
|
"logits/chosen": -2.078514575958252,
|
|
"logits/rejected": -2.1803154945373535,
|
|
"logps/chosen": -0.2796784043312073,
|
|
"logps/rejected": -0.2615285813808441,
|
|
"loss": 4405.5849609375,
|
|
"loss/core": 4405.5576171875,
|
|
"loss/preference_sft": 0.2796784043312073,
|
|
"loss/reference_anchor": 1.3149816989898682,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 4.588896751403809,
|
|
"rewards/margins": 3.0177664756774902,
|
|
"rewards/rejected": 1.5711300373077393,
|
|
"step": 645
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.34622058272361755,
|
|
"drift/rejected_abs": 0.09166648238897324,
|
|
"epoch": 0.6210438313627135,
|
|
"grad_norm": 1000.0,
|
|
"learning_rate": 5.469948505629e-08,
|
|
"logits/chosen": -2.298170566558838,
|
|
"logits/rejected": -2.3131611347198486,
|
|
"logps/chosen": -0.29029572010040283,
|
|
"logps/rejected": -0.2863155007362366,
|
|
"loss": 4411.43017578125,
|
|
"loss/core": 4411.4130859375,
|
|
"loss/preference_sft": 0.29029572010040283,
|
|
"loss/reference_anchor": 0.8013036847114563,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.4620628356933594,
|
|
"rewards/margins": 2.561060905456543,
|
|
"rewards/rejected": 0.9010015726089478,
|
|
"step": 650
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28553155064582825,
|
|
"drift/rejected_abs": 0.17389452457427979,
|
|
"epoch": 0.6258210916039652,
|
|
"grad_norm": 1688.0,
|
|
"learning_rate": 5.270845797900168e-08,
|
|
"logits/chosen": -1.9482574462890625,
|
|
"logits/rejected": -1.9404575824737549,
|
|
"logps/chosen": -0.2796974182128906,
|
|
"logps/rejected": -0.30062079429626465,
|
|
"loss": 4429.71923828125,
|
|
"loss/core": 4429.712890625,
|
|
"loss/preference_sft": 0.2796974182128906,
|
|
"loss/reference_anchor": 0.2853711247444153,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.8548009395599365,
|
|
"rewards/margins": 1.118624210357666,
|
|
"rewards/rejected": 1.73617684841156,
|
|
"step": 655
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2055063545703888,
|
|
"drift/rejected_abs": 0.10912273079156876,
|
|
"epoch": 0.6305983518452167,
|
|
"grad_norm": 1080.0,
|
|
"learning_rate": 5.0744616794160104e-08,
|
|
"logits/chosen": -2.2815113067626953,
|
|
"logits/rejected": -2.3383548259735107,
|
|
"logps/chosen": -0.2732482850551605,
|
|
"logps/rejected": -0.2792763113975525,
|
|
"loss": 4431.6318359375,
|
|
"loss/core": 4431.62744140625,
|
|
"loss/preference_sft": 0.2732482850551605,
|
|
"loss/reference_anchor": 0.17782394587993622,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.055063486099243,
|
|
"rewards/margins": 0.9659796953201294,
|
|
"rewards/rejected": 1.0890839099884033,
|
|
"step": 660
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1800767034292221,
|
|
"drift/rejected_abs": 0.12754087150096893,
|
|
"epoch": 0.6353756120864684,
|
|
"grad_norm": 1536.0,
|
|
"learning_rate": 4.880870002211963e-08,
|
|
"logits/chosen": -2.1525440216064453,
|
|
"logits/rejected": -2.227254867553711,
|
|
"logps/chosen": -0.29735860228538513,
|
|
"logps/rejected": -0.2968386113643646,
|
|
"loss": 4437.6201171875,
|
|
"loss/core": 4437.6162109375,
|
|
"loss/preference_sft": 0.29735860228538513,
|
|
"loss/reference_anchor": 0.21875838935375214,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.7989108562469482,
|
|
"rewards/margins": 0.5276275873184204,
|
|
"rewards/rejected": 1.2712833881378174,
|
|
"step": 665
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2005433738231659,
|
|
"drift/rejected_abs": 0.12166965007781982,
|
|
"epoch": 0.64015287232772,
|
|
"grad_norm": 876.0,
|
|
"learning_rate": 4.6901435682004996e-08,
|
|
"logits/chosen": -2.293273448944092,
|
|
"logits/rejected": -2.3247194290161133,
|
|
"logps/chosen": -0.28415507078170776,
|
|
"logps/rejected": -0.2904112935066223,
|
|
"loss": 4434.0439453125,
|
|
"loss/core": 4434.03857421875,
|
|
"loss/preference_sft": 0.28415507078170776,
|
|
"loss/reference_anchor": 0.20630235970020294,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.002830982208252,
|
|
"rewards/margins": 0.7875508069992065,
|
|
"rewards/rejected": 1.2152799367904663,
|
|
"step": 670
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19195790588855743,
|
|
"drift/rejected_abs": 0.10611511766910553,
|
|
"epoch": 0.6449301325689717,
|
|
"grad_norm": 976.0,
|
|
"learning_rate": 4.502354101793314e-08,
|
|
"logits/chosen": -2.1044974327087402,
|
|
"logits/rejected": -2.231261730194092,
|
|
"logps/chosen": -0.2956617772579193,
|
|
"logps/rejected": -0.28804105520248413,
|
|
"loss": 4433.052734375,
|
|
"loss/core": 4433.0478515625,
|
|
"loss/preference_sft": 0.2956617772579193,
|
|
"loss/reference_anchor": 0.15880243480205536,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.9193670749664307,
|
|
"rewards/margins": 0.8593287467956543,
|
|
"rewards/rejected": 1.0600383281707764,
|
|
"step": 675
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.12193242460489273,
|
|
"drift/rejected_abs": 0.14080289006233215,
|
|
"epoch": 0.6497073928102234,
|
|
"grad_norm": 4992.0,
|
|
"learning_rate": 4.3175722229287034e-08,
|
|
"logits/chosen": -2.1528353691101074,
|
|
"logits/rejected": -2.117715358734131,
|
|
"logps/chosen": -0.2846739888191223,
|
|
"logps/rejected": -0.27910977602005005,
|
|
"loss": 4447.14404296875,
|
|
"loss/core": 4447.1396484375,
|
|
"loss/preference_sft": 0.2846739888191223,
|
|
"loss/reference_anchor": 0.1636960357427597,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.2183479070663452,
|
|
"rewards/margins": -0.1849554032087326,
|
|
"rewards/rejected": 1.4033033847808838,
|
|
"step": 680
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14501406252384186,
|
|
"drift/rejected_abs": 0.1529538929462433,
|
|
"epoch": 0.654484653051475,
|
|
"grad_norm": 1912.0,
|
|
"learning_rate": 4.135867420514276e-08,
|
|
"logits/chosen": -2.2104806900024414,
|
|
"logits/rejected": -2.2139463424682617,
|
|
"logps/chosen": -0.29691627621650696,
|
|
"logps/rejected": -0.29713374376296997,
|
|
"loss": 4445.89892578125,
|
|
"loss/core": 4445.89208984375,
|
|
"loss/preference_sft": 0.29691627621650696,
|
|
"loss/reference_anchor": 0.28842589259147644,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.4487031698226929,
|
|
"rewards/margins": -0.07711444050073624,
|
|
"rewards/rejected": 1.525817632675171,
|
|
"step": 685
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3045670986175537,
|
|
"drift/rejected_abs": 0.17205330729484558,
|
|
"epoch": 0.6592619132927267,
|
|
"grad_norm": 1632.0,
|
|
"learning_rate": 3.957308026295035e-08,
|
|
"logits/chosen": -1.9819128513336182,
|
|
"logits/rejected": -1.9787757396697998,
|
|
"logps/chosen": -0.26376795768737793,
|
|
"logps/rejected": -0.27589425444602966,
|
|
"loss": 4426.86669921875,
|
|
"loss/core": 4426.86083984375,
|
|
"loss/preference_sft": 0.26376795768737793,
|
|
"loss/reference_anchor": 0.3071330487728119,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.0436840057373047,
|
|
"rewards/margins": 1.3387844562530518,
|
|
"rewards/rejected": 1.7048994302749634,
|
|
"step": 690
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14530780911445618,
|
|
"drift/rejected_abs": 0.08450115472078323,
|
|
"epoch": 0.6640391735339782,
|
|
"grad_norm": 828.0,
|
|
"learning_rate": 3.7819611891566084e-08,
|
|
"logits/chosen": -2.2839925289154053,
|
|
"logits/rejected": -2.149257183074951,
|
|
"logps/chosen": -0.29438167810440063,
|
|
"logps/rejected": -0.3029247224330902,
|
|
"loss": 4435.900390625,
|
|
"loss/core": 4435.8994140625,
|
|
"loss/preference_sft": 0.29438167810440063,
|
|
"loss/reference_anchor": 0.06458897143602371,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.4513713121414185,
|
|
"rewards/margins": 0.645209550857544,
|
|
"rewards/rejected": 0.8061617612838745,
|
|
"step": 695
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.30958184599876404,
|
|
"drift/rejected_abs": 0.14133182168006897,
|
|
"epoch": 0.6688164337752299,
|
|
"grad_norm": 3184.0,
|
|
"learning_rate": 3.609892849873286e-08,
|
|
"logits/chosen": -2.1187186241149902,
|
|
"logits/rejected": -2.1548023223876953,
|
|
"logps/chosen": -0.28238362073898315,
|
|
"logps/rejected": -0.268256813287735,
|
|
"loss": 4422.44287109375,
|
|
"loss/core": 4422.43359375,
|
|
"loss/preference_sft": 0.28238362073898315,
|
|
"loss/reference_anchor": 0.4544747471809387,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.094094753265381,
|
|
"rewards/margins": 1.6822659969329834,
|
|
"rewards/rejected": 1.411828875541687,
|
|
"step": 700
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1411522626876831,
|
|
"drift/rejected_abs": 0.12172625958919525,
|
|
"epoch": 0.6735936940164815,
|
|
"grad_norm": 5376.0,
|
|
"learning_rate": 3.4411677163103894e-08,
|
|
"logits/chosen": -2.031183958053589,
|
|
"logits/rejected": -2.114187479019165,
|
|
"logps/chosen": -0.27659520506858826,
|
|
"logps/rejected": -0.2737487256526947,
|
|
"loss": 4441.9150390625,
|
|
"loss/core": 4441.912109375,
|
|
"loss/preference_sft": 0.27659520506858826,
|
|
"loss/reference_anchor": 0.11837001889944077,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.4112656116485596,
|
|
"rewards/margins": 0.19559058547019958,
|
|
"rewards/rejected": 1.2156749963760376,
|
|
"step": 705
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2575318515300751,
|
|
"drift/rejected_abs": 0.13676753640174866,
|
|
"epoch": 0.6783709542577332,
|
|
"grad_norm": 684.0,
|
|
"learning_rate": 3.2758492390902945e-08,
|
|
"logits/chosen": -2.029358148574829,
|
|
"logits/rejected": -2.0675930976867676,
|
|
"logps/chosen": -0.28309616446495056,
|
|
"logps/rejected": -0.28654059767723083,
|
|
"loss": 4428.4951171875,
|
|
"loss/core": 4428.49072265625,
|
|
"loss/preference_sft": 0.28309616446495056,
|
|
"loss/reference_anchor": 0.1890076845884323,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.574094533920288,
|
|
"rewards/margins": 1.2082717418670654,
|
|
"rewards/rejected": 1.3658227920532227,
|
|
"step": 710
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2860047519207001,
|
|
"drift/rejected_abs": 0.1380338966846466,
|
|
"epoch": 0.6831482144989849,
|
|
"grad_norm": 508.0,
|
|
"learning_rate": 3.11399958773126e-08,
|
|
"logits/chosen": -2.116067886352539,
|
|
"logits/rejected": -2.1624484062194824,
|
|
"logps/chosen": -0.2840300500392914,
|
|
"logps/rejected": -0.29530325531959534,
|
|
"loss": 4424.92333984375,
|
|
"loss/core": 4424.91796875,
|
|
"loss/preference_sft": 0.2840300500392914,
|
|
"loss/reference_anchor": 0.24914495646953583,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.857480764389038,
|
|
"rewards/margins": 1.4812989234924316,
|
|
"rewards/rejected": 1.376182198524475,
|
|
"step": 715
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22536201775074005,
|
|
"drift/rejected_abs": 0.10613832622766495,
|
|
"epoch": 0.6879254747402365,
|
|
"grad_norm": 632.0,
|
|
"learning_rate": 2.9556796272679972e-08,
|
|
"logits/chosen": -2.2809996604919434,
|
|
"logits/rejected": -2.2536635398864746,
|
|
"logps/chosen": -0.29658135771751404,
|
|
"logps/rejected": -0.29319730401039124,
|
|
"loss": 4428.8095703125,
|
|
"loss/core": 4428.8037109375,
|
|
"loss/preference_sft": 0.29658135771751404,
|
|
"loss/reference_anchor": 0.24946120381355286,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.252169132232666,
|
|
"rewards/margins": 1.1916897296905518,
|
|
"rewards/rejected": 1.0604794025421143,
|
|
"step": 720
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.11407455056905746,
|
|
"drift/rejected_abs": 0.07727280259132385,
|
|
"epoch": 0.6927027349814882,
|
|
"grad_norm": 840.0,
|
|
"learning_rate": 2.8009488953628215e-08,
|
|
"logits/chosen": -2.1286325454711914,
|
|
"logits/rejected": -2.1653316020965576,
|
|
"logps/chosen": -0.318909227848053,
|
|
"logps/rejected": -0.31315773725509644,
|
|
"loss": 4439.55126953125,
|
|
"loss/core": 4439.5498046875,
|
|
"loss/preference_sft": 0.318909227848053,
|
|
"loss/reference_anchor": 0.04532434046268463,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.1400318145751953,
|
|
"rewards/margins": 0.3686821460723877,
|
|
"rewards/rejected": 0.7713496685028076,
|
|
"step": 725
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2567460536956787,
|
|
"drift/rejected_abs": 0.08764373511075974,
|
|
"epoch": 0.6974799952227397,
|
|
"grad_norm": 1560.0,
|
|
"learning_rate": 2.649865579915976e-08,
|
|
"logits/chosen": -2.2115912437438965,
|
|
"logits/rejected": -2.209638833999634,
|
|
"logps/chosen": -0.29488250613212585,
|
|
"logps/rejected": -0.2987883985042572,
|
|
"loss": 4422.2177734375,
|
|
"loss/core": 4422.21142578125,
|
|
"loss/preference_sft": 0.29488250613212585,
|
|
"loss/reference_anchor": 0.2926619052886963,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.567460775375366,
|
|
"rewards/margins": 1.693508505821228,
|
|
"rewards/rejected": 0.8739525079727173,
|
|
"step": 730
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2040727585554123,
|
|
"drift/rejected_abs": 0.13325053453445435,
|
|
"epoch": 0.7022572554639914,
|
|
"grad_norm": 1256.0,
|
|
"learning_rate": 2.5024864971835507e-08,
|
|
"logits/chosen": -2.227663516998291,
|
|
"logits/rejected": -2.234522581100464,
|
|
"logps/chosen": -0.26899558305740356,
|
|
"logps/rejected": -0.28572389483451843,
|
|
"loss": 4435.130859375,
|
|
"loss/core": 4435.1279296875,
|
|
"loss/preference_sft": 0.26899558305740356,
|
|
"loss/reference_anchor": 0.1256342977285385,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.039360523223877,
|
|
"rewards/margins": 0.708215594291687,
|
|
"rewards/rejected": 1.3311448097229004,
|
|
"step": 735
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2198764532804489,
|
|
"drift/rejected_abs": 0.1182849183678627,
|
|
"epoch": 0.707034515705243,
|
|
"grad_norm": 948.0,
|
|
"learning_rate": 2.3588670704111997e-08,
|
|
"logits/chosen": -2.0449001789093018,
|
|
"logits/rejected": -2.077310800552368,
|
|
"logps/chosen": -0.2672049403190613,
|
|
"logps/rejected": -0.28309932351112366,
|
|
"loss": 4430.99560546875,
|
|
"loss/core": 4430.9921875,
|
|
"loss/preference_sft": 0.2672049403190613,
|
|
"loss/reference_anchor": 0.17148524522781372,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.195984363555908,
|
|
"rewards/margins": 1.0152232646942139,
|
|
"rewards/rejected": 1.1807608604431152,
|
|
"step": 740
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.318195104598999,
|
|
"drift/rejected_abs": 0.21116037666797638,
|
|
"epoch": 0.7118117759464947,
|
|
"grad_norm": 2128.0,
|
|
"learning_rate": 2.219061308991721e-08,
|
|
"logits/chosen": -2.014070987701416,
|
|
"logits/rejected": -2.0407216548919678,
|
|
"logps/chosen": -0.28043967485427856,
|
|
"logps/rejected": -0.2645443081855774,
|
|
"loss": 4431.36669921875,
|
|
"loss/core": 4431.3525390625,
|
|
"loss/preference_sft": 0.28043967485427856,
|
|
"loss/reference_anchor": 0.6777635812759399,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.1789755821228027,
|
|
"rewards/margins": 1.0692497491836548,
|
|
"rewards/rejected": 2.1097254753112793,
|
|
"step": 745
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25101321935653687,
|
|
"drift/rejected_abs": 0.15045872330665588,
|
|
"epoch": 0.7165890361877463,
|
|
"grad_norm": 912.0,
|
|
"learning_rate": 2.0831217881543557e-08,
|
|
"logits/chosen": -2.0724101066589355,
|
|
"logits/rejected": -2.101039409637451,
|
|
"logps/chosen": -0.25144457817077637,
|
|
"logps/rejected": -0.27629944682121277,
|
|
"loss": 4431.208984375,
|
|
"loss/core": 4431.20361328125,
|
|
"loss/preference_sft": 0.25144457817077637,
|
|
"loss/reference_anchor": 0.23283448815345764,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.5094053745269775,
|
|
"rewards/margins": 1.0123567581176758,
|
|
"rewards/rejected": 1.4970487356185913,
|
|
"step": 750
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4007979929447174,
|
|
"drift/rejected_abs": 0.2181040346622467,
|
|
"epoch": 0.721366296428998,
|
|
"grad_norm": 412.0,
|
|
"learning_rate": 1.951099629193366e-08,
|
|
"logits/chosen": -2.212562084197998,
|
|
"logits/rejected": -2.208326816558838,
|
|
"logps/chosen": -0.36083754897117615,
|
|
"logps/rejected": -0.28320175409317017,
|
|
"loss": 4421.8203125,
|
|
"loss/core": 4421.7998046875,
|
|
"loss/preference_sft": 0.36083754897117615,
|
|
"loss/reference_anchor": 1.0116541385650635,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 4.007124423980713,
|
|
"rewards/margins": 1.8269456624984741,
|
|
"rewards/rejected": 2.180178642272949,
|
|
"step": 755
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1610032469034195,
|
|
"drift/rejected_abs": 0.07328177988529205,
|
|
"epoch": 0.7261435566702497,
|
|
"grad_norm": 836.0,
|
|
"learning_rate": 1.823044480243431e-08,
|
|
"logits/chosen": -2.1159017086029053,
|
|
"logits/rejected": -2.1593313217163086,
|
|
"logps/chosen": -0.29487186670303345,
|
|
"logps/rejected": -0.29895296692848206,
|
|
"loss": 4432.8603515625,
|
|
"loss/core": 4432.85693359375,
|
|
"loss/preference_sft": 0.29487186670303345,
|
|
"loss/reference_anchor": 0.1201050877571106,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.608341932296753,
|
|
"rewards/margins": 0.8831087350845337,
|
|
"rewards/rejected": 0.7252334356307983,
|
|
"step": 760
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14802543818950653,
|
|
"drift/rejected_abs": 0.09172812104225159,
|
|
"epoch": 0.7309208169115012,
|
|
"grad_norm": 968.0,
|
|
"learning_rate": 1.699004497608994e-08,
|
|
"logits/chosen": -2.2781732082366943,
|
|
"logits/rejected": -2.3036012649536133,
|
|
"logps/chosen": -0.27361828088760376,
|
|
"logps/rejected": -0.2817005515098572,
|
|
"loss": 4436.9404296875,
|
|
"loss/core": 4436.93896484375,
|
|
"loss/preference_sft": 0.27361828088760376,
|
|
"loss/reference_anchor": 0.04991341382265091,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.478417158126831,
|
|
"rewards/margins": 0.5637811422348022,
|
|
"rewards/rejected": 0.914635956287384,
|
|
"step": 765
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2051977664232254,
|
|
"drift/rejected_abs": 0.10878431797027588,
|
|
"epoch": 0.7356980771527529,
|
|
"grad_norm": 2448.0,
|
|
"learning_rate": 1.5790263276546658e-08,
|
|
"logits/chosen": -2.144563674926758,
|
|
"logits/rejected": -2.1687493324279785,
|
|
"logps/chosen": -0.2964807152748108,
|
|
"logps/rejected": -0.28825658559799194,
|
|
"loss": 4431.6572265625,
|
|
"loss/core": 4431.6533203125,
|
|
"loss/preference_sft": 0.2964807152748108,
|
|
"loss/reference_anchor": 0.15945471823215485,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.049602508544922,
|
|
"rewards/margins": 0.9648269414901733,
|
|
"rewards/rejected": 1.0847753286361694,
|
|
"step": 770
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20840251445770264,
|
|
"drift/rejected_abs": 0.09104669094085693,
|
|
"epoch": 0.7404753373940045,
|
|
"grad_norm": 2640.0,
|
|
"learning_rate": 1.4631550892634126e-08,
|
|
"logits/chosen": -2.2751145362854004,
|
|
"logits/rejected": -2.253349781036377,
|
|
"logps/chosen": -0.26738208532333374,
|
|
"logps/rejected": -0.26774922013282776,
|
|
"loss": 4428.873046875,
|
|
"loss/core": 4428.869140625,
|
|
"loss/preference_sft": 0.26738208532333374,
|
|
"loss/reference_anchor": 0.17535707354545593,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.0836570262908936,
|
|
"rewards/margins": 1.174851655960083,
|
|
"rewards/rejected": 0.908805251121521,
|
|
"step": 775
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28770846128463745,
|
|
"drift/rejected_abs": 0.17277604341506958,
|
|
"epoch": 0.7452525976352562,
|
|
"grad_norm": 1576.0,
|
|
"learning_rate": 1.3514343568692132e-08,
|
|
"logits/chosen": -2.053027391433716,
|
|
"logits/rejected": -2.161233425140381,
|
|
"logps/chosen": -0.2801096737384796,
|
|
"logps/rejected": -0.2799223065376282,
|
|
"loss": 4429.0986328125,
|
|
"loss/core": 4429.0927734375,
|
|
"loss/preference_sft": 0.2801096737384796,
|
|
"loss/reference_anchor": 0.25047561526298523,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.874798536300659,
|
|
"rewards/margins": 1.1635240316390991,
|
|
"rewards/rejected": 1.71127450466156,
|
|
"step": 780
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24193139374256134,
|
|
"drift/rejected_abs": 0.15544036030769348,
|
|
"epoch": 0.7500298578765078,
|
|
"grad_norm": 1832.0,
|
|
"learning_rate": 1.2439061440704724e-08,
|
|
"logits/chosen": -1.9472965002059937,
|
|
"logits/rejected": -1.9168212413787842,
|
|
"logps/chosen": -0.2860909104347229,
|
|
"logps/rejected": -0.2906906306743622,
|
|
"loss": 4433.3818359375,
|
|
"loss/core": 4433.375,
|
|
"loss/preference_sft": 0.2860909104347229,
|
|
"loss/reference_anchor": 0.28603917360305786,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.4173223972320557,
|
|
"rewards/margins": 0.8639556765556335,
|
|
"rewards/rejected": 1.5533664226531982,
|
|
"step": 785
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17823731899261475,
|
|
"drift/rejected_abs": 0.08329329639673233,
|
|
"epoch": 0.7548071181177595,
|
|
"grad_norm": 764.0,
|
|
"learning_rate": 1.1406108878304468e-08,
|
|
"logits/chosen": -2.137251377105713,
|
|
"logits/rejected": -2.205977439880371,
|
|
"logps/chosen": -0.3069455027580261,
|
|
"logps/rejected": -0.3350505530834198,
|
|
"loss": 4431.853515625,
|
|
"loss/core": 4431.8515625,
|
|
"loss/preference_sft": 0.3069455027580261,
|
|
"loss/reference_anchor": 0.0917288064956665,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.7821671962738037,
|
|
"rewards/margins": 0.9510431289672852,
|
|
"rewards/rejected": 0.8311241269111633,
|
|
"step": 790
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24515828490257263,
|
|
"drift/rejected_abs": 0.21105511486530304,
|
|
"epoch": 0.7595843783590112,
|
|
"grad_norm": 4512.0,
|
|
"learning_rate": 1.0415874332705381e-08,
|
|
"logits/chosen": -2.0945167541503906,
|
|
"logits/rejected": -2.070615291595459,
|
|
"logps/chosen": -0.2601197361946106,
|
|
"logps/rejected": -0.26042866706848145,
|
|
"loss": 4440.27392578125,
|
|
"loss/core": 4440.26611328125,
|
|
"loss/preference_sft": 0.2601197361946106,
|
|
"loss/reference_anchor": 0.35151952505111694,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.451155424118042,
|
|
"rewards/margins": 0.34183400869369507,
|
|
"rewards/rejected": 2.109321355819702,
|
|
"step": 795
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.35011768341064453,
|
|
"drift/rejected_abs": 0.09590541571378708,
|
|
"epoch": 0.7643616386002627,
|
|
"grad_norm": 880.0,
|
|
"learning_rate": 9.468730190622484e-09,
|
|
"logits/chosen": -2.0071511268615723,
|
|
"logits/rejected": -2.0719618797302246,
|
|
"logps/chosen": -0.31157320737838745,
|
|
"logps/rejected": -0.2820609211921692,
|
|
"loss": 4411.7822265625,
|
|
"loss/core": 4411.767578125,
|
|
"loss/preference_sft": 0.31157320737838745,
|
|
"loss/reference_anchor": 0.7128745317459106,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.5011768341064453,
|
|
"rewards/margins": 2.5428547859191895,
|
|
"rewards/rejected": 0.958321750164032,
|
|
"step": 800
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20914137363433838,
|
|
"drift/rejected_abs": 0.09575755894184113,
|
|
"epoch": 0.7691388988415144,
|
|
"grad_norm": 1840.0,
|
|
"learning_rate": 8.565032634232194e-09,
|
|
"logits/chosen": -2.109005928039551,
|
|
"logits/rejected": -2.2536232471466064,
|
|
"logps/chosen": -0.2827692925930023,
|
|
"logps/rejected": -0.2778649926185608,
|
|
"loss": 4429.55029296875,
|
|
"loss/core": 4429.54638671875,
|
|
"loss/preference_sft": 0.2827692925930023,
|
|
"loss/reference_anchor": 0.17659203708171844,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.0893187522888184,
|
|
"rewards/margins": 1.1325154304504395,
|
|
"rewards/rejected": 0.9568031430244446,
|
|
"step": 805
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2004438191652298,
|
|
"drift/rejected_abs": 0.08489207923412323,
|
|
"epoch": 0.773916159082766,
|
|
"grad_norm": 1696.0,
|
|
"learning_rate": 7.70512150722702e-09,
|
|
"logits/chosen": -2.398157835006714,
|
|
"logits/rejected": -2.384509563446045,
|
|
"logps/chosen": -0.27724185585975647,
|
|
"logps/rejected": -0.2638983428478241,
|
|
"loss": 4429.2548828125,
|
|
"loss/core": 4429.25,
|
|
"loss/preference_sft": 0.27724185585975647,
|
|
"loss/reference_anchor": 0.17924687266349792,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.0037176609039307,
|
|
"rewards/margins": 1.1569417715072632,
|
|
"rewards/rejected": 0.8467755317687988,
|
|
"step": 810
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2000669687986374,
|
|
"drift/rejected_abs": 0.12345165014266968,
|
|
"epoch": 0.7786934193240177,
|
|
"grad_norm": 8064.0,
|
|
"learning_rate": 6.8893201870139915e-09,
|
|
"logits/chosen": -2.0791401863098145,
|
|
"logits/rejected": -2.0309667587280273,
|
|
"logps/chosen": -0.28644925355911255,
|
|
"logps/rejected": -0.30002540349960327,
|
|
"loss": 4434.1357421875,
|
|
"loss/core": 4434.1328125,
|
|
"loss/preference_sft": 0.28644925355911255,
|
|
"loss/reference_anchor": 0.10940603911876678,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.9967515468597412,
|
|
"rewards/margins": 0.7785899043083191,
|
|
"rewards/rejected": 1.2181618213653564,
|
|
"step": 815
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2549762427806854,
|
|
"drift/rejected_abs": 0.12483543157577515,
|
|
"epoch": 0.7834706795652693,
|
|
"grad_norm": 1976.0,
|
|
"learning_rate": 6.117935463105808e-09,
|
|
"logits/chosen": -2.0269968509674072,
|
|
"logits/rejected": -2.0181238651275635,
|
|
"logps/chosen": -0.289524108171463,
|
|
"logps/rejected": -0.2884969115257263,
|
|
"loss": 4427.1259765625,
|
|
"loss/core": 4427.119140625,
|
|
"loss/preference_sft": 0.289524108171463,
|
|
"loss/reference_anchor": 0.3114100396633148,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.548508405685425,
|
|
"rewards/margins": 1.3080947399139404,
|
|
"rewards/rejected": 1.2404136657714844,
|
|
"step": 820
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22770798206329346,
|
|
"drift/rejected_abs": 0.07756448537111282,
|
|
"epoch": 0.788247939806521,
|
|
"grad_norm": 540.0,
|
|
"learning_rate": 5.391257421749826e-09,
|
|
"logits/chosen": -2.086367130279541,
|
|
"logits/rejected": -2.2468793392181396,
|
|
"logps/chosen": -0.30201080441474915,
|
|
"logps/rejected": -0.342225044965744,
|
|
"loss": 4424.48291015625,
|
|
"loss/core": 4424.47705078125,
|
|
"loss/preference_sft": 0.30201080441474915,
|
|
"loss/reference_anchor": 0.25235939025878906,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.2739996910095215,
|
|
"rewards/margins": 1.5204788446426392,
|
|
"rewards/rejected": 0.7535209059715271,
|
|
"step": 825
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18832604587078094,
|
|
"drift/rejected_abs": 0.07938660681247711,
|
|
"epoch": 0.7930252000477725,
|
|
"grad_norm": 1336.0,
|
|
"learning_rate": 4.709559336838462e-09,
|
|
"logits/chosen": -2.1965444087982178,
|
|
"logits/rejected": -2.2099156379699707,
|
|
"logps/chosen": -0.2985237240791321,
|
|
"logps/rejected": -0.2869217097759247,
|
|
"loss": 4430.0537109375,
|
|
"loss/core": 4430.05078125,
|
|
"loss/preference_sft": 0.2985237240791321,
|
|
"loss/reference_anchor": 0.11851098388433456,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.8832508325576782,
|
|
"rewards/margins": 1.0893845558166504,
|
|
"rewards/rejected": 0.7938660979270935,
|
|
"step": 830
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21169748902320862,
|
|
"drift/rejected_abs": 0.0859481543302536,
|
|
"epoch": 0.7978024602890242,
|
|
"grad_norm": 1048.0,
|
|
"learning_rate": 4.073097567142025e-09,
|
|
"logits/chosen": -2.2265546321868896,
|
|
"logits/rejected": -2.2355074882507324,
|
|
"logps/chosen": -0.253441721200943,
|
|
"logps/rejected": -0.27110329270362854,
|
|
"loss": 4427.9150390625,
|
|
"loss/core": 4427.91064453125,
|
|
"loss/preference_sft": 0.253441721200943,
|
|
"loss/reference_anchor": 0.21069765090942383,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.1147894859313965,
|
|
"rewards/margins": 1.2563145160675049,
|
|
"rewards/rejected": 0.8584749102592468,
|
|
"step": 835
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3630954623222351,
|
|
"drift/rejected_abs": 0.11634461581707001,
|
|
"epoch": 0.8025797205302759,
|
|
"grad_norm": 6592.0,
|
|
"learning_rate": 3.482111459902695e-09,
|
|
"logits/chosen": -2.0539486408233643,
|
|
"logits/rejected": -2.109269857406616,
|
|
"logps/chosen": -0.2834726870059967,
|
|
"logps/rejected": -0.2910478711128235,
|
|
"loss": 4412.4912109375,
|
|
"loss/core": 4412.4755859375,
|
|
"loss/preference_sft": 0.2834726870059967,
|
|
"loss/reference_anchor": 0.7740469574928284,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.6292176246643066,
|
|
"rewards/margins": 2.487287998199463,
|
|
"rewards/rejected": 1.1419297456741333,
|
|
"step": 840
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2685644328594208,
|
|
"drift/rejected_abs": 0.23484978079795837,
|
|
"epoch": 0.8073569807715275,
|
|
"grad_norm": 2464.0,
|
|
"learning_rate": 2.9368232608258797e-09,
|
|
"logits/chosen": -1.914307951927185,
|
|
"logits/rejected": -1.9539330005645752,
|
|
"logps/chosen": -0.3026157021522522,
|
|
"logps/rejected": -0.30641502141952515,
|
|
"loss": 4440.4267578125,
|
|
"loss/core": 4440.4130859375,
|
|
"loss/preference_sft": 0.3026157021522522,
|
|
"loss/reference_anchor": 0.6450716257095337,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.6856446266174316,
|
|
"rewards/margins": 0.3389059603214264,
|
|
"rewards/rejected": 2.346738576889038,
|
|
"step": 845
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24007916450500488,
|
|
"drift/rejected_abs": 0.11487044394016266,
|
|
"epoch": 0.8121342410127792,
|
|
"grad_norm": 1616.0,
|
|
"learning_rate": 2.4374380305025866e-09,
|
|
"logits/chosen": -1.966447114944458,
|
|
"logits/rejected": -2.0533547401428223,
|
|
"logps/chosen": -0.286264568567276,
|
|
"logps/rejected": -0.2925958037376404,
|
|
"loss": 4427.82861328125,
|
|
"loss/core": 4427.8251953125,
|
|
"loss/preference_sft": 0.286264568567276,
|
|
"loss/reference_anchor": 0.16112756729125977,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.4002561569213867,
|
|
"rewards/margins": 1.252990961074829,
|
|
"rewards/rejected": 1.1472651958465576,
|
|
"step": 850
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2156863957643509,
|
|
"drift/rejected_abs": 0.12507030367851257,
|
|
"epoch": 0.8169115012540308,
|
|
"grad_norm": 1456.0,
|
|
"learning_rate": 1.984143567294594e-09,
|
|
"logits/chosen": -1.9173269271850586,
|
|
"logits/rejected": -1.8677393198013306,
|
|
"logps/chosen": -0.2849237322807312,
|
|
"logps/rejected": -0.2810590863227844,
|
|
"loss": 4432.4453125,
|
|
"loss/core": 4432.4404296875,
|
|
"loss/preference_sft": 0.2849237322807312,
|
|
"loss/reference_anchor": 0.21630068123340607,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.156566619873047,
|
|
"rewards/margins": 0.9074234962463379,
|
|
"rewards/rejected": 1.2491432428359985,
|
|
"step": 855
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19415462017059326,
|
|
"drift/rejected_abs": 0.12510396540164948,
|
|
"epoch": 0.8216887614952825,
|
|
"grad_norm": 916.0,
|
|
"learning_rate": 1.577110336711096e-09,
|
|
"logits/chosen": -2.001447916030884,
|
|
"logits/rejected": -2.0701100826263428,
|
|
"logps/chosen": -0.28479576110839844,
|
|
"logps/rejected": -0.3271367847919464,
|
|
"loss": 4435.3583984375,
|
|
"loss/core": 4435.35302734375,
|
|
"loss/preference_sft": 0.28479576110839844,
|
|
"loss/reference_anchor": 0.26654165983200073,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.9405349493026733,
|
|
"rewards/margins": 0.6905874013900757,
|
|
"rewards/rejected": 1.2499475479125977,
|
|
"step": 860
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22797951102256775,
|
|
"drift/rejected_abs": 0.13979032635688782,
|
|
"epoch": 0.826466021736534,
|
|
"grad_norm": 4576.0,
|
|
"learning_rate": 1.2164914073037048e-09,
|
|
"logits/chosen": -1.9181400537490845,
|
|
"logits/rejected": -1.976419448852539,
|
|
"logps/chosen": -0.25987738370895386,
|
|
"logps/rejected": -0.28612470626831055,
|
|
"loss": 4431.91015625,
|
|
"loss/core": 4431.9052734375,
|
|
"loss/preference_sft": 0.25987738370895386,
|
|
"loss/reference_anchor": 0.20739074051380157,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.2794971466064453,
|
|
"rewards/margins": 0.950373649597168,
|
|
"rewards/rejected": 1.3291234970092773,
|
|
"step": 865
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24134759604930878,
|
|
"drift/rejected_abs": 0.13137051463127136,
|
|
"epoch": 0.8312432819777857,
|
|
"grad_norm": 7136.0,
|
|
"learning_rate": 9.024223931035357e-10,
|
|
"logits/chosen": -2.3466553688049316,
|
|
"logits/rejected": -2.3163647651672363,
|
|
"logps/chosen": -0.3023620545864105,
|
|
"logps/rejected": -0.30314552783966064,
|
|
"loss": 4430.294921875,
|
|
"loss/core": 4430.28369140625,
|
|
"loss/preference_sft": 0.3023620545864105,
|
|
"loss/reference_anchor": 0.53348708152771,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.4105780124664307,
|
|
"rewards/margins": 1.0996638536453247,
|
|
"rewards/rejected": 1.3109138011932373,
|
|
"step": 870
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.169838547706604,
|
|
"drift/rejected_abs": 0.1980375051498413,
|
|
"epoch": 0.8360205422190374,
|
|
"grad_norm": 2000.0,
|
|
"learning_rate": 6.350214026223516e-10,
|
|
"logits/chosen": -2.20188570022583,
|
|
"logits/rejected": -2.0546789169311523,
|
|
"logps/chosen": -0.27584901452064514,
|
|
"logps/rejected": -0.2932647168636322,
|
|
"loss": 4448.4404296875,
|
|
"loss/core": 4448.4365234375,
|
|
"loss/preference_sft": 0.27584901452064514,
|
|
"loss/reference_anchor": 0.2003796100616455,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 1.6970628499984741,
|
|
"rewards/margins": -0.28227338194847107,
|
|
"rewards/rejected": 1.979336142539978,
|
|
"step": 875
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2957570552825928,
|
|
"drift/rejected_abs": 0.15153485536575317,
|
|
"epoch": 0.840797802460289,
|
|
"grad_norm": 1232.0,
|
|
"learning_rate": 4.143889944367429e-10,
|
|
"logits/chosen": -2.06880521774292,
|
|
"logits/rejected": -2.0875306129455566,
|
|
"logps/chosen": -0.30057868361473083,
|
|
"logps/rejected": -0.2992440164089203,
|
|
"loss": 4425.47802734375,
|
|
"loss/core": 4425.4697265625,
|
|
"loss/preference_sft": 0.30057868361473083,
|
|
"loss/reference_anchor": 0.42805910110473633,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.956110954284668,
|
|
"rewards/margins": 1.4640635251998901,
|
|
"rewards/rejected": 1.4920471906661987,
|
|
"step": 880
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.11761297285556793,
|
|
"drift/rejected_abs": 0.054164666682481766,
|
|
"epoch": 0.8455750627015407,
|
|
"grad_norm": 876.0,
|
|
"learning_rate": 2.406081393722531e-10,
|
|
"logits/chosen": -2.1882359981536865,
|
|
"logits/rejected": -2.2277369499206543,
|
|
"logps/chosen": -0.30893659591674805,
|
|
"logps/rejected": -0.300253301858902,
|
|
"loss": 4435.9912109375,
|
|
"loss/core": 4435.990234375,
|
|
"loss/preference_sft": 0.30893659591674805,
|
|
"loss/reference_anchor": 0.027945518493652344,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.1728156805038452,
|
|
"rewards/margins": 0.6360281109809875,
|
|
"rewards/rejected": 0.5367876291275024,
|
|
"step": 885
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20437291264533997,
|
|
"drift/rejected_abs": 0.07434792816638947,
|
|
"epoch": 0.8503523229427923,
|
|
"grad_norm": 4288.0,
|
|
"learning_rate": 1.1374418930135133e-10,
|
|
"logits/chosen": -2.072187662124634,
|
|
"logits/rejected": -2.139744997024536,
|
|
"logps/chosen": -0.33536940813064575,
|
|
"logps/rejected": -0.32796454429626465,
|
|
"loss": 4427.37646484375,
|
|
"loss/core": 4427.3720703125,
|
|
"loss/preference_sft": 0.33536940813064575,
|
|
"loss/reference_anchor": 0.1952354460954666,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.042240619659424,
|
|
"rewards/margins": 1.299561619758606,
|
|
"rewards/rejected": 0.7426789402961731,
|
|
"step": 890
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2200593650341034,
|
|
"drift/rejected_abs": 0.10724915564060211,
|
|
"epoch": 0.855129583184044,
|
|
"grad_norm": 1336.0,
|
|
"learning_rate": 3.3844852567285756e-11,
|
|
"logits/chosen": -2.085082530975342,
|
|
"logits/rejected": -2.0716962814331055,
|
|
"logps/chosen": -0.28429746627807617,
|
|
"logps/rejected": -0.27482885122299194,
|
|
"loss": 4429.484375,
|
|
"loss/core": 4429.48046875,
|
|
"loss/preference_sft": 0.28429746627807617,
|
|
"loss/reference_anchor": 0.15885330736637115,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.200329542160034,
|
|
"rewards/margins": 1.1366140842437744,
|
|
"rewards/rejected": 1.0637153387069702,
|
|
"step": 895
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4483451843261719,
|
|
"drift/rejected_abs": 0.1666850745677948,
|
|
"epoch": 0.8599068434252956,
|
|
"grad_norm": 2816.0,
|
|
"learning_rate": 9.401760429905703e-13,
|
|
"logits/chosen": -2.159799814224243,
|
|
"logits/rejected": -2.1224822998046875,
|
|
"logps/chosen": -0.3090743124485016,
|
|
"logps/rejected": -0.3085591197013855,
|
|
"loss": 4408.07568359375,
|
|
"loss/core": 4408.0546875,
|
|
"loss/preference_sft": 0.3090743124485016,
|
|
"loss/reference_anchor": 1.011836051940918,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.482909202575684,
|
|
"rewards/margins": 2.827746629714966,
|
|
"rewards/rejected": 1.6551628112792969,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.8599068434252956,
|
|
"step": 900,
|
|
"total_flos": 0.0,
|
|
"train_loss": 4429.094605034722,
|
|
"train_runtime": 7055.131,
|
|
"train_samples_per_second": 2.041,
|
|
"train_steps_per_second": 0.128
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 900,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 900,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|