3644 lines
136 KiB
JSON
3644 lines
136 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.8599068434252956,
|
|
"eval_steps": 500,
|
|
"global_step": 900,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"drift/chosen_abs": 0.24220439791679382,
|
|
"drift/rejected_abs": 0.09696104377508163,
|
|
"epoch": 0.004777260241251642,
|
|
"grad_norm": 0.9375,
|
|
"learning_rate": 1.4814814814814814e-08,
|
|
"logits/chosen": -1.9986450672149658,
|
|
"logits/rejected": -2.0149338245391846,
|
|
"logps/chosen": -0.2793148458003998,
|
|
"logps/rejected": -0.2823529839515686,
|
|
"loss": 0.6970014572143555,
|
|
"loss/core": 0.6860443949699402,
|
|
"loss/preference_sft": 0.2793148458003998,
|
|
"loss/reference_anchor": 0.191210076212883,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.420919179916382,
|
|
"rewards/margins": 1.4643528461456299,
|
|
"rewards/rejected": 0.9565665125846863,
|
|
"step": 5
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16979221999645233,
|
|
"drift/rejected_abs": 0.15968844294548035,
|
|
"epoch": 0.009554520482503284,
|
|
"grad_norm": 28.875,
|
|
"learning_rate": 3.3333333333333334e-08,
|
|
"logits/chosen": -2.343964099884033,
|
|
"logits/rejected": -2.3917829990386963,
|
|
"logps/chosen": -0.2886165380477905,
|
|
"logps/rejected": -0.2906612753868103,
|
|
"loss": 0.719261646270752,
|
|
"loss/core": 0.6927183866500854,
|
|
"loss/preference_sft": 0.2886165380477905,
|
|
"loss/reference_anchor": 0.5020042657852173,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.6975072622299194,
|
|
"rewards/margins": 0.10625555366277695,
|
|
"rewards/rejected": 1.5912517309188843,
|
|
"step": 10
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21399614214897156,
|
|
"drift/rejected_abs": 0.11550233513116837,
|
|
"epoch": 0.014331780723754926,
|
|
"grad_norm": 2.125,
|
|
"learning_rate": 5.1851851851851846e-08,
|
|
"logits/chosen": -2.2301273345947266,
|
|
"logits/rejected": -2.1927645206451416,
|
|
"logps/chosen": -0.2770230770111084,
|
|
"logps/rejected": -0.2827666401863098,
|
|
"loss": 0.6988939642906189,
|
|
"loss/core": 0.6884175539016724,
|
|
"loss/preference_sft": 0.2770230770111084,
|
|
"loss/reference_anchor": 0.1818254441022873,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.138073682785034,
|
|
"rewards/margins": 0.9850578308105469,
|
|
"rewards/rejected": 1.1530158519744873,
|
|
"step": 15
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19165699183940887,
|
|
"drift/rejected_abs": 0.08792886883020401,
|
|
"epoch": 0.01910904096500657,
|
|
"grad_norm": 0.29296875,
|
|
"learning_rate": 7.037037037037038e-08,
|
|
"logits/chosen": -2.1778664588928223,
|
|
"logits/rejected": -2.2186977863311768,
|
|
"logps/chosen": -0.30127808451652527,
|
|
"logps/rejected": -0.2866690754890442,
|
|
"loss": 0.7005060315132141,
|
|
"loss/core": 0.6881817579269409,
|
|
"loss/preference_sft": 0.30127808451652527,
|
|
"loss/reference_anchor": 0.2163558453321457,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.9165700674057007,
|
|
"rewards/margins": 1.038002610206604,
|
|
"rewards/rejected": 0.8785673379898071,
|
|
"step": 20
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3831271529197693,
|
|
"drift/rejected_abs": 0.1648741066455841,
|
|
"epoch": 0.02388630120625821,
|
|
"grad_norm": 0.671875,
|
|
"learning_rate": 8.888888888888888e-08,
|
|
"logits/chosen": -2.076991558074951,
|
|
"logits/rejected": -2.1371612548828125,
|
|
"logps/chosen": -0.2725714147090912,
|
|
"logps/rejected": -0.25773054361343384,
|
|
"loss": 0.7282012104988098,
|
|
"loss/core": 0.6838706135749817,
|
|
"loss/preference_sft": 0.2725714147090912,
|
|
"loss/reference_anchor": 0.8593538999557495,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.8294517993927,
|
|
"rewards/margins": 2.182237148284912,
|
|
"rewards/rejected": 1.6472148895263672,
|
|
"step": 25
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22680720686912537,
|
|
"drift/rejected_abs": 0.11056505143642426,
|
|
"epoch": 0.028663561447509853,
|
|
"grad_norm": 5.25,
|
|
"learning_rate": 1.074074074074074e-07,
|
|
"logits/chosen": -2.348992347717285,
|
|
"logits/rejected": -2.3324456214904785,
|
|
"logps/chosen": -0.3011496067047119,
|
|
"logps/rejected": -0.32406991720199585,
|
|
"loss": 0.7054672837257385,
|
|
"loss/core": 0.6874420642852783,
|
|
"loss/preference_sft": 0.3011496067047119,
|
|
"loss/reference_anchor": 0.3303898274898529,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.26779842376709,
|
|
"rewards/margins": 1.2391281127929688,
|
|
"rewards/rejected": 1.028670310974121,
|
|
"step": 30
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2659522593021393,
|
|
"drift/rejected_abs": 0.11598372459411621,
|
|
"epoch": 0.033440821688761495,
|
|
"grad_norm": 0.251953125,
|
|
"learning_rate": 1.2592592592592592e-07,
|
|
"logits/chosen": -2.201605796813965,
|
|
"logits/rejected": -2.18267822265625,
|
|
"logps/chosen": -0.26762351393699646,
|
|
"logps/rejected": -0.2652774751186371,
|
|
"loss": 0.7072523236274719,
|
|
"loss/core": 0.6861520409584045,
|
|
"loss/preference_sft": 0.26762351393699646,
|
|
"loss/reference_anchor": 0.395240843296051,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.659522771835327,
|
|
"rewards/margins": 1.5000025033950806,
|
|
"rewards/rejected": 1.1595203876495361,
|
|
"step": 35
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.44708338379859924,
|
|
"drift/rejected_abs": 0.1636328548192978,
|
|
"epoch": 0.03821808193001314,
|
|
"grad_norm": 2.546875,
|
|
"learning_rate": 1.4444444444444442e-07,
|
|
"logits/chosen": -2.0019490718841553,
|
|
"logits/rejected": -2.062816858291626,
|
|
"logps/chosen": -0.29337409138679504,
|
|
"logps/rejected": -0.2815210223197937,
|
|
"loss": 0.7314882874488831,
|
|
"loss/core": 0.6806979775428772,
|
|
"loss/preference_sft": 0.29337409138679504,
|
|
"loss/reference_anchor": 0.9864678382873535,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.470605373382568,
|
|
"rewards/margins": 2.8643717765808105,
|
|
"rewards/rejected": 1.6062335968017578,
|
|
"step": 40
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20861148834228516,
|
|
"drift/rejected_abs": 0.08526524901390076,
|
|
"epoch": 0.04299534217126478,
|
|
"grad_norm": 0.31640625,
|
|
"learning_rate": 1.6296296296296298e-07,
|
|
"logits/chosen": -2.199146270751953,
|
|
"logits/rejected": -2.3609375953674316,
|
|
"logps/chosen": -0.28975796699523926,
|
|
"logps/rejected": -0.2888508439064026,
|
|
"loss": 0.6954045295715332,
|
|
"loss/core": 0.6870486736297607,
|
|
"loss/preference_sft": 0.28975796699523926,
|
|
"loss/reference_anchor": 0.1381419450044632,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.085977792739868,
|
|
"rewards/margins": 1.248346209526062,
|
|
"rewards/rejected": 0.8376315236091614,
|
|
"step": 45
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28775015473365784,
|
|
"drift/rejected_abs": 0.1457628458738327,
|
|
"epoch": 0.04777260241251642,
|
|
"grad_norm": 3.75,
|
|
"learning_rate": 1.8148148148148149e-07,
|
|
"logits/chosen": -2.168269395828247,
|
|
"logits/rejected": -2.164731979370117,
|
|
"logps/chosen": -0.28189998865127563,
|
|
"logps/rejected": -0.28475481271743774,
|
|
"loss": 0.7057908177375793,
|
|
"loss/core": 0.6865909695625305,
|
|
"loss/preference_sft": 0.28189998865127563,
|
|
"loss/reference_anchor": 0.3558071553707123,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.8775012493133545,
|
|
"rewards/margins": 1.4253501892089844,
|
|
"rewards/rejected": 1.4521510601043701,
|
|
"step": 50
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22688527405261993,
|
|
"drift/rejected_abs": 0.2027493268251419,
|
|
"epoch": 0.05254986265376806,
|
|
"grad_norm": 0.162109375,
|
|
"learning_rate": 2e-07,
|
|
"logits/chosen": -2.196228504180908,
|
|
"logits/rejected": -2.184458017349243,
|
|
"logps/chosen": -0.28806787729263306,
|
|
"logps/rejected": -0.29169613122940063,
|
|
"loss": 0.7314178347587585,
|
|
"loss/core": 0.6928032636642456,
|
|
"loss/preference_sft": 0.28806787729263306,
|
|
"loss/reference_anchor": 0.7434841394424438,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.2677197456359863,
|
|
"rewards/margins": 0.25824469327926636,
|
|
"rewards/rejected": 2.009474992752075,
|
|
"step": 55
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2152278870344162,
|
|
"drift/rejected_abs": 0.1564132422208786,
|
|
"epoch": 0.057327122895019705,
|
|
"grad_norm": 20.75,
|
|
"learning_rate": 2.1851851851851852e-07,
|
|
"logits/chosen": -2.091794013977051,
|
|
"logits/rejected": -2.1202189922332764,
|
|
"logps/chosen": -0.27512186765670776,
|
|
"logps/rejected": -0.2831602692604065,
|
|
"loss": 0.7002712488174438,
|
|
"loss/core": 0.6903932690620422,
|
|
"loss/preference_sft": 0.27512186765670776,
|
|
"loss/reference_anchor": 0.17004787921905518,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.149622917175293,
|
|
"rewards/margins": 0.5874365568161011,
|
|
"rewards/rejected": 1.5621862411499023,
|
|
"step": 60
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2973586320877075,
|
|
"drift/rejected_abs": 0.11919713020324707,
|
|
"epoch": 0.06210438313627135,
|
|
"grad_norm": 0.33984375,
|
|
"learning_rate": 2.3703703703703703e-07,
|
|
"logits/chosen": -1.9600942134857178,
|
|
"logits/rejected": -2.063352108001709,
|
|
"logps/chosen": -0.30903658270835876,
|
|
"logps/rejected": -0.3050587773323059,
|
|
"loss": 0.7107466459274292,
|
|
"loss/core": 0.6846051812171936,
|
|
"loss/preference_sft": 0.30903658270835876,
|
|
"loss/reference_anchor": 0.49192652106285095,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.9714887142181396,
|
|
"rewards/margins": 1.7871677875518799,
|
|
"rewards/rejected": 1.1843211650848389,
|
|
"step": 65
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.36345982551574707,
|
|
"drift/rejected_abs": 0.22663752734661102,
|
|
"epoch": 0.06688164337752299,
|
|
"grad_norm": 1.703125,
|
|
"learning_rate": 2.5555555555555553e-07,
|
|
"logits/chosen": -1.971685767173767,
|
|
"logits/rejected": -1.9568895101547241,
|
|
"logps/chosen": -0.3803972005844116,
|
|
"logps/rejected": -0.2971033453941345,
|
|
"loss": 0.7371006011962891,
|
|
"loss/core": 0.6875497698783875,
|
|
"loss/preference_sft": 0.3803972005844116,
|
|
"loss/reference_anchor": 0.9529773592948914,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.63312029838562,
|
|
"rewards/margins": 1.450933575630188,
|
|
"rewards/rejected": 2.1821866035461426,
|
|
"step": 70
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2583789527416229,
|
|
"drift/rejected_abs": 0.15998633205890656,
|
|
"epoch": 0.07165890361877464,
|
|
"grad_norm": 0.44140625,
|
|
"learning_rate": 2.7407407407407406e-07,
|
|
"logits/chosen": -2.086479663848877,
|
|
"logits/rejected": -2.0509562492370605,
|
|
"logps/chosen": -0.31756237149238586,
|
|
"logps/rejected": -0.2885989248752594,
|
|
"loss": 0.7078580856323242,
|
|
"loss/core": 0.688626229763031,
|
|
"loss/preference_sft": 0.31756237149238586,
|
|
"loss/reference_anchor": 0.35288190841674805,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.5812525749206543,
|
|
"rewards/margins": 0.9826623201370239,
|
|
"rewards/rejected": 1.5985901355743408,
|
|
"step": 75
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3245569169521332,
|
|
"drift/rejected_abs": 0.127544105052948,
|
|
"epoch": 0.07643616386002627,
|
|
"grad_norm": 0.29296875,
|
|
"learning_rate": 2.9259259259259254e-07,
|
|
"logits/chosen": -2.0680508613586426,
|
|
"logits/rejected": -2.1884958744049072,
|
|
"logps/chosen": -0.2601710259914398,
|
|
"logps/rejected": -0.274718701839447,
|
|
"loss": 0.7082482576370239,
|
|
"loss/core": 0.6839421987533569,
|
|
"loss/preference_sft": 0.2601710259914398,
|
|
"loss/reference_anchor": 0.46010416746139526,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.245285749435425,
|
|
"rewards/margins": 2.0003161430358887,
|
|
"rewards/rejected": 1.2449694871902466,
|
|
"step": 80
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.34736722707748413,
|
|
"drift/rejected_abs": 0.1335596740245819,
|
|
"epoch": 0.08121342410127792,
|
|
"grad_norm": 0.423828125,
|
|
"learning_rate": 3.111111111111111e-07,
|
|
"logits/chosen": -2.187932252883911,
|
|
"logits/rejected": -2.2362093925476074,
|
|
"logps/chosen": -0.31001752614974976,
|
|
"logps/rejected": -0.27728381752967834,
|
|
"loss": 0.7267902493476868,
|
|
"loss/core": 0.683385968208313,
|
|
"loss/preference_sft": 0.31001752614974976,
|
|
"loss/reference_anchor": 0.8370832204818726,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.4731335639953613,
|
|
"rewards/margins": 2.138108015060425,
|
|
"rewards/rejected": 1.3350250720977783,
|
|
"step": 85
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1755516231060028,
|
|
"drift/rejected_abs": 0.08407329767942429,
|
|
"epoch": 0.08599068434252956,
|
|
"grad_norm": 0.162109375,
|
|
"learning_rate": 3.296296296296296e-07,
|
|
"logits/chosen": -2.420429229736328,
|
|
"logits/rejected": -2.4072763919830322,
|
|
"logps/chosen": -0.28123074769973755,
|
|
"logps/rejected": -0.2965225577354431,
|
|
"loss": 0.6978365778923035,
|
|
"loss/core": 0.6886924505233765,
|
|
"loss/preference_sft": 0.28123074769973755,
|
|
"loss/reference_anchor": 0.15476098656654358,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.7550804615020752,
|
|
"rewards/margins": 0.9150702357292175,
|
|
"rewards/rejected": 0.8400100469589233,
|
|
"step": 90
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3212953805923462,
|
|
"drift/rejected_abs": 0.21096964180469513,
|
|
"epoch": 0.09076794458378121,
|
|
"grad_norm": 0.94140625,
|
|
"learning_rate": 3.4814814814814814e-07,
|
|
"logits/chosen": -2.0551528930664062,
|
|
"logits/rejected": -2.111891269683838,
|
|
"logps/chosen": -0.2670920491218567,
|
|
"logps/rejected": -0.278430700302124,
|
|
"loss": 0.7121036648750305,
|
|
"loss/core": 0.6880958080291748,
|
|
"loss/preference_sft": 0.2670920491218567,
|
|
"loss/reference_anchor": 0.4534483850002289,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.212806224822998,
|
|
"rewards/margins": 1.103410005569458,
|
|
"rewards/rejected": 2.109395980834961,
|
|
"step": 95
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24620285630226135,
|
|
"drift/rejected_abs": 0.1283082365989685,
|
|
"epoch": 0.09554520482503284,
|
|
"grad_norm": 0.39453125,
|
|
"learning_rate": 3.666666666666666e-07,
|
|
"logits/chosen": -2.2331910133361816,
|
|
"logits/rejected": -2.278862953186035,
|
|
"logps/chosen": -0.2573774755001068,
|
|
"logps/rejected": -0.2470593899488449,
|
|
"loss": 0.6977821588516235,
|
|
"loss/core": 0.6874433755874634,
|
|
"loss/preference_sft": 0.2573774755001068,
|
|
"loss/reference_anchor": 0.18103909492492676,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.460810422897339,
|
|
"rewards/margins": 1.1784250736236572,
|
|
"rewards/rejected": 1.2823854684829712,
|
|
"step": 100
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20025713741779327,
|
|
"drift/rejected_abs": 0.07980473339557648,
|
|
"epoch": 0.10032246506628449,
|
|
"grad_norm": 5.8125,
|
|
"learning_rate": 3.8518518518518515e-07,
|
|
"logits/chosen": -2.33634614944458,
|
|
"logits/rejected": -2.421203374862671,
|
|
"logps/chosen": -0.2838136553764343,
|
|
"logps/rejected": -0.28014278411865234,
|
|
"loss": 0.6988551020622253,
|
|
"loss/core": 0.6874551773071289,
|
|
"loss/preference_sft": 0.2838136553764343,
|
|
"loss/reference_anchor": 0.1996181458234787,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.001106023788452,
|
|
"rewards/margins": 1.206559419631958,
|
|
"rewards/rejected": 0.7945466041564941,
|
|
"step": 105
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.44312191009521484,
|
|
"drift/rejected_abs": 0.18942782282829285,
|
|
"epoch": 0.10509972530753613,
|
|
"grad_norm": 0.4921875,
|
|
"learning_rate": 4.0370370370370373e-07,
|
|
"logits/chosen": -1.9062073230743408,
|
|
"logits/rejected": -1.858873724937439,
|
|
"logps/chosen": -0.34249261021614075,
|
|
"logps/rejected": -0.29758021235466003,
|
|
"loss": 0.7256662249565125,
|
|
"loss/core": 0.6815618276596069,
|
|
"loss/preference_sft": 0.34249261021614075,
|
|
"loss/reference_anchor": 0.8478387594223022,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 4.430351734161377,
|
|
"rewards/margins": 2.5402467250823975,
|
|
"rewards/rejected": 1.89010488986969,
|
|
"step": 110
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17720595002174377,
|
|
"drift/rejected_abs": 0.14460505545139313,
|
|
"epoch": 0.10987698554878778,
|
|
"grad_norm": 2.0625,
|
|
"learning_rate": 4.222222222222222e-07,
|
|
"logits/chosen": -2.087794542312622,
|
|
"logits/rejected": -2.0313782691955566,
|
|
"logps/chosen": -0.30076345801353455,
|
|
"logps/rejected": -0.2990095615386963,
|
|
"loss": 0.7037674188613892,
|
|
"loss/core": 0.6917927861213684,
|
|
"loss/preference_sft": 0.30076345801353455,
|
|
"loss/reference_anchor": 0.20941567420959473,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.771735429763794,
|
|
"rewards/margins": 0.32677847146987915,
|
|
"rewards/rejected": 1.4449571371078491,
|
|
"step": 115
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1807553470134735,
|
|
"drift/rejected_abs": 0.11256897449493408,
|
|
"epoch": 0.11465424579003941,
|
|
"grad_norm": 7.9375,
|
|
"learning_rate": 4.4074074074074074e-07,
|
|
"logits/chosen": -2.038364887237549,
|
|
"logits/rejected": -2.074918270111084,
|
|
"logps/chosen": -0.3024388551712036,
|
|
"logps/rejected": -0.2960285246372223,
|
|
"loss": 0.6956692337989807,
|
|
"loss/core": 0.6897962689399719,
|
|
"loss/preference_sft": 0.3024388551712036,
|
|
"loss/reference_anchor": 0.08721615374088287,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.8071529865264893,
|
|
"rewards/margins": 0.6840036511421204,
|
|
"rewards/rejected": 1.1231491565704346,
|
|
"step": 120
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2066231518983841,
|
|
"drift/rejected_abs": 0.10167888551950455,
|
|
"epoch": 0.11943150603129106,
|
|
"grad_norm": 0.2734375,
|
|
"learning_rate": 4.592592592592592e-07,
|
|
"logits/chosen": -2.1104722023010254,
|
|
"logits/rejected": -2.141206979751587,
|
|
"logps/chosen": -0.29907721281051636,
|
|
"logps/rejected": -0.30693620443344116,
|
|
"loss": 0.6962321400642395,
|
|
"loss/core": 0.6880409717559814,
|
|
"loss/preference_sft": 0.29907721281051636,
|
|
"loss/reference_anchor": 0.13391605019569397,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.0647101402282715,
|
|
"rewards/margins": 1.0507093667984009,
|
|
"rewards/rejected": 1.014000654220581,
|
|
"step": 125
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18080219626426697,
|
|
"drift/rejected_abs": 0.0888272374868393,
|
|
"epoch": 0.1242087662725427,
|
|
"grad_norm": 0.5703125,
|
|
"learning_rate": 4.777777777777778e-07,
|
|
"logits/chosen": -1.999070405960083,
|
|
"logits/rejected": -2.092583179473877,
|
|
"logps/chosen": -0.27993208169937134,
|
|
"logps/rejected": -0.2788156569004059,
|
|
"loss": 0.6937012076377869,
|
|
"loss/core": 0.68858802318573,
|
|
"loss/preference_sft": 0.27993208169937134,
|
|
"loss/reference_anchor": 0.07427060604095459,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.8078792095184326,
|
|
"rewards/margins": 0.9212325811386108,
|
|
"rewards/rejected": 0.8866468667984009,
|
|
"step": 130
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3372951149940491,
|
|
"drift/rejected_abs": 0.13560067117214203,
|
|
"epoch": 0.12898602651379434,
|
|
"grad_norm": 0.765625,
|
|
"learning_rate": 4.962962962962963e-07,
|
|
"logits/chosen": -2.0918431282043457,
|
|
"logits/rejected": -2.1655802726745605,
|
|
"logps/chosen": -0.30224984884262085,
|
|
"logps/rejected": -0.29795363545417786,
|
|
"loss": 0.7072810530662537,
|
|
"loss/core": 0.6833052635192871,
|
|
"loss/preference_sft": 0.30224984884262085,
|
|
"loss/reference_anchor": 0.44929036498069763,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.3726329803466797,
|
|
"rewards/margins": 2.055931806564331,
|
|
"rewards/rejected": 1.3167011737823486,
|
|
"step": 135
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16283419728279114,
|
|
"drift/rejected_abs": 0.07167728990316391,
|
|
"epoch": 0.13376328675504598,
|
|
"grad_norm": 0.294921875,
|
|
"learning_rate": 4.999662714939607e-07,
|
|
"logits/chosen": -2.4538674354553223,
|
|
"logits/rejected": -2.5128872394561768,
|
|
"logps/chosen": -0.2825615108013153,
|
|
"logps/rejected": -0.28934210538864136,
|
|
"loss": 0.6929519176483154,
|
|
"loss/core": 0.688643753528595,
|
|
"loss/preference_sft": 0.2825615108013153,
|
|
"loss/reference_anchor": 0.057906895875930786,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.6276466846466064,
|
|
"rewards/margins": 0.9115911722183228,
|
|
"rewards/rejected": 0.7160555720329285,
|
|
"step": 140
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3645097315311432,
|
|
"drift/rejected_abs": 0.17297443747520447,
|
|
"epoch": 0.13854054699629761,
|
|
"grad_norm": 0.53125,
|
|
"learning_rate": 4.998292650357557e-07,
|
|
"logits/chosen": -1.7773208618164062,
|
|
"logits/rejected": -1.8101383447647095,
|
|
"logps/chosen": -0.29743704199790955,
|
|
"logps/rejected": -0.3080594837665558,
|
|
"loss": 0.7377547025680542,
|
|
"loss/core": 0.6842484474182129,
|
|
"loss/preference_sft": 0.29743704199790955,
|
|
"loss/reference_anchor": 1.0403814315795898,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.643198013305664,
|
|
"rewards/margins": 1.9210885763168335,
|
|
"rewards/rejected": 1.7221091985702515,
|
|
"step": 145
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3650224804878235,
|
|
"drift/rejected_abs": 0.14471963047981262,
|
|
"epoch": 0.14331780723754928,
|
|
"grad_norm": 2.484375,
|
|
"learning_rate": 4.995869303113768e-07,
|
|
"logits/chosen": -2.234192371368408,
|
|
"logits/rejected": -2.2828991413116455,
|
|
"logps/chosen": -0.31630638241767883,
|
|
"logps/rejected": -0.31051549315452576,
|
|
"loss": 0.7256408333778381,
|
|
"loss/core": 0.682770311832428,
|
|
"loss/preference_sft": 0.31630638241767883,
|
|
"loss/reference_anchor": 0.8257803916931152,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.649608612060547,
|
|
"rewards/margins": 2.215174913406372,
|
|
"rewards/rejected": 1.4344332218170166,
|
|
"step": 150
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2868257462978363,
|
|
"drift/rejected_abs": 0.0814853385090828,
|
|
"epoch": 0.1480950674788009,
|
|
"grad_norm": 0.455078125,
|
|
"learning_rate": 4.992393694893844e-07,
|
|
"logits/chosen": -2.098144292831421,
|
|
"logits/rejected": -2.1760802268981934,
|
|
"logps/chosen": -0.28282657265663147,
|
|
"logps/rejected": -0.29578837752342224,
|
|
"loss": 0.7028457522392273,
|
|
"loss/core": 0.6833733320236206,
|
|
"loss/preference_sft": 0.28282657265663147,
|
|
"loss/reference_anchor": 0.36116594076156616,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.8672358989715576,
|
|
"rewards/margins": 2.075387716293335,
|
|
"rewards/rejected": 0.7918483018875122,
|
|
"step": 155
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3154504895210266,
|
|
"drift/rejected_abs": 0.14014051854610443,
|
|
"epoch": 0.15287232772005255,
|
|
"grad_norm": 6.78125,
|
|
"learning_rate": 4.987867291017661e-07,
|
|
"logits/chosen": -2.05576491355896,
|
|
"logits/rejected": -2.029587507247925,
|
|
"logps/chosen": -0.310093492269516,
|
|
"logps/rejected": -0.31532084941864014,
|
|
"loss": 0.7215666174888611,
|
|
"loss/core": 0.6853370070457458,
|
|
"loss/preference_sft": 0.310093492269516,
|
|
"loss/reference_anchor": 0.6935839653015137,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.1545047760009766,
|
|
"rewards/margins": 1.7532867193222046,
|
|
"rewards/rejected": 1.401218056678772,
|
|
"step": 160
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16013780236244202,
|
|
"drift/rejected_abs": 0.07896161824464798,
|
|
"epoch": 0.15764958796130418,
|
|
"grad_norm": 0.48046875,
|
|
"learning_rate": 4.982291999821587e-07,
|
|
"logits/chosen": -2.2207119464874268,
|
|
"logits/rejected": -2.152768850326538,
|
|
"logps/chosen": -0.3112797141075134,
|
|
"logps/rejected": -0.3058459758758545,
|
|
"loss": 0.6951904892921448,
|
|
"loss/core": 0.6891487240791321,
|
|
"loss/preference_sft": 0.3112797141075134,
|
|
"loss/reference_anchor": 0.08970671147108078,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.6005077362060547,
|
|
"rewards/margins": 0.8129563331604004,
|
|
"rewards/rejected": 0.7875513434410095,
|
|
"step": 165
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2601718008518219,
|
|
"drift/rejected_abs": 0.27558404207229614,
|
|
"epoch": 0.16242684820255585,
|
|
"grad_norm": 7.875,
|
|
"learning_rate": 4.975670171853925e-07,
|
|
"logits/chosen": -2.1900925636291504,
|
|
"logits/rejected": -2.1342360973358154,
|
|
"logps/chosen": -0.27432286739349365,
|
|
"logps/rejected": -0.2613202929496765,
|
|
"loss": 0.7430840730667114,
|
|
"loss/core": 0.6951981782913208,
|
|
"loss/preference_sft": 0.27432286739349365,
|
|
"loss/reference_anchor": 0.9302865266799927,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.6010234355926514,
|
|
"rewards/margins": -0.15333333611488342,
|
|
"rewards/rejected": 2.754356861114502,
|
|
"step": 170
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2583249509334564,
|
|
"drift/rejected_abs": 0.12602022290229797,
|
|
"epoch": 0.16720410844380748,
|
|
"grad_norm": 13.4375,
|
|
"learning_rate": 4.968004598883922e-07,
|
|
"logits/chosen": -2.1797878742218018,
|
|
"logits/rejected": -2.2193121910095215,
|
|
"logps/chosen": -0.2869110703468323,
|
|
"logps/rejected": -0.301838755607605,
|
|
"loss": 0.7159601449966431,
|
|
"loss/core": 0.6867267489433289,
|
|
"loss/preference_sft": 0.2869110703468323,
|
|
"loss/reference_anchor": 0.5559752583503723,
|
|
"rewards/accuracies": 0.987500011920929,
|
|
"rewards/chosen": 2.583249568939209,
|
|
"rewards/margins": 1.3249006271362305,
|
|
"rewards/rejected": 1.2583489418029785,
|
|
"step": 175
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3220704197883606,
|
|
"drift/rejected_abs": 0.15029826760292053,
|
|
"epoch": 0.17198136868505912,
|
|
"grad_norm": 15.625,
|
|
"learning_rate": 4.959298512724752e-07,
|
|
"logits/chosen": -2.19909930229187,
|
|
"logits/rejected": -2.1092610359191895,
|
|
"logps/chosen": -0.27359136939048767,
|
|
"logps/rejected": -0.2609673738479614,
|
|
"loss": 0.7153160572052002,
|
|
"loss/core": 0.6853613257408142,
|
|
"loss/preference_sft": 0.27359136939048767,
|
|
"loss/reference_anchor": 0.5717340707778931,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.2184219360351562,
|
|
"rewards/margins": 1.718634009361267,
|
|
"rewards/rejected": 1.49978768825531,
|
|
"step": 180
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.10404728353023529,
|
|
"drift/rejected_abs": 0.0626899003982544,
|
|
"epoch": 0.17675862892631075,
|
|
"grad_norm": 0.314453125,
|
|
"learning_rate": 4.949555583870983e-07,
|
|
"logits/chosen": -2.204803466796875,
|
|
"logits/rejected": -2.1623644828796387,
|
|
"logps/chosen": -0.28916627168655396,
|
|
"logps/rejected": -0.2922504246234894,
|
|
"loss": 0.69410240650177,
|
|
"loss/core": 0.691106915473938,
|
|
"loss/preference_sft": 0.28916627168655396,
|
|
"loss/reference_anchor": 0.03099246881902218,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.0380046367645264,
|
|
"rewards/margins": 0.4127095639705658,
|
|
"rewards/rejected": 0.625295102596283,
|
|
"step": 185
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28229960799217224,
|
|
"drift/rejected_abs": 0.12531210482120514,
|
|
"epoch": 0.18153588916756241,
|
|
"grad_norm": 0.9375,
|
|
"learning_rate": 4.938779919951092e-07,
|
|
"logits/chosen": -2.2970175743103027,
|
|
"logits/rejected": -2.2739336490631104,
|
|
"logps/chosen": -0.2851516008377075,
|
|
"logps/rejected": -0.2831699550151825,
|
|
"loss": 0.7151316404342651,
|
|
"loss/core": 0.6858740448951721,
|
|
"loss/preference_sft": 0.2851516008377075,
|
|
"loss/reference_anchor": 0.5566385984420776,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.822661876678467,
|
|
"rewards/margins": 1.5726929903030396,
|
|
"rewards/rejected": 1.2499691247940063,
|
|
"step": 190
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26670801639556885,
|
|
"drift/rejected_abs": 0.097129687666893,
|
|
"epoch": 0.18631314940881405,
|
|
"grad_norm": 0.337890625,
|
|
"learning_rate": 4.926976063995686e-07,
|
|
"logits/chosen": -1.9850791692733765,
|
|
"logits/rejected": -2.0053915977478027,
|
|
"logps/chosen": -0.2911596894264221,
|
|
"logps/rejected": -0.2897758185863495,
|
|
"loss": 0.7147169709205627,
|
|
"loss/core": 0.6856262683868408,
|
|
"loss/preference_sft": 0.2911596894264221,
|
|
"loss/reference_anchor": 0.5526987314224243,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.6668624877929688,
|
|
"rewards/margins": 1.7010908126831055,
|
|
"rewards/rejected": 0.9657715559005737,
|
|
"step": 195
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26155006885528564,
|
|
"drift/rejected_abs": 0.12203337997198105,
|
|
"epoch": 0.19109040965006568,
|
|
"grad_norm": 0.69140625,
|
|
"learning_rate": 4.914148992522156e-07,
|
|
"logits/chosen": -1.8971683979034424,
|
|
"logits/rejected": -1.9370143413543701,
|
|
"logps/chosen": -0.3235601484775543,
|
|
"logps/rejected": -0.3167949318885803,
|
|
"loss": 0.7003489136695862,
|
|
"loss/core": 0.6863821744918823,
|
|
"loss/preference_sft": 0.3235601484775543,
|
|
"loss/reference_anchor": 0.24697670340538025,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.6145803928375244,
|
|
"rewards/margins": 1.3957717418670654,
|
|
"rewards/rejected": 1.2188085317611694,
|
|
"step": 200
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3286622166633606,
|
|
"drift/rejected_abs": 0.2124643623828888,
|
|
"epoch": 0.19586766989131732,
|
|
"grad_norm": 0.23046875,
|
|
"learning_rate": 4.90030411343657e-07,
|
|
"logits/chosen": -2.119131565093994,
|
|
"logits/rejected": -2.1706948280334473,
|
|
"logps/chosen": -0.2504412829875946,
|
|
"logps/rejected": -0.2682013511657715,
|
|
"loss": 0.7217109799385071,
|
|
"loss/core": 0.6881678104400635,
|
|
"loss/preference_sft": 0.2504412829875946,
|
|
"loss/reference_anchor": 0.6458190679550171,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.2861106395721436,
|
|
"rewards/margins": 1.170446753501892,
|
|
"rewards/rejected": 2.115664005279541,
|
|
"step": 205
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2844642698764801,
|
|
"drift/rejected_abs": 0.15010371804237366,
|
|
"epoch": 0.20064493013256898,
|
|
"grad_norm": 0.447265625,
|
|
"learning_rate": 4.885447263753696e-07,
|
|
"logits/chosen": -2.1142611503601074,
|
|
"logits/rejected": -2.138841152191162,
|
|
"logps/chosen": -0.3113865256309509,
|
|
"logps/rejected": -0.32046282291412354,
|
|
"loss": 0.7174853086471558,
|
|
"loss/core": 0.6867169141769409,
|
|
"loss/preference_sft": 0.3113865256309509,
|
|
"loss/reference_anchor": 0.584227442741394,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.8422324657440186,
|
|
"rewards/margins": 1.3444972038269043,
|
|
"rewards/rejected": 1.4977353811264038,
|
|
"step": 210
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3206380307674408,
|
|
"drift/rejected_abs": 0.16881483793258667,
|
|
"epoch": 0.20542219037382062,
|
|
"grad_norm": 0.828125,
|
|
"learning_rate": 4.869584707136108e-07,
|
|
"logits/chosen": -2.0880212783813477,
|
|
"logits/rejected": -2.079676389694214,
|
|
"logps/chosen": -0.2742551863193512,
|
|
"logps/rejected": -0.28655797243118286,
|
|
"loss": 0.7276748418807983,
|
|
"loss/core": 0.6868326663970947,
|
|
"loss/preference_sft": 0.2742551863193512,
|
|
"loss/reference_anchor": 0.7894173860549927,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.203636884689331,
|
|
"rewards/margins": 1.5241820812225342,
|
|
"rewards/rejected": 1.6794548034667969,
|
|
"step": 215
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1884107142686844,
|
|
"drift/rejected_abs": 0.10744212567806244,
|
|
"epoch": 0.21019945061507225,
|
|
"grad_norm": 1.421875,
|
|
"learning_rate": 4.852723131253429e-07,
|
|
"logits/chosen": -2.1026856899261475,
|
|
"logits/rejected": -2.0164294242858887,
|
|
"logps/chosen": -0.3028913140296936,
|
|
"logps/rejected": -0.3013783395290375,
|
|
"loss": 0.6961045265197754,
|
|
"loss/core": 0.689160168170929,
|
|
"loss/preference_sft": 0.3028913140296936,
|
|
"loss/reference_anchor": 0.10859811305999756,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.881739616394043,
|
|
"rewards/margins": 0.8081823587417603,
|
|
"rewards/rejected": 1.0735574960708618,
|
|
"step": 220
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.36267831921577454,
|
|
"drift/rejected_abs": 0.14132729172706604,
|
|
"epoch": 0.2149767108563239,
|
|
"grad_norm": 0.35546875,
|
|
"learning_rate": 4.834869644962788e-07,
|
|
"logits/chosen": -2.1880686283111572,
|
|
"logits/rejected": -2.1765666007995605,
|
|
"logps/chosen": -0.29012617468833923,
|
|
"logps/rejected": -0.2754373252391815,
|
|
"loss": 0.7051090598106384,
|
|
"loss/core": 0.6825994253158569,
|
|
"loss/preference_sft": 0.29012617468833923,
|
|
"loss/reference_anchor": 0.42117825150489807,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.6267833709716797,
|
|
"rewards/margins": 2.217150926589966,
|
|
"rewards/rejected": 1.4096323251724243,
|
|
"step": 225
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18478451669216156,
|
|
"drift/rejected_abs": 0.07309754192829132,
|
|
"epoch": 0.21975397109757555,
|
|
"grad_norm": 5.34375,
|
|
"learning_rate": 4.816031775311732e-07,
|
|
"logits/chosen": -2.2757763862609863,
|
|
"logits/rejected": -2.2832772731781006,
|
|
"logps/chosen": -0.2824876308441162,
|
|
"logps/rejected": -0.31052327156066895,
|
|
"loss": 0.693640410900116,
|
|
"loss/core": 0.6876736283302307,
|
|
"loss/preference_sft": 0.2824876308441162,
|
|
"loss/reference_anchor": 0.0910874456167221,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.8467649221420288,
|
|
"rewards/margins": 1.1201399564743042,
|
|
"rewards/rejected": 0.7266249060630798,
|
|
"step": 230
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1901187002658844,
|
|
"drift/rejected_abs": 0.13333821296691895,
|
|
"epoch": 0.2245312313388272,
|
|
"grad_norm": 0.447265625,
|
|
"learning_rate": 4.796217464364808e-07,
|
|
"logits/chosen": -2.0125269889831543,
|
|
"logits/rejected": -2.0471303462982178,
|
|
"logps/chosen": -0.29326948523521423,
|
|
"logps/rejected": -0.2864275574684143,
|
|
"loss": 0.703149676322937,
|
|
"loss/core": 0.6904449462890625,
|
|
"loss/preference_sft": 0.29326948523521423,
|
|
"loss/reference_anchor": 0.2247694432735443,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.8984577655792236,
|
|
"rewards/margins": 0.5684404373168945,
|
|
"rewards/rejected": 1.330017328262329,
|
|
"step": 235
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17612558603286743,
|
|
"drift/rejected_abs": 0.10248030722141266,
|
|
"epoch": 0.22930849158007882,
|
|
"grad_norm": 0.49609375,
|
|
"learning_rate": 4.775435065855182e-07,
|
|
"logits/chosen": -2.0281519889831543,
|
|
"logits/rejected": -2.046496868133545,
|
|
"logps/chosen": -0.29431360960006714,
|
|
"logps/rejected": -0.2945192754268646,
|
|
"loss": 0.6945276260375977,
|
|
"loss/core": 0.6894940733909607,
|
|
"loss/preference_sft": 0.29431360960006714,
|
|
"loss/reference_anchor": 0.07123853266239166,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.760162353515625,
|
|
"rewards/margins": 0.7375157475471497,
|
|
"rewards/rejected": 1.0226467847824097,
|
|
"step": 240
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3607032895088196,
|
|
"drift/rejected_abs": 0.11152495443820953,
|
|
"epoch": 0.23408575182133046,
|
|
"grad_norm": 12.125,
|
|
"learning_rate": 4.753693341662701e-07,
|
|
"logits/chosen": -2.0682380199432373,
|
|
"logits/rejected": -2.056436538696289,
|
|
"logps/chosen": -0.2773371934890747,
|
|
"logps/rejected": -0.2957698106765747,
|
|
"loss": 0.7130360007286072,
|
|
"loss/core": 0.6818927526473999,
|
|
"loss/preference_sft": 0.2773371934890747,
|
|
"loss/reference_anchor": 0.5951310396194458,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.606236219406128,
|
|
"rewards/margins": 2.500974178314209,
|
|
"rewards/rejected": 1.105262279510498,
|
|
"step": 245
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2335798293352127,
|
|
"drift/rejected_abs": 0.10776785761117935,
|
|
"epoch": 0.23886301206258212,
|
|
"grad_norm": 0.58203125,
|
|
"learning_rate": 4.731001458119869e-07,
|
|
"logits/chosen": -1.9081608057022095,
|
|
"logits/rejected": -1.988088607788086,
|
|
"logps/chosen": -0.30037543177604675,
|
|
"logps/rejected": -0.2945137917995453,
|
|
"loss": 0.6957753896713257,
|
|
"loss/core": 0.6870092153549194,
|
|
"loss/preference_sft": 0.30037543177604675,
|
|
"loss/reference_anchor": 0.14528658986091614,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.333603620529175,
|
|
"rewards/margins": 1.2645257711410522,
|
|
"rewards/rejected": 1.069077730178833,
|
|
"step": 250
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2451537400484085,
|
|
"drift/rejected_abs": 0.16124595701694489,
|
|
"epoch": 0.24364027230383375,
|
|
"grad_norm": 0.447265625,
|
|
"learning_rate": 4.707368982147317e-07,
|
|
"logits/chosen": -2.2194371223449707,
|
|
"logits/rejected": -2.215038299560547,
|
|
"logps/chosen": -0.2850092053413391,
|
|
"logps/rejected": -0.2866383194923401,
|
|
"loss": 0.7060943841934204,
|
|
"loss/core": 0.689082145690918,
|
|
"loss/preference_sft": 0.2850092053413391,
|
|
"loss/reference_anchor": 0.3117434084415436,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.4506688117980957,
|
|
"rewards/margins": 0.8393514752388,
|
|
"rewards/rejected": 1.6113172769546509,
|
|
"step": 255
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2390270233154297,
|
|
"drift/rejected_abs": 0.15710371732711792,
|
|
"epoch": 0.2484175325450854,
|
|
"grad_norm": 0.18359375,
|
|
"learning_rate": 4.682805877220377e-07,
|
|
"logits/chosen": -2.1546220779418945,
|
|
"logits/rejected": -2.227632999420166,
|
|
"logps/chosen": -0.28877514600753784,
|
|
"logps/rejected": -0.2719945013523102,
|
|
"loss": 0.7041021585464478,
|
|
"loss/core": 0.6894248127937317,
|
|
"loss/preference_sft": 0.28877514600753784,
|
|
"loss/reference_anchor": 0.2646702826023102,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.3893117904663086,
|
|
"rewards/margins": 0.8218982815742493,
|
|
"rewards/rejected": 1.567413568496704,
|
|
"step": 260
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.12290436029434204,
|
|
"drift/rejected_abs": 0.11683180183172226,
|
|
"epoch": 0.25319479278633705,
|
|
"grad_norm": 0.1875,
|
|
"learning_rate": 4.657322499168474e-07,
|
|
"logits/chosen": -2.037142276763916,
|
|
"logits/rejected": -2.044814348220825,
|
|
"logps/chosen": -0.2997164726257324,
|
|
"logps/rejected": -0.3027000427246094,
|
|
"loss": 0.7055035829544067,
|
|
"loss/core": 0.6932748556137085,
|
|
"loss/preference_sft": 0.2997164726257324,
|
|
"loss/reference_anchor": 0.21460387110710144,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.2280583381652832,
|
|
"rewards/margins": 0.06538932025432587,
|
|
"rewards/rejected": 1.1626689434051514,
|
|
"step": 265
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.32724061608314514,
|
|
"drift/rejected_abs": 0.21098506450653076,
|
|
"epoch": 0.2579720530275887,
|
|
"grad_norm": 0.369140625,
|
|
"learning_rate": 4.630929591809094e-07,
|
|
"logits/chosen": -1.9422146081924438,
|
|
"logits/rejected": -2.031545400619507,
|
|
"logps/chosen": -0.28938454389572144,
|
|
"logps/rejected": -0.2925392985343933,
|
|
"loss": 0.7183395624160767,
|
|
"loss/core": 0.6880876421928406,
|
|
"loss/preference_sft": 0.28938454389572144,
|
|
"loss/reference_anchor": 0.5760983228683472,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.2724032402038574,
|
|
"rewards/margins": 1.1642584800720215,
|
|
"rewards/rejected": 2.108144998550415,
|
|
"step": 270
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23237237334251404,
|
|
"drift/rejected_abs": 0.1247030645608902,
|
|
"epoch": 0.2627493132688403,
|
|
"grad_norm": 1.8515625,
|
|
"learning_rate": 4.603638282418183e-07,
|
|
"logits/chosen": -2.0928521156311035,
|
|
"logits/rejected": -2.0464084148406982,
|
|
"logps/chosen": -0.2967087924480438,
|
|
"logps/rejected": -0.30262163281440735,
|
|
"loss": 0.7003341913223267,
|
|
"loss/core": 0.6873778104782104,
|
|
"loss/preference_sft": 0.2967087924480438,
|
|
"loss/reference_anchor": 0.22945299744606018,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.3227198123931885,
|
|
"rewards/margins": 1.1851005554199219,
|
|
"rewards/rejected": 1.1376192569732666,
|
|
"step": 275
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2046871930360794,
|
|
"drift/rejected_abs": 0.08532479405403137,
|
|
"epoch": 0.26752657351009196,
|
|
"grad_norm": 0.330078125,
|
|
"learning_rate": 4.5754600770388765e-07,
|
|
"logits/chosen": -2.061699867248535,
|
|
"logits/rejected": -1.97652268409729,
|
|
"logps/chosen": -0.28828340768814087,
|
|
"logps/rejected": -0.3085075914859772,
|
|
"loss": 0.6936501264572144,
|
|
"loss/core": 0.6868374347686768,
|
|
"loss/preference_sft": 0.28828340768814087,
|
|
"loss/reference_anchor": 0.10742677748203278,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.044452667236328,
|
|
"rewards/margins": 1.2943838834762573,
|
|
"rewards/rejected": 0.7500687837600708,
|
|
"step": 280
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2622529864311218,
|
|
"drift/rejected_abs": 0.1607482135295868,
|
|
"epoch": 0.2723038337513436,
|
|
"grad_norm": 4.625,
|
|
"learning_rate": 4.5464068556305366e-07,
|
|
"logits/chosen": -2.086787700653076,
|
|
"logits/rejected": -2.0828757286071777,
|
|
"logps/chosen": -0.29549530148506165,
|
|
"logps/rejected": -0.2951886057853699,
|
|
"loss": 0.7032809853553772,
|
|
"loss/core": 0.6883570551872253,
|
|
"loss/preference_sft": 0.29549530148506165,
|
|
"loss/reference_anchor": 0.26892945170402527,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.6219353675842285,
|
|
"rewards/margins": 1.0172988176345825,
|
|
"rewards/rejected": 1.604636788368225,
|
|
"step": 285
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14772982895374298,
|
|
"drift/rejected_abs": 0.14406272768974304,
|
|
"epoch": 0.27708109399259523,
|
|
"grad_norm": 20.75,
|
|
"learning_rate": 4.516490867060156e-07,
|
|
"logits/chosen": -2.1649439334869385,
|
|
"logits/rejected": -2.1754214763641357,
|
|
"logps/chosen": -0.2871306836605072,
|
|
"logps/rejected": -0.2890948951244354,
|
|
"loss": 0.7047442197799683,
|
|
"loss/core": 0.6933256387710571,
|
|
"loss/preference_sft": 0.2871306836605072,
|
|
"loss/reference_anchor": 0.19965848326683044,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.4758152961730957,
|
|
"rewards/margins": 0.03618486598134041,
|
|
"rewards/rejected": 1.4396305084228516,
|
|
"step": 290
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.13134348392486572,
|
|
"drift/rejected_abs": 0.06761081516742706,
|
|
"epoch": 0.28185835423384686,
|
|
"grad_norm": 0.2470703125,
|
|
"learning_rate": 4.4857247239382153e-07,
|
|
"logits/chosen": -2.563570499420166,
|
|
"logits/rejected": -2.5265564918518066,
|
|
"logps/chosen": -0.24425753951072693,
|
|
"logps/rejected": -0.2567130923271179,
|
|
"loss": 0.6926940679550171,
|
|
"loss/core": 0.6899658441543579,
|
|
"loss/preference_sft": 0.24425753951072693,
|
|
"loss/reference_anchor": 0.03013770654797554,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.3127048015594482,
|
|
"rewards/margins": 0.639759361743927,
|
|
"rewards/rejected": 0.6729455590248108,
|
|
"step": 295
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3717862665653229,
|
|
"drift/rejected_abs": 0.10653592646121979,
|
|
"epoch": 0.28663561447509855,
|
|
"grad_norm": 1.0625,
|
|
"learning_rate": 4.4541213973012e-07,
|
|
"logits/chosen": -1.9797508716583252,
|
|
"logits/rejected": -1.9465057849884033,
|
|
"logps/chosen": -0.3131170868873596,
|
|
"logps/rejected": -0.3174610435962677,
|
|
"loss": 0.7069228887557983,
|
|
"loss/core": 0.6806262731552124,
|
|
"loss/preference_sft": 0.3131170868873596,
|
|
"loss/reference_anchor": 0.49461978673934937,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.7166762351989746,
|
|
"rewards/margins": 2.6619973182678223,
|
|
"rewards/rejected": 1.0546791553497314,
|
|
"step": 300
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21675559878349304,
|
|
"drift/rejected_abs": 0.08982045203447342,
|
|
"epoch": 0.2914128747163502,
|
|
"grad_norm": 0.4375,
|
|
"learning_rate": 4.4216942111429964e-07,
|
|
"logits/chosen": -2.1226086616516113,
|
|
"logits/rejected": -2.1839969158172607,
|
|
"logps/chosen": -0.26876428723335266,
|
|
"logps/rejected": -0.2606787085533142,
|
|
"loss": 0.6940160989761353,
|
|
"loss/core": 0.6869435906410217,
|
|
"loss/preference_sft": 0.26876428723335266,
|
|
"loss/reference_anchor": 0.11457415670156479,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.165956735610962,
|
|
"rewards/margins": 1.2678639888763428,
|
|
"rewards/rejected": 0.8980928659439087,
|
|
"step": 305
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2610951066017151,
|
|
"drift/rejected_abs": 0.21184654533863068,
|
|
"epoch": 0.2961901349576018,
|
|
"grad_norm": 2.484375,
|
|
"learning_rate": 4.388456836797484e-07,
|
|
"logits/chosen": -2.1281096935272217,
|
|
"logits/rejected": -2.0378966331481934,
|
|
"logps/chosen": -0.2733157277107239,
|
|
"logps/rejected": -0.28335532546043396,
|
|
"loss": 0.7152631282806396,
|
|
"loss/core": 0.6907567977905273,
|
|
"loss/preference_sft": 0.2733157277107239,
|
|
"loss/reference_anchor": 0.46279603242874146,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.6094913482666016,
|
|
"rewards/margins": 0.6211642026901245,
|
|
"rewards/rejected": 1.9883270263671875,
|
|
"step": 310
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2902871370315552,
|
|
"drift/rejected_abs": 0.13477492332458496,
|
|
"epoch": 0.30096739519885346,
|
|
"grad_norm": 0.271484375,
|
|
"learning_rate": 4.354423287174686e-07,
|
|
"logits/chosen": -1.9599071741104126,
|
|
"logits/rejected": -2.0659215450286865,
|
|
"logps/chosen": -0.27025145292282104,
|
|
"logps/rejected": -0.2758514881134033,
|
|
"loss": 0.698331892490387,
|
|
"loss/core": 0.6854726076126099,
|
|
"loss/preference_sft": 0.27025145292282104,
|
|
"loss/reference_anchor": 0.23015980422496796,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.9021315574645996,
|
|
"rewards/margins": 1.5580607652664185,
|
|
"rewards/rejected": 1.3440710306167603,
|
|
"step": 315
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15860998630523682,
|
|
"drift/rejected_abs": 0.12193727493286133,
|
|
"epoch": 0.3057446554401051,
|
|
"grad_norm": 0.244140625,
|
|
"learning_rate": 4.31960791085291e-07,
|
|
"logits/chosen": -1.9995390176773071,
|
|
"logits/rejected": -2.0159947872161865,
|
|
"logps/chosen": -0.28982076048851013,
|
|
"logps/rejected": -0.2935449182987213,
|
|
"loss": 0.6978195905685425,
|
|
"loss/core": 0.6914322376251221,
|
|
"loss/preference_sft": 0.28982076048851013,
|
|
"loss/reference_anchor": 0.09876595437526703,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.584044098854065,
|
|
"rewards/margins": 0.36467140913009644,
|
|
"rewards/rejected": 1.2193728685379028,
|
|
"step": 320
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3709779977798462,
|
|
"drift/rejected_abs": 0.1553051918745041,
|
|
"epoch": 0.31052191568135673,
|
|
"grad_norm": 13.5625,
|
|
"learning_rate": 4.2840253860293806e-07,
|
|
"logits/chosen": -2.2568116188049316,
|
|
"logits/rejected": -2.2034993171691895,
|
|
"logps/chosen": -0.2853277027606964,
|
|
"logps/rejected": -0.2819185256958008,
|
|
"loss": 0.731056809425354,
|
|
"loss/core": 0.683963418006897,
|
|
"loss/preference_sft": 0.2853277027606964,
|
|
"loss/reference_anchor": 0.9133332967758179,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.708606004714966,
|
|
"rewards/margins": 2.156642436981201,
|
|
"rewards/rejected": 1.5519633293151855,
|
|
"step": 325
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1991027593612671,
|
|
"drift/rejected_abs": 0.11322907358407974,
|
|
"epoch": 0.31529917592260837,
|
|
"grad_norm": 0.20703125,
|
|
"learning_rate": 4.24769071433189e-07,
|
|
"logits/chosen": -2.227357864379883,
|
|
"logits/rejected": -2.2161545753479004,
|
|
"logps/chosen": -0.2826363146305084,
|
|
"logps/rejected": -0.27046969532966614,
|
|
"loss": 0.6973029375076294,
|
|
"loss/core": 0.6889919638633728,
|
|
"loss/preference_sft": 0.2826363146305084,
|
|
"loss/reference_anchor": 0.13795623183250427,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.9903936386108398,
|
|
"rewards/margins": 0.8586524724960327,
|
|
"rewards/rejected": 1.1317412853240967,
|
|
"step": 330
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2119426280260086,
|
|
"drift/rejected_abs": 0.08484821766614914,
|
|
"epoch": 0.32007643616386,
|
|
"grad_norm": 3.09375,
|
|
"learning_rate": 4.2106192144940987e-07,
|
|
"logits/chosen": -2.0478832721710205,
|
|
"logits/rejected": -2.033395767211914,
|
|
"logps/chosen": -0.3026389479637146,
|
|
"logps/rejected": -0.32717952132225037,
|
|
"loss": 0.6988995671272278,
|
|
"loss/core": 0.6861973404884338,
|
|
"loss/preference_sft": 0.3026389479637146,
|
|
"loss/reference_anchor": 0.22377829253673553,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.1165146827697754,
|
|
"rewards/margins": 1.5143001079559326,
|
|
"rewards/rejected": 0.6022144556045532,
|
|
"step": 335
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2043064534664154,
|
|
"drift/rejected_abs": 0.08732999116182327,
|
|
"epoch": 0.3248536964051117,
|
|
"grad_norm": 2.265625,
|
|
"learning_rate": 4.172826515897145e-07,
|
|
"logits/chosen": -2.094050645828247,
|
|
"logits/rejected": -2.2169277667999268,
|
|
"logps/chosen": -0.2799380123615265,
|
|
"logps/rejected": -0.28169578313827515,
|
|
"loss": 0.6966040134429932,
|
|
"loss/core": 0.6874774694442749,
|
|
"loss/preference_sft": 0.2799380123615265,
|
|
"loss/reference_anchor": 0.15453729033470154,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.0414788722991943,
|
|
"rewards/margins": 1.1714600324630737,
|
|
"rewards/rejected": 0.8700188398361206,
|
|
"step": 340
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.164683997631073,
|
|
"drift/rejected_abs": 0.08445769548416138,
|
|
"epoch": 0.3296309566463633,
|
|
"grad_norm": 0.4921875,
|
|
"learning_rate": 4.1343285519802783e-07,
|
|
"logits/chosen": -2.3189194202423096,
|
|
"logits/rejected": -2.378885269165039,
|
|
"logps/chosen": -0.2820732891559601,
|
|
"logps/rejected": -0.29256579279899597,
|
|
"loss": 0.6956444978713989,
|
|
"loss/core": 0.68927001953125,
|
|
"loss/preference_sft": 0.2820732891559601,
|
|
"loss/reference_anchor": 0.09928078949451447,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.6447336673736572,
|
|
"rewards/margins": 0.8007295727729797,
|
|
"rewards/rejected": 0.8440041542053223,
|
|
"step": 345
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19914595782756805,
|
|
"drift/rejected_abs": 0.09469764679670334,
|
|
"epoch": 0.33440821688761496,
|
|
"grad_norm": 0.5234375,
|
|
"learning_rate": 4.0951415535233056e-07,
|
|
"logits/chosen": -2.0590832233428955,
|
|
"logits/rejected": -2.0390827655792236,
|
|
"logps/chosen": -0.2612057030200958,
|
|
"logps/rejected": -0.25672319531440735,
|
|
"loss": 0.6943715810775757,
|
|
"loss/core": 0.6880162954330444,
|
|
"loss/preference_sft": 0.2612057030200958,
|
|
"loss/reference_anchor": 0.10098560899496078,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.98989999294281,
|
|
"rewards/margins": 1.0453819036483765,
|
|
"rewards/rejected": 0.9445180892944336,
|
|
"step": 350
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18552103638648987,
|
|
"drift/rejected_abs": 0.13385716080665588,
|
|
"epoch": 0.3391854771288666,
|
|
"grad_norm": 0.375,
|
|
"learning_rate": 4.0552820418036847e-07,
|
|
"logits/chosen": -2.1061980724334717,
|
|
"logits/rejected": -2.232513904571533,
|
|
"logps/chosen": -0.28036990761756897,
|
|
"logps/rejected": -0.29538074135780334,
|
|
"loss": 0.7025222182273865,
|
|
"loss/core": 0.6908389329910278,
|
|
"loss/preference_sft": 0.28036990761756897,
|
|
"loss/reference_anchor": 0.20562918484210968,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.8541920185089111,
|
|
"rewards/margins": 0.5308849215507507,
|
|
"rewards/rejected": 1.3233067989349365,
|
|
"step": 355
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2627464234828949,
|
|
"drift/rejected_abs": 0.12656167149543762,
|
|
"epoch": 0.34396273737011823,
|
|
"grad_norm": 2.390625,
|
|
"learning_rate": 4.0147668216311276e-07,
|
|
"logits/chosen": -2.1296074390411377,
|
|
"logits/rejected": -2.192688465118408,
|
|
"logps/chosen": -0.2808237075805664,
|
|
"logps/rejected": -0.28292280435562134,
|
|
"loss": 0.7036789655685425,
|
|
"loss/core": 0.6867524981498718,
|
|
"loss/preference_sft": 0.2808237075805664,
|
|
"loss/reference_anchor": 0.310445636510849,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.626671314239502,
|
|
"rewards/margins": 1.3620039224624634,
|
|
"rewards/rejected": 1.2646673917770386,
|
|
"step": 360
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2869020104408264,
|
|
"drift/rejected_abs": 0.08114350587129593,
|
|
"epoch": 0.34873999761136987,
|
|
"grad_norm": 9.625,
|
|
"learning_rate": 3.9736129742626843e-07,
|
|
"logits/chosen": -2.067612886428833,
|
|
"logits/rejected": -2.1757853031158447,
|
|
"logps/chosen": -0.2856762707233429,
|
|
"logps/rejected": -0.283938467502594,
|
|
"loss": 0.7096842527389526,
|
|
"loss/core": 0.683948278427124,
|
|
"loss/preference_sft": 0.2856762707233429,
|
|
"loss/reference_anchor": 0.4861505925655365,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.8673691749572754,
|
|
"rewards/margins": 2.0584607124328613,
|
|
"rewards/rejected": 0.8089081048965454,
|
|
"step": 365
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1641189157962799,
|
|
"drift/rejected_abs": 0.10590697824954987,
|
|
"epoch": 0.3535172578526215,
|
|
"grad_norm": 1.296875,
|
|
"learning_rate": 3.931837850201263e-07,
|
|
"logits/chosen": -2.267899990081787,
|
|
"logits/rejected": -2.313075304031372,
|
|
"logps/chosen": -0.2852608561515808,
|
|
"logps/rejected": -0.28572216629981995,
|
|
"loss": 0.6960805654525757,
|
|
"loss/core": 0.6902772188186646,
|
|
"loss/preference_sft": 0.2852608561515808,
|
|
"loss/reference_anchor": 0.08754244446754456,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.640094518661499,
|
|
"rewards/margins": 0.5825773477554321,
|
|
"rewards/rejected": 1.0575170516967773,
|
|
"step": 370
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.30126437544822693,
|
|
"drift/rejected_abs": 0.2238939106464386,
|
|
"epoch": 0.35829451809387314,
|
|
"grad_norm": 0.37109375,
|
|
"learning_rate": 3.889459061880643e-07,
|
|
"logits/chosen": -2.046881914138794,
|
|
"logits/rejected": -2.0237812995910645,
|
|
"logps/chosen": -0.2588959336280823,
|
|
"logps/rejected": -0.25813889503479004,
|
|
"loss": 0.7113808393478394,
|
|
"loss/core": 0.6897670030593872,
|
|
"loss/preference_sft": 0.2588959336280823,
|
|
"loss/reference_anchor": 0.4063888192176819,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.0123302936553955,
|
|
"rewards/margins": 0.7923107147216797,
|
|
"rewards/rejected": 2.220019817352295,
|
|
"step": 375
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2056204378604889,
|
|
"drift/rejected_abs": 0.15891826152801514,
|
|
"epoch": 0.36307177833512483,
|
|
"grad_norm": 0.94140625,
|
|
"learning_rate": 3.846494476240056e-07,
|
|
"logits/chosen": -2.1194448471069336,
|
|
"logits/rejected": -2.1285529136657715,
|
|
"logps/chosen": -0.27972573041915894,
|
|
"logps/rejected": -0.2927209138870239,
|
|
"loss": 0.7116562128067017,
|
|
"loss/core": 0.6913924217224121,
|
|
"loss/preference_sft": 0.27972573041915894,
|
|
"loss/reference_anchor": 0.37730082869529724,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.053956985473633,
|
|
"rewards/margins": 0.48597222566604614,
|
|
"rewards/rejected": 1.5679845809936523,
|
|
"step": 380
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21692092716693878,
|
|
"drift/rejected_abs": 0.1518086940050125,
|
|
"epoch": 0.36784903857637646,
|
|
"grad_norm": 0.87109375,
|
|
"learning_rate": 3.8029622071914626e-07,
|
|
"logits/chosen": -2.1106455326080322,
|
|
"logits/rejected": -2.1308746337890625,
|
|
"logps/chosen": -0.2711559236049652,
|
|
"logps/rejected": -0.27569103240966797,
|
|
"loss": 0.7008264064788818,
|
|
"loss/core": 0.6900737881660461,
|
|
"loss/preference_sft": 0.2711559236049652,
|
|
"loss/reference_anchor": 0.18793678283691406,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.168698787689209,
|
|
"rewards/margins": 0.6546289920806885,
|
|
"rewards/rejected": 1.514069676399231,
|
|
"step": 385
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17692413926124573,
|
|
"drift/rejected_abs": 0.08787186443805695,
|
|
"epoch": 0.3726262988176281,
|
|
"grad_norm": 0.2119140625,
|
|
"learning_rate": 3.758880607982714e-07,
|
|
"logits/chosen": -1.869436502456665,
|
|
"logits/rejected": -1.9471441507339478,
|
|
"logps/chosen": -0.3066801428794861,
|
|
"logps/rejected": -0.3042067885398865,
|
|
"loss": 0.6953903436660767,
|
|
"loss/core": 0.6888266205787659,
|
|
"loss/preference_sft": 0.3066801428794861,
|
|
"loss/reference_anchor": 0.10060696303844452,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.7669336795806885,
|
|
"rewards/margins": 0.8882148861885071,
|
|
"rewards/rejected": 0.8787187337875366,
|
|
"step": 390
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16815339028835297,
|
|
"drift/rejected_abs": 0.08043881505727768,
|
|
"epoch": 0.37740355905887973,
|
|
"grad_norm": 0.212890625,
|
|
"learning_rate": 3.714268263459801e-07,
|
|
"logits/chosen": -2.2196106910705566,
|
|
"logits/rejected": -2.3220858573913574,
|
|
"logps/chosen": -0.29513680934906006,
|
|
"logps/rejected": -0.28997355699539185,
|
|
"loss": 0.6943039894104004,
|
|
"loss/core": 0.6888213753700256,
|
|
"loss/preference_sft": 0.29513680934906006,
|
|
"loss/reference_anchor": 0.08013781160116196,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 1.680734634399414,
|
|
"rewards/margins": 0.8773964643478394,
|
|
"rewards/rejected": 0.8033380508422852,
|
|
"step": 395
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.29227569699287415,
|
|
"drift/rejected_abs": 0.08427596837282181,
|
|
"epoch": 0.38218081930013137,
|
|
"grad_norm": 1.5390625,
|
|
"learning_rate": 3.6691439822314666e-07,
|
|
"logits/chosen": -2.1000590324401855,
|
|
"logits/rejected": -2.135270595550537,
|
|
"logps/chosen": -0.2931859791278839,
|
|
"logps/rejected": -0.2932865023612976,
|
|
"loss": 0.7107817530632019,
|
|
"loss/core": 0.6836472749710083,
|
|
"loss/preference_sft": 0.2931859791278839,
|
|
"loss/reference_anchor": 0.5133715867996216,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.9214870929718018,
|
|
"rewards/margins": 2.0816428661346436,
|
|
"rewards/rejected": 0.8398441076278687,
|
|
"step": 400
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23128154873847961,
|
|
"drift/rejected_abs": 0.12374065816402435,
|
|
"epoch": 0.386958079541383,
|
|
"grad_norm": 0.310546875,
|
|
"learning_rate": 3.623526788739477e-07,
|
|
"logits/chosen": -2.131155014038086,
|
|
"logits/rejected": -2.1849007606506348,
|
|
"logps/chosen": -0.3152233064174652,
|
|
"logps/rejected": -0.3035925030708313,
|
|
"loss": 0.7062971591949463,
|
|
"loss/core": 0.6880842447280884,
|
|
"loss/preference_sft": 0.3152233064174652,
|
|
"loss/reference_anchor": 0.33273500204086304,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.312814235687256,
|
|
"rewards/margins": 1.0756020545959473,
|
|
"rewards/rejected": 1.2372119426727295,
|
|
"step": 405
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20869719982147217,
|
|
"drift/rejected_abs": 0.11958236992359161,
|
|
"epoch": 0.39173533978263464,
|
|
"grad_norm": 0.71875,
|
|
"learning_rate": 3.5774359152378985e-07,
|
|
"logits/chosen": -2.248558521270752,
|
|
"logits/rejected": -2.2788643836975098,
|
|
"logps/chosen": -0.2772386372089386,
|
|
"logps/rejected": -0.28785425424575806,
|
|
"loss": 0.6970154047012329,
|
|
"loss/core": 0.6887233257293701,
|
|
"loss/preference_sft": 0.2772386372089386,
|
|
"loss/reference_anchor": 0.138116717338562,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.0857138633728027,
|
|
"rewards/margins": 0.8951884508132935,
|
|
"rewards/rejected": 1.1905254125595093,
|
|
"step": 410
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.09970752894878387,
|
|
"drift/rejected_abs": 0.03773488104343414,
|
|
"epoch": 0.3965126000238863,
|
|
"grad_norm": 0.2275390625,
|
|
"learning_rate": 3.530890793684759e-07,
|
|
"logits/chosen": -2.1164305210113525,
|
|
"logits/rejected": -2.1951279640197754,
|
|
"logps/chosen": -0.33025726675987244,
|
|
"logps/rejected": -0.32088834047317505,
|
|
"loss": 0.693244218826294,
|
|
"loss/core": 0.6900933384895325,
|
|
"loss/preference_sft": 0.33025726675987244,
|
|
"loss/reference_anchor": 0.02999264933168888,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 0.9955142736434937,
|
|
"rewards/margins": 0.6189162731170654,
|
|
"rewards/rejected": 0.376598060131073,
|
|
"step": 415
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2045033723115921,
|
|
"drift/rejected_abs": 0.07196368277072906,
|
|
"epoch": 0.40128986026513797,
|
|
"grad_norm": 1.9609375,
|
|
"learning_rate": 3.4839110475495146e-07,
|
|
"logits/chosen": -2.349240779876709,
|
|
"logits/rejected": -2.316765785217285,
|
|
"logps/chosen": -0.28340643644332886,
|
|
"logps/rejected": -0.2993830144405365,
|
|
"loss": 0.6966205835342407,
|
|
"loss/core": 0.6868051290512085,
|
|
"loss/preference_sft": 0.28340643644332886,
|
|
"loss/reference_anchor": 0.16796544194221497,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.0448544025421143,
|
|
"rewards/margins": 1.3295873403549194,
|
|
"rewards/rejected": 0.7152668237686157,
|
|
"step": 420
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22461578249931335,
|
|
"drift/rejected_abs": 0.16200877726078033,
|
|
"epoch": 0.4060671205063896,
|
|
"grad_norm": 6.9375,
|
|
"learning_rate": 3.43651648353978e-07,
|
|
"logits/chosen": -2.012849807739258,
|
|
"logits/rejected": -2.072317123413086,
|
|
"logps/chosen": -0.29064491391181946,
|
|
"logps/rejected": -0.2678697109222412,
|
|
"loss": 0.7116137742996216,
|
|
"loss/core": 0.6908143758773804,
|
|
"loss/preference_sft": 0.29064491391181946,
|
|
"loss/reference_anchor": 0.3869226574897766,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.2447874546051025,
|
|
"rewards/margins": 0.6263939142227173,
|
|
"rewards/rejected": 1.6183935403823853,
|
|
"step": 425
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.43855953216552734,
|
|
"drift/rejected_abs": 0.14190329611301422,
|
|
"epoch": 0.41084438074764124,
|
|
"grad_norm": 16.25,
|
|
"learning_rate": 3.388727083250795e-07,
|
|
"logits/chosen": -2.0672688484191895,
|
|
"logits/rejected": -2.0907437801361084,
|
|
"logps/chosen": -0.2815065085887909,
|
|
"logps/rejected": -0.26665714383125305,
|
|
"loss": 0.7262654304504395,
|
|
"loss/core": 0.6797505617141724,
|
|
"loss/preference_sft": 0.2815065085887909,
|
|
"loss/reference_anchor": 0.9021477699279785,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 4.383257865905762,
|
|
"rewards/margins": 2.965740442276001,
|
|
"rewards/rejected": 1.4175175428390503,
|
|
"step": 430
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.162430077791214,
|
|
"drift/rejected_abs": 0.10076061636209488,
|
|
"epoch": 0.41562164098889287,
|
|
"grad_norm": 0.73046875,
|
|
"learning_rate": 3.3405629947411683e-07,
|
|
"logits/chosen": -2.350315570831299,
|
|
"logits/rejected": -2.3338377475738525,
|
|
"logps/chosen": -0.27856045961380005,
|
|
"logps/rejected": -0.293110191822052,
|
|
"loss": 0.6974702477455139,
|
|
"loss/core": 0.69013512134552,
|
|
"loss/preference_sft": 0.27856045961380005,
|
|
"loss/reference_anchor": 0.11884655803442001,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.6225159168243408,
|
|
"rewards/margins": 0.6160438656806946,
|
|
"rewards/rejected": 1.006471872329712,
|
|
"step": 435
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.299646258354187,
|
|
"drift/rejected_abs": 0.13414156436920166,
|
|
"epoch": 0.4203989012301445,
|
|
"grad_norm": 13.6875,
|
|
"learning_rate": 3.2920445240384325e-07,
|
|
"logits/chosen": -2.05702543258667,
|
|
"logits/rejected": -2.1284549236297607,
|
|
"logps/chosen": -0.3045402765274048,
|
|
"logps/rejected": -0.30474570393562317,
|
|
"loss": 0.711657702922821,
|
|
"loss/core": 0.6856769323348999,
|
|
"loss/preference_sft": 0.3045402765274048,
|
|
"loss/reference_anchor": 0.48916101455688477,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.993908643722534,
|
|
"rewards/margins": 1.6548278331756592,
|
|
"rewards/rejected": 1.339080810546875,
|
|
"step": 440
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2965596318244934,
|
|
"drift/rejected_abs": 0.1601937860250473,
|
|
"epoch": 0.42517616147139614,
|
|
"grad_norm": 1.0078125,
|
|
"learning_rate": 3.2431921265780063e-07,
|
|
"logits/chosen": -1.9329261779785156,
|
|
"logits/rejected": -1.909558653831482,
|
|
"logps/chosen": -0.2952488958835602,
|
|
"logps/rejected": -0.298814594745636,
|
|
"loss": 0.7007791996002197,
|
|
"loss/core": 0.6862999796867371,
|
|
"loss/preference_sft": 0.2952488958835602,
|
|
"loss/reference_anchor": 0.2600591480731964,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.9641549587249756,
|
|
"rewards/margins": 1.417994737625122,
|
|
"rewards/rejected": 1.5461599826812744,
|
|
"step": 445
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.34632277488708496,
|
|
"drift/rejected_abs": 0.156765878200531,
|
|
"epoch": 0.4299534217126478,
|
|
"grad_norm": 13.0625,
|
|
"learning_rate": 3.1940263985791616e-07,
|
|
"logits/chosen": -1.973149061203003,
|
|
"logits/rejected": -1.990882158279419,
|
|
"logps/chosen": -0.297830194234848,
|
|
"logps/rejected": -0.2937723696231842,
|
|
"loss": 0.7078851461410522,
|
|
"loss/core": 0.6842607259750366,
|
|
"loss/preference_sft": 0.297830194234848,
|
|
"loss/reference_anchor": 0.44270557165145874,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.462675094604492,
|
|
"rewards/margins": 1.8995628356933594,
|
|
"rewards/rejected": 1.5631126165390015,
|
|
"step": 450
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.341677725315094,
|
|
"drift/rejected_abs": 0.14796192944049835,
|
|
"epoch": 0.4347306819538994,
|
|
"grad_norm": 4.9375,
|
|
"learning_rate": 3.144568068361634e-07,
|
|
"logits/chosen": -1.9354276657104492,
|
|
"logits/rejected": -2.1260530948638916,
|
|
"logps/chosen": -0.3276836574077606,
|
|
"logps/rejected": -0.3140363097190857,
|
|
"loss": 0.7205722332000732,
|
|
"loss/core": 0.6841714382171631,
|
|
"loss/preference_sft": 0.3276836574077606,
|
|
"loss/reference_anchor": 0.6952458620071411,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.415968418121338,
|
|
"rewards/margins": 1.9381797313690186,
|
|
"rewards/rejected": 1.4777886867523193,
|
|
"step": 455
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3663923144340515,
|
|
"drift/rejected_abs": 0.18600957095623016,
|
|
"epoch": 0.4395079421951511,
|
|
"grad_norm": 1.1015625,
|
|
"learning_rate": 3.0948379876065465e-07,
|
|
"logits/chosen": -2.0609209537506104,
|
|
"logits/rejected": -2.048144578933716,
|
|
"logps/chosen": -0.2960844337940216,
|
|
"logps/rejected": -0.3075362741947174,
|
|
"loss": 0.7149934768676758,
|
|
"loss/core": 0.6845480799674988,
|
|
"loss/preference_sft": 0.2960844337940216,
|
|
"loss/reference_anchor": 0.579299807548523,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.6639232635498047,
|
|
"rewards/margins": 1.8042666912078857,
|
|
"rewards/rejected": 1.859656572341919,
|
|
"step": 460
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23091194033622742,
|
|
"drift/rejected_abs": 0.09295056760311127,
|
|
"epoch": 0.44428520243640274,
|
|
"grad_norm": 0.341796875,
|
|
"learning_rate": 3.0448571225653193e-07,
|
|
"logits/chosen": -1.8885440826416016,
|
|
"logits/rejected": -1.988734245300293,
|
|
"logps/chosen": -0.29641199111938477,
|
|
"logps/rejected": -0.2917849123477936,
|
|
"loss": 0.6937438249588013,
|
|
"loss/core": 0.6863740682601929,
|
|
"loss/preference_sft": 0.29641199111938477,
|
|
"loss/reference_anchor": 0.11775638163089752,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.3084988594055176,
|
|
"rewards/margins": 1.3800839185714722,
|
|
"rewards/rejected": 0.9284147024154663,
|
|
"step": 465
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2984088957309723,
|
|
"drift/rejected_abs": 0.12458224594593048,
|
|
"epoch": 0.4490624626776544,
|
|
"grad_norm": 22.5,
|
|
"learning_rate": 2.9946465452202747e-07,
|
|
"logits/chosen": -2.1285500526428223,
|
|
"logits/rejected": -2.196565628051758,
|
|
"logps/chosen": -0.28165316581726074,
|
|
"logps/rejected": -0.2599329352378845,
|
|
"loss": 0.7142145037651062,
|
|
"loss/core": 0.6846899390220642,
|
|
"loss/preference_sft": 0.28165316581726074,
|
|
"loss/reference_anchor": 0.5623257756233215,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.984055757522583,
|
|
"rewards/margins": 1.8138116598129272,
|
|
"rewards/rejected": 1.170243501663208,
|
|
"step": 470
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.182987779378891,
|
|
"drift/rejected_abs": 0.14684465527534485,
|
|
"epoch": 0.453839722918906,
|
|
"grad_norm": 3.4375,
|
|
"learning_rate": 2.944227424400672e-07,
|
|
"logits/chosen": -2.0931344032287598,
|
|
"logits/rejected": -2.0336596965789795,
|
|
"logps/chosen": -0.29563307762145996,
|
|
"logps/rejected": -0.3208538889884949,
|
|
"loss": 0.7051785588264465,
|
|
"loss/core": 0.6917919516563416,
|
|
"loss/preference_sft": 0.29563307762145996,
|
|
"loss/reference_anchor": 0.23816709220409393,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.8292009830474854,
|
|
"rewards/margins": 0.3628115653991699,
|
|
"rewards/rejected": 1.4663891792297363,
|
|
"step": 475
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23586270213127136,
|
|
"drift/rejected_abs": 0.10744066536426544,
|
|
"epoch": 0.45861698316015764,
|
|
"grad_norm": 0.466796875,
|
|
"learning_rate": 2.8936210168579037e-07,
|
|
"logits/chosen": -2.2626140117645264,
|
|
"logits/rejected": -2.318941593170166,
|
|
"logps/chosen": -0.275493860244751,
|
|
"logps/rejected": -0.2723752558231354,
|
|
"loss": 0.7019254565238953,
|
|
"loss/core": 0.6869186758995056,
|
|
"loss/preference_sft": 0.275493860244751,
|
|
"loss/reference_anchor": 0.27258676290512085,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.3575544357299805,
|
|
"rewards/margins": 1.3319319486618042,
|
|
"rewards/rejected": 1.0256226062774658,
|
|
"step": 480
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17448770999908447,
|
|
"drift/rejected_abs": 0.08884555101394653,
|
|
"epoch": 0.4633942434014093,
|
|
"grad_norm": 0.478515625,
|
|
"learning_rate": 2.842848658303637e-07,
|
|
"logits/chosen": -2.2037127017974854,
|
|
"logits/rejected": -2.2105116844177246,
|
|
"logps/chosen": -0.29139575362205505,
|
|
"logps/rejected": -0.29565316438674927,
|
|
"loss": 0.694310188293457,
|
|
"loss/core": 0.6889179944992065,
|
|
"loss/preference_sft": 0.29139575362205505,
|
|
"loss/reference_anchor": 0.07870315760374069,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.7447458505630493,
|
|
"rewards/margins": 0.8566008806228638,
|
|
"rewards/rejected": 0.8881451487541199,
|
|
"step": 485
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20757827162742615,
|
|
"drift/rejected_abs": 0.10591663420200348,
|
|
"epoch": 0.4681715036426609,
|
|
"grad_norm": 0.65625,
|
|
"learning_rate": 2.79193175441464e-07,
|
|
"logits/chosen": -2.1531708240509033,
|
|
"logits/rejected": -2.1945109367370605,
|
|
"logps/chosen": -0.3073252737522125,
|
|
"logps/rejected": -0.29394882917404175,
|
|
"loss": 0.6971133947372437,
|
|
"loss/core": 0.688152015209198,
|
|
"loss/preference_sft": 0.3073252737522125,
|
|
"loss/reference_anchor": 0.1484949290752411,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.0757830142974854,
|
|
"rewards/margins": 1.0181604623794556,
|
|
"rewards/rejected": 1.0576225519180298,
|
|
"step": 490
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25867894291877747,
|
|
"drift/rejected_abs": 0.08946122229099274,
|
|
"epoch": 0.47294876388391255,
|
|
"grad_norm": 0.390625,
|
|
"learning_rate": 2.74089177180815e-07,
|
|
"logits/chosen": -2.2221720218658447,
|
|
"logits/rejected": -2.3168718814849854,
|
|
"logps/chosen": -0.3004676401615143,
|
|
"logps/rejected": -0.29068127274513245,
|
|
"loss": 0.7042246460914612,
|
|
"loss/core": 0.6852529048919678,
|
|
"loss/preference_sft": 0.3004676401615143,
|
|
"loss/reference_anchor": 0.34938645362854004,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.584899663925171,
|
|
"rewards/margins": 1.6914997100830078,
|
|
"rewards/rejected": 0.8933998346328735,
|
|
"step": 495
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4131302833557129,
|
|
"drift/rejected_abs": 0.20293641090393066,
|
|
"epoch": 0.47772602412516424,
|
|
"grad_norm": 4.125,
|
|
"learning_rate": 2.6897502289915026e-07,
|
|
"logits/chosen": -2.021841526031494,
|
|
"logits/rejected": -1.9711917638778687,
|
|
"logps/chosen": -0.28432515263557434,
|
|
"logps/rejected": -0.3025287389755249,
|
|
"loss": 0.741477370262146,
|
|
"loss/core": 0.6841000318527222,
|
|
"loss/preference_sft": 0.28432515263557434,
|
|
"loss/reference_anchor": 1.1191157102584839,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 4.131275177001953,
|
|
"rewards/margins": 2.104186534881592,
|
|
"rewards/rejected": 2.0270884037017822,
|
|
"step": 500
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.36249279975891113,
|
|
"drift/rejected_abs": 0.1847756803035736,
|
|
"epoch": 0.4825032843664159,
|
|
"grad_norm": 6.1875,
|
|
"learning_rate": 2.638528687289925e-07,
|
|
"logits/chosen": -2.1491918563842773,
|
|
"logits/rejected": -2.1820521354675293,
|
|
"logps/chosen": -0.2843039035797119,
|
|
"logps/rejected": -0.28839248418807983,
|
|
"loss": 0.7243824005126953,
|
|
"loss/core": 0.6854826807975769,
|
|
"loss/preference_sft": 0.2843039035797119,
|
|
"loss/reference_anchor": 0.7495636940002441,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 3.623505115509033,
|
|
"rewards/margins": 1.8764804601669312,
|
|
"rewards/rejected": 1.7470247745513916,
|
|
"step": 505
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20058460533618927,
|
|
"drift/rejected_abs": 0.08017744868993759,
|
|
"epoch": 0.4872805446076675,
|
|
"grad_norm": 0.1259765625,
|
|
"learning_rate": 2.5872487417562527e-07,
|
|
"logits/chosen": -2.1467156410217285,
|
|
"logits/rejected": -2.2804579734802246,
|
|
"logps/chosen": -0.3167296051979065,
|
|
"logps/rejected": -0.2912823259830475,
|
|
"loss": 0.6994425654411316,
|
|
"loss/core": 0.6874934434890747,
|
|
"loss/preference_sft": 0.3167296051979065,
|
|
"loss/reference_anchor": 0.2073097974061966,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.002615451812744,
|
|
"rewards/margins": 1.2019274234771729,
|
|
"rewards/rejected": 0.8006879687309265,
|
|
"step": 510
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2543594241142273,
|
|
"drift/rejected_abs": 0.16676507890224457,
|
|
"epoch": 0.49205780484891914,
|
|
"grad_norm": 0.38671875,
|
|
"learning_rate": 2.535932012066433e-07,
|
|
"logits/chosen": -2.0313689708709717,
|
|
"logits/rejected": -2.034614324569702,
|
|
"logps/chosen": -0.2830977141857147,
|
|
"logps/rejected": -0.3066386282444,
|
|
"loss": 0.7011044025421143,
|
|
"loss/core": 0.6888419985771179,
|
|
"loss/preference_sft": 0.2830977141857147,
|
|
"loss/reference_anchor": 0.216938778758049,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.5404956340789795,
|
|
"rewards/margins": 0.8886449933052063,
|
|
"rewards/rejected": 1.6518504619598389,
|
|
"step": 515
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.30793458223342896,
|
|
"drift/rejected_abs": 0.09768765419721603,
|
|
"epoch": 0.4968350650901708,
|
|
"grad_norm": 2.0,
|
|
"learning_rate": 2.4846001334046535e-07,
|
|
"logits/chosen": -2.058237314224243,
|
|
"logits/rejected": -2.226836681365967,
|
|
"logps/chosen": -0.322182834148407,
|
|
"logps/rejected": -0.33398357033729553,
|
|
"loss": 0.7052879929542542,
|
|
"loss/core": 0.6832786798477173,
|
|
"loss/preference_sft": 0.322182834148407,
|
|
"loss/reference_anchor": 0.4079677164554596,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.0726497173309326,
|
|
"rewards/margins": 2.1081771850585938,
|
|
"rewards/rejected": 0.9644724726676941,
|
|
"step": 520
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.42069754004478455,
|
|
"drift/rejected_abs": 0.14602303504943848,
|
|
"epoch": 0.5016123253314224,
|
|
"grad_norm": 30.625,
|
|
"learning_rate": 2.433274747341919e-07,
|
|
"logits/chosen": -2.194186210632324,
|
|
"logits/rejected": -2.162163496017456,
|
|
"logps/chosen": -0.3106693625450134,
|
|
"logps/rejected": -0.30101826786994934,
|
|
"loss": 0.73024982213974,
|
|
"loss/core": 0.6811276078224182,
|
|
"loss/preference_sft": 0.3106693625450134,
|
|
"loss/reference_anchor": 0.951376736164093,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.204687118530273,
|
|
"rewards/margins": 2.749366521835327,
|
|
"rewards/rejected": 1.455320119857788,
|
|
"step": 525
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.30520981550216675,
|
|
"drift/rejected_abs": 0.1496325582265854,
|
|
"epoch": 0.5063895855726741,
|
|
"grad_norm": 0.140625,
|
|
"learning_rate": 2.3819774927119522e-07,
|
|
"logits/chosen": -2.0378127098083496,
|
|
"logits/rejected": -2.001634359359741,
|
|
"logps/chosen": -0.28599685430526733,
|
|
"logps/rejected": -0.28872916102409363,
|
|
"loss": 0.7070541977882385,
|
|
"loss/core": 0.6858066320419312,
|
|
"loss/preference_sft": 0.28599685430526733,
|
|
"loss/reference_anchor": 0.3963504433631897,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.050769090652466,
|
|
"rewards/margins": 1.5627717971801758,
|
|
"rewards/rejected": 1.4879971742630005,
|
|
"step": 530
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.30915752053260803,
|
|
"drift/rejected_abs": 0.10786645114421844,
|
|
"epoch": 0.5111668458139257,
|
|
"grad_norm": 13.6875,
|
|
"learning_rate": 2.3307299964882312e-07,
|
|
"logits/chosen": -2.186264991760254,
|
|
"logits/rejected": -2.108275890350342,
|
|
"logps/chosen": -0.2858208417892456,
|
|
"logps/rejected": -0.27927789092063904,
|
|
"loss": 0.7080153226852417,
|
|
"loss/core": 0.6839091181755066,
|
|
"loss/preference_sft": 0.2858208417892456,
|
|
"loss/reference_anchor": 0.4535427987575531,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.0912933349609375,
|
|
"rewards/margins": 2.0143520832061768,
|
|
"rewards/rejected": 1.0769412517547607,
|
|
"step": 535
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15930701792240143,
|
|
"drift/rejected_abs": 0.14954623579978943,
|
|
"epoch": 0.5159441060551774,
|
|
"grad_norm": 0.193359375,
|
|
"learning_rate": 2.279553864666046e-07,
|
|
"logits/chosen": -2.106029748916626,
|
|
"logits/rejected": -2.0835556983947754,
|
|
"logps/chosen": -0.27827921509742737,
|
|
"logps/rejected": -0.2890811562538147,
|
|
"loss": 0.7021555304527283,
|
|
"loss/core": 0.6928835511207581,
|
|
"loss/preference_sft": 0.27827921509742737,
|
|
"loss/reference_anchor": 0.15761205554008484,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.5930081605911255,
|
|
"rewards/margins": 0.09992774575948715,
|
|
"rewards/rejected": 1.4930803775787354,
|
|
"step": 540
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26317763328552246,
|
|
"drift/rejected_abs": 0.08720904588699341,
|
|
"epoch": 0.520721366296429,
|
|
"grad_norm": 23.375,
|
|
"learning_rate": 2.22847067315338e-07,
|
|
"logits/chosen": -2.331496477127075,
|
|
"logits/rejected": -2.172362804412842,
|
|
"logps/chosen": -0.2841755151748657,
|
|
"logps/rejected": -0.2938932776451111,
|
|
"loss": 0.7100147008895874,
|
|
"loss/core": 0.6851332187652588,
|
|
"loss/preference_sft": 0.2841755151748657,
|
|
"loss/reference_anchor": 0.46921229362487793,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.6317765712738037,
|
|
"rewards/margins": 1.762070655822754,
|
|
"rewards/rejected": 0.8697060346603394,
|
|
"step": 545
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2769598066806793,
|
|
"drift/rejected_abs": 0.16232618689537048,
|
|
"epoch": 0.5254986265376806,
|
|
"grad_norm": 0.53515625,
|
|
"learning_rate": 2.1775019586744921e-07,
|
|
"logits/chosen": -1.8962993621826172,
|
|
"logits/rejected": -1.9161226749420166,
|
|
"logps/chosen": -0.27603915333747864,
|
|
"logps/rejected": -0.2979462742805481,
|
|
"loss": 0.7087868452072144,
|
|
"loss/core": 0.6875404119491577,
|
|
"loss/preference_sft": 0.27603915333747864,
|
|
"loss/reference_anchor": 0.3973239064216614,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.7692787647247314,
|
|
"rewards/margins": 1.1475410461425781,
|
|
"rewards/rejected": 1.6217377185821533,
|
|
"step": 550
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.32226184010505676,
|
|
"drift/rejected_abs": 0.09745451807975769,
|
|
"epoch": 0.5302758867789323,
|
|
"grad_norm": 0.2578125,
|
|
"learning_rate": 2.126669209690008e-07,
|
|
"logits/chosen": -2.0920777320861816,
|
|
"logits/rejected": -2.14959979057312,
|
|
"logps/chosen": -0.31904277205467224,
|
|
"logps/rejected": -0.2770870327949524,
|
|
"loss": 0.7026501893997192,
|
|
"loss/core": 0.6825853586196899,
|
|
"loss/preference_sft": 0.31904277205467224,
|
|
"loss/reference_anchor": 0.3693930506706238,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.221327543258667,
|
|
"rewards/margins": 2.247218608856201,
|
|
"rewards/rejected": 0.9741086959838867,
|
|
"step": 555
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2818199098110199,
|
|
"drift/rejected_abs": 0.1541328728199005,
|
|
"epoch": 0.5350531470201839,
|
|
"grad_norm": 0.609375,
|
|
"learning_rate": 2.075993857337368e-07,
|
|
"logits/chosen": -2.180386781692505,
|
|
"logits/rejected": -2.1680946350097656,
|
|
"logps/chosen": -0.2756226062774658,
|
|
"logps/rejected": -0.27983418107032776,
|
|
"loss": 0.7056413888931274,
|
|
"loss/core": 0.6869414448738098,
|
|
"loss/preference_sft": 0.2756226062774658,
|
|
"loss/reference_anchor": 0.3464350402355194,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.816300868988037,
|
|
"rewards/margins": 1.276263952255249,
|
|
"rewards/rejected": 1.540036916732788,
|
|
"step": 560
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2778550386428833,
|
|
"drift/rejected_abs": 0.16609425842761993,
|
|
"epoch": 0.5398304072614356,
|
|
"grad_norm": 0.3203125,
|
|
"learning_rate": 2.0254972663954353e-07,
|
|
"logits/chosen": -2.0759246349334717,
|
|
"logits/rejected": -2.1022229194641113,
|
|
"logps/chosen": -0.2878085672855377,
|
|
"logps/rejected": -0.3052247166633606,
|
|
"loss": 0.7213435173034668,
|
|
"loss/core": 0.6878103017807007,
|
|
"loss/preference_sft": 0.2878085672855377,
|
|
"loss/reference_anchor": 0.6418825387954712,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.7778897285461426,
|
|
"rewards/margins": 1.1175713539123535,
|
|
"rewards/rejected": 1.6603180170059204,
|
|
"step": 565
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3346030116081238,
|
|
"drift/rejected_abs": 0.16794054210186005,
|
|
"epoch": 0.5446076675026872,
|
|
"grad_norm": 2.421875,
|
|
"learning_rate": 1.9752007262770854e-07,
|
|
"logits/chosen": -2.0751147270202637,
|
|
"logits/rejected": -2.1243863105773926,
|
|
"logps/chosen": -0.28862354159355164,
|
|
"logps/rejected": -0.28968754410743713,
|
|
"loss": 0.7091944813728333,
|
|
"loss/core": 0.6851403713226318,
|
|
"loss/preference_sft": 0.28862354159355164,
|
|
"loss/reference_anchor": 0.45222073793411255,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.3444488048553467,
|
|
"rewards/margins": 1.6680787801742554,
|
|
"rewards/rejected": 1.6763699054718018,
|
|
"step": 570
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18410475552082062,
|
|
"drift/rejected_abs": 0.0659659132361412,
|
|
"epoch": 0.5493849277439389,
|
|
"grad_norm": 0.3046875,
|
|
"learning_rate": 1.9251254420535664e-07,
|
|
"logits/chosen": -2.152061939239502,
|
|
"logits/rejected": -2.1839194297790527,
|
|
"logps/chosen": -0.3009091913700104,
|
|
"logps/rejected": -0.32279980182647705,
|
|
"loss": 0.6969389319419861,
|
|
"loss/core": 0.6871550679206848,
|
|
"loss/preference_sft": 0.3009091913700104,
|
|
"loss/reference_anchor": 0.16558721661567688,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.84026300907135,
|
|
"rewards/margins": 1.2756303548812866,
|
|
"rewards/rejected": 0.5646325349807739,
|
|
"step": 575
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1417587250471115,
|
|
"drift/rejected_abs": 0.09436655044555664,
|
|
"epoch": 0.5541621879851905,
|
|
"grad_norm": 0.1552734375,
|
|
"learning_rate": 1.8752925255144224e-07,
|
|
"logits/chosen": -2.428544044494629,
|
|
"logits/rejected": -2.4844648838043213,
|
|
"logps/chosen": -0.26681989431381226,
|
|
"logps/rejected": -0.28266364336013794,
|
|
"loss": 0.696236252784729,
|
|
"loss/core": 0.690847635269165,
|
|
"loss/preference_sft": 0.26681989431381226,
|
|
"loss/reference_anchor": 0.08109016716480255,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.4170500040054321,
|
|
"rewards/margins": 0.47523316740989685,
|
|
"rewards/rejected": 0.9418169260025024,
|
|
"step": 580
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22228363156318665,
|
|
"drift/rejected_abs": 0.13576579093933105,
|
|
"epoch": 0.5589394482264421,
|
|
"grad_norm": 0.5390625,
|
|
"learning_rate": 1.8257229862667433e-07,
|
|
"logits/chosen": -2.1347758769989014,
|
|
"logits/rejected": -2.0975234508514404,
|
|
"logps/chosen": -0.2703164219856262,
|
|
"logps/rejected": -0.28436440229415894,
|
|
"loss": 0.7252476811408997,
|
|
"loss/core": 0.6889234781265259,
|
|
"loss/preference_sft": 0.2703164219856262,
|
|
"loss/reference_anchor": 0.6994535326957703,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.221714496612549,
|
|
"rewards/margins": 0.8691756129264832,
|
|
"rewards/rejected": 1.3525389432907104,
|
|
"step": 585
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3615594506263733,
|
|
"drift/rejected_abs": 0.10348814725875854,
|
|
"epoch": 0.5637167084676937,
|
|
"grad_norm": 0.2373046875,
|
|
"learning_rate": 1.7764377228774873e-07,
|
|
"logits/chosen": -2.0233283042907715,
|
|
"logits/rejected": -2.119215250015259,
|
|
"logps/chosen": -0.28815799951553345,
|
|
"logps/rejected": -0.2871782183647156,
|
|
"loss": 0.7047743797302246,
|
|
"loss/core": 0.6811040639877319,
|
|
"loss/preference_sft": 0.28815799951553345,
|
|
"loss/reference_anchor": 0.44459062814712524,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.6147727966308594,
|
|
"rewards/margins": 2.584536075592041,
|
|
"rewards/rejected": 1.0302363634109497,
|
|
"step": 590
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17077796161174774,
|
|
"drift/rejected_abs": 0.08986127376556396,
|
|
"epoch": 0.5684939687089454,
|
|
"grad_norm": 0.43359375,
|
|
"learning_rate": 1.7274575140626315e-07,
|
|
"logits/chosen": -2.044630765914917,
|
|
"logits/rejected": -2.03894305229187,
|
|
"logps/chosen": -0.27572837471961975,
|
|
"logps/rejected": -0.2722456455230713,
|
|
"loss": 0.6933053135871887,
|
|
"loss/core": 0.6890828013420105,
|
|
"loss/preference_sft": 0.27572837471961975,
|
|
"loss/reference_anchor": 0.056876588612794876,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.705609917640686,
|
|
"rewards/margins": 0.8199650049209595,
|
|
"rewards/rejected": 0.8856450319290161,
|
|
"step": 595
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18729227781295776,
|
|
"drift/rejected_abs": 0.10635222494602203,
|
|
"epoch": 0.5732712289501971,
|
|
"grad_norm": 0.703125,
|
|
"learning_rate": 1.6788030099268407e-07,
|
|
"logits/chosen": -2.2757256031036377,
|
|
"logits/rejected": -2.265326738357544,
|
|
"logps/chosen": -0.26656943559646606,
|
|
"logps/rejected": -0.28453245759010315,
|
|
"loss": 0.6980012655258179,
|
|
"loss/core": 0.6892417073249817,
|
|
"loss/preference_sft": 0.26656943559646606,
|
|
"loss/reference_anchor": 0.14853349328041077,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.8706554174423218,
|
|
"rewards/margins": 0.809914767742157,
|
|
"rewards/rejected": 1.0607407093048096,
|
|
"step": 600
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.41853857040405273,
|
|
"drift/rejected_abs": 0.12051737308502197,
|
|
"epoch": 0.5780484891914487,
|
|
"grad_norm": 1.2265625,
|
|
"learning_rate": 1.630494723257363e-07,
|
|
"logits/chosen": -2.1154239177703857,
|
|
"logits/rejected": -2.1893069744110107,
|
|
"logps/chosen": -0.28015339374542236,
|
|
"logps/rejected": -0.27586129307746887,
|
|
"loss": 0.7251626253128052,
|
|
"loss/core": 0.6798030138015747,
|
|
"loss/preference_sft": 0.28015339374542236,
|
|
"loss/reference_anchor": 0.8791759610176086,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.183618068695068,
|
|
"rewards/margins": 2.979365110397339,
|
|
"rewards/rejected": 1.2042533159255981,
|
|
"step": 605
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4256836473941803,
|
|
"drift/rejected_abs": 0.19338993728160858,
|
|
"epoch": 0.5828257494327004,
|
|
"grad_norm": 1.65625,
|
|
"learning_rate": 1.5825530208758198e-07,
|
|
"logits/chosen": -2.1129722595214844,
|
|
"logits/rejected": -2.1302802562713623,
|
|
"logps/chosen": -0.2779896557331085,
|
|
"logps/rejected": -0.27780717611312866,
|
|
"loss": 0.7297109961509705,
|
|
"loss/core": 0.6822241544723511,
|
|
"loss/preference_sft": 0.2779896557331085,
|
|
"loss/reference_anchor": 0.9219374656677246,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 4.254659175872803,
|
|
"rewards/margins": 2.343705177307129,
|
|
"rewards/rejected": 1.9109541177749634,
|
|
"step": 610
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2733246088027954,
|
|
"drift/rejected_abs": 0.14853966236114502,
|
|
"epoch": 0.587603009673952,
|
|
"grad_norm": 0.5,
|
|
"learning_rate": 1.534998115051533e-07,
|
|
"logits/chosen": -1.9654070138931274,
|
|
"logits/rejected": -2.0324883460998535,
|
|
"logps/chosen": -0.31496065855026245,
|
|
"logps/rejected": -0.30436021089553833,
|
|
"loss": 0.7030231952667236,
|
|
"loss/core": 0.6871589422225952,
|
|
"loss/preference_sft": 0.31496065855026245,
|
|
"loss/reference_anchor": 0.2857890725135803,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.7304234504699707,
|
|
"rewards/margins": 1.2481533288955688,
|
|
"rewards/rejected": 1.4822700023651123,
|
|
"step": 615
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1698610484600067,
|
|
"drift/rejected_abs": 0.14360573887825012,
|
|
"epoch": 0.5923802699152036,
|
|
"grad_norm": 0.373046875,
|
|
"learning_rate": 1.4878500549800112e-07,
|
|
"logits/chosen": -2.1398520469665527,
|
|
"logits/rejected": -2.166027069091797,
|
|
"logps/chosen": -0.28533971309661865,
|
|
"logps/rejected": -0.29425957798957825,
|
|
"loss": 0.7028168439865112,
|
|
"loss/core": 0.6921274662017822,
|
|
"loss/preference_sft": 0.28533971309661865,
|
|
"loss/reference_anchor": 0.18525299429893494,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.6973737478256226,
|
|
"rewards/margins": 0.26329460740089417,
|
|
"rewards/rejected": 1.4340791702270508,
|
|
"step": 620
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3897588551044464,
|
|
"drift/rejected_abs": 0.17362374067306519,
|
|
"epoch": 0.5971575301564552,
|
|
"grad_norm": 29.375,
|
|
"learning_rate": 1.4411287183301902e-07,
|
|
"logits/chosen": -1.9303457736968994,
|
|
"logits/rejected": -1.958958625793457,
|
|
"logps/chosen": -0.30389657616615295,
|
|
"logps/rejected": -0.347369909286499,
|
|
"loss": 0.7363072633743286,
|
|
"loss/core": 0.6822855472564697,
|
|
"loss/preference_sft": 0.30389657616615295,
|
|
"loss/reference_anchor": 1.0500433444976807,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.895702362060547,
|
|
"rewards/margins": 2.4964680671691895,
|
|
"rewards/rejected": 1.3992340564727783,
|
|
"step": 625
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.29475387930870056,
|
|
"drift/rejected_abs": 0.12502244114875793,
|
|
"epoch": 0.6019347903977069,
|
|
"grad_norm": 5.09375,
|
|
"learning_rate": 1.394853802863985e-07,
|
|
"logits/chosen": -1.929779052734375,
|
|
"logits/rejected": -2.04491925239563,
|
|
"logps/chosen": -0.27201491594314575,
|
|
"logps/rejected": -0.2829875349998474,
|
|
"loss": 0.7087033987045288,
|
|
"loss/core": 0.685075581073761,
|
|
"loss/preference_sft": 0.27201491594314575,
|
|
"loss/reference_anchor": 0.44535598158836365,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.9475390911102295,
|
|
"rewards/margins": 1.699634313583374,
|
|
"rewards/rejected": 1.2479045391082764,
|
|
"step": 630
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2301911860704422,
|
|
"drift/rejected_abs": 0.10143622010946274,
|
|
"epoch": 0.6067120506389586,
|
|
"grad_norm": 4.9375,
|
|
"learning_rate": 1.3490448181317024e-07,
|
|
"logits/chosen": -1.981834053993225,
|
|
"logits/rejected": -2.0678226947784424,
|
|
"logps/chosen": -0.3054197430610657,
|
|
"logps/rejected": -0.30018460750579834,
|
|
"loss": 0.6994880437850952,
|
|
"loss/core": 0.687004566192627,
|
|
"loss/preference_sft": 0.3054197430610657,
|
|
"loss/reference_anchor": 0.21912848949432373,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.3007302284240723,
|
|
"rewards/margins": 1.2872453927993774,
|
|
"rewards/rejected": 1.0134851932525635,
|
|
"step": 635
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1967557668685913,
|
|
"drift/rejected_abs": 0.07957569509744644,
|
|
"epoch": 0.6114893108802102,
|
|
"grad_norm": 0.25390625,
|
|
"learning_rate": 1.303721077246784e-07,
|
|
"logits/chosen": -2.207252264022827,
|
|
"logits/rejected": -2.2832891941070557,
|
|
"logps/chosen": -0.30064091086387634,
|
|
"logps/rejected": -0.29895979166030884,
|
|
"loss": 0.6989712119102478,
|
|
"loss/core": 0.6875656843185425,
|
|
"loss/preference_sft": 0.30064091086387634,
|
|
"loss/reference_anchor": 0.19804587960243225,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.9660539627075195,
|
|
"rewards/margins": 1.1706269979476929,
|
|
"rewards/rejected": 0.7954270839691162,
|
|
"step": 640
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16765008866786957,
|
|
"drift/rejected_abs": 0.11835210025310516,
|
|
"epoch": 0.6162665711214619,
|
|
"grad_norm": 0.86328125,
|
|
"learning_rate": 1.2589016887433846e-07,
|
|
"logits/chosen": -2.1545424461364746,
|
|
"logits/rejected": -2.1316208839416504,
|
|
"logps/chosen": -0.29050302505493164,
|
|
"logps/rejected": -0.3003830313682556,
|
|
"loss": 0.6977421045303345,
|
|
"loss/core": 0.6908085942268372,
|
|
"loss/preference_sft": 0.29050302505493164,
|
|
"loss/reference_anchor": 0.10962003469467163,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.6752145290374756,
|
|
"rewards/margins": 0.49554985761642456,
|
|
"rewards/rejected": 1.1796646118164062,
|
|
"step": 645
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.13382819294929504,
|
|
"drift/rejected_abs": 0.0856524258852005,
|
|
"epoch": 0.6210438313627135,
|
|
"grad_norm": 0.3046875,
|
|
"learning_rate": 1.2146055485201943e-07,
|
|
"logits/chosen": -2.22021222114563,
|
|
"logits/rejected": -2.171934127807617,
|
|
"logps/chosen": -0.28915685415267944,
|
|
"logps/rejected": -0.29866427183151245,
|
|
"loss": 0.6960299015045166,
|
|
"loss/core": 0.6908055543899536,
|
|
"loss/preference_sft": 0.28915685415267944,
|
|
"loss/reference_anchor": 0.075570248067379,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.3368127346038818,
|
|
"rewards/margins": 0.4924766421318054,
|
|
"rewards/rejected": 0.844336211681366,
|
|
"step": 650
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20044943690299988,
|
|
"drift/rejected_abs": 0.09594675153493881,
|
|
"epoch": 0.6258210916039652,
|
|
"grad_norm": 2.671875,
|
|
"learning_rate": 1.1708513318739094e-07,
|
|
"logits/chosen": -2.0582327842712402,
|
|
"logits/rejected": -2.1220715045928955,
|
|
"logps/chosen": -0.282463014125824,
|
|
"logps/rejected": -0.29273414611816406,
|
|
"loss": 0.6957246661186218,
|
|
"loss/core": 0.6880175471305847,
|
|
"loss/preference_sft": 0.282463014125824,
|
|
"loss/reference_anchor": 0.125896617770195,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.002502918243408,
|
|
"rewards/margins": 1.0442477464675903,
|
|
"rewards/rejected": 0.9582554697990417,
|
|
"step": 655
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27859437465667725,
|
|
"drift/rejected_abs": 0.1346025913953781,
|
|
"epoch": 0.6305983518452167,
|
|
"grad_norm": 0.2265625,
|
|
"learning_rate": 1.1276574856257095e-07,
|
|
"logits/chosen": -2.0280890464782715,
|
|
"logits/rejected": -2.0235238075256348,
|
|
"logps/chosen": -0.29317161440849304,
|
|
"logps/rejected": -0.3068060576915741,
|
|
"loss": 0.7208054065704346,
|
|
"loss/core": 0.6866048574447632,
|
|
"loss/preference_sft": 0.29317161440849304,
|
|
"loss/reference_anchor": 0.6546904444694519,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.784419536590576,
|
|
"rewards/margins": 1.4846069812774658,
|
|
"rewards/rejected": 1.2998125553131104,
|
|
"step": 660
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23545166850090027,
|
|
"drift/rejected_abs": 0.14730089902877808,
|
|
"epoch": 0.6353756120864684,
|
|
"grad_norm": 2.484375,
|
|
"learning_rate": 1.0850422203440554e-07,
|
|
"logits/chosen": -2.188843250274658,
|
|
"logits/rejected": -2.1637561321258545,
|
|
"logps/chosen": -0.2837633192539215,
|
|
"logps/rejected": -0.2823261618614197,
|
|
"loss": 0.7029173374176025,
|
|
"loss/core": 0.6890594363212585,
|
|
"loss/preference_sft": 0.2837633192539215,
|
|
"loss/reference_anchor": 0.24877965450286865,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.353299379348755,
|
|
"rewards/margins": 0.8823181390762329,
|
|
"rewards/rejected": 1.470981240272522,
|
|
"step": 665
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3646673560142517,
|
|
"drift/rejected_abs": 0.12857022881507874,
|
|
"epoch": 0.64015287232772,
|
|
"grad_norm": 0.2197265625,
|
|
"learning_rate": 1.0430235026670978e-07,
|
|
"logits/chosen": -1.9835231304168701,
|
|
"logits/rejected": -2.0292749404907227,
|
|
"logps/chosen": -0.27422162890434265,
|
|
"logps/rejected": -0.32053428888320923,
|
|
"loss": 0.7076139450073242,
|
|
"loss/core": 0.6819421648979187,
|
|
"loss/preference_sft": 0.27422162890434265,
|
|
"loss/reference_anchor": 0.4860144555568695,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.6459412574768066,
|
|
"rewards/margins": 2.392371654510498,
|
|
"rewards/rejected": 1.2535693645477295,
|
|
"step": 670
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.162667378783226,
|
|
"drift/rejected_abs": 0.0890541672706604,
|
|
"epoch": 0.6449301325689717,
|
|
"grad_norm": 0.73046875,
|
|
"learning_rate": 1.0016190477279274e-07,
|
|
"logits/chosen": -2.0347633361816406,
|
|
"logits/rejected": -2.0776588916778564,
|
|
"logps/chosen": -0.2881300151348114,
|
|
"logps/rejected": -0.2884693443775177,
|
|
"loss": 0.6940901875495911,
|
|
"loss/core": 0.6894824504852295,
|
|
"loss/preference_sft": 0.2881300151348114,
|
|
"loss/reference_anchor": 0.06333953142166138,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.6251804828643799,
|
|
"rewards/margins": 0.74686199426651,
|
|
"rewards/rejected": 0.8783184885978699,
|
|
"step": 675
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21540336310863495,
|
|
"drift/rejected_abs": 0.1353510469198227,
|
|
"epoch": 0.6497073928102234,
|
|
"grad_norm": 0.2265625,
|
|
"learning_rate": 9.608463116858542e-08,
|
|
"logits/chosen": -2.1595706939697266,
|
|
"logits/rejected": -2.077641725540161,
|
|
"logps/chosen": -0.2814193069934845,
|
|
"logps/rejected": -0.29484206438064575,
|
|
"loss": 0.7012907266616821,
|
|
"loss/core": 0.6893535852432251,
|
|
"loss/preference_sft": 0.2814193069934845,
|
|
"loss/reference_anchor": 0.21059997379779816,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.1523005962371826,
|
|
"rewards/margins": 0.7994104623794556,
|
|
"rewards/rejected": 1.3528902530670166,
|
|
"step": 680
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28880634903907776,
|
|
"drift/rejected_abs": 0.15391790866851807,
|
|
"epoch": 0.654484653051475,
|
|
"grad_norm": 0.30078125,
|
|
"learning_rate": 9.207224843668731e-08,
|
|
"logits/chosen": -2.0853352546691895,
|
|
"logits/rejected": -2.0695528984069824,
|
|
"logps/chosen": -0.25348979234695435,
|
|
"logps/rejected": -0.27719801664352417,
|
|
"loss": 0.7120725512504578,
|
|
"loss/core": 0.6871707439422607,
|
|
"loss/preference_sft": 0.25348979234695435,
|
|
"loss/reference_anchor": 0.47268733382225037,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.887314558029175,
|
|
"rewards/margins": 1.3541991710662842,
|
|
"rewards/rejected": 1.5331155061721802,
|
|
"step": 685
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17689254879951477,
|
|
"drift/rejected_abs": 0.10154334455728531,
|
|
"epoch": 0.6592619132927267,
|
|
"grad_norm": 0.26171875,
|
|
"learning_rate": 8.812644820164158e-08,
|
|
"logits/chosen": -1.8848329782485962,
|
|
"logits/rejected": -1.9065767526626587,
|
|
"logps/chosen": -0.30832767486572266,
|
|
"logps/rejected": -0.33626437187194824,
|
|
"loss": 0.6952409744262695,
|
|
"loss/core": 0.6894530057907104,
|
|
"loss/preference_sft": 0.30832767486572266,
|
|
"loss/reference_anchor": 0.08492577075958252,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.7667601108551025,
|
|
"rewards/margins": 0.7572234869003296,
|
|
"rewards/rejected": 1.0095365047454834,
|
|
"step": 690
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19672070443630219,
|
|
"drift/rejected_abs": 0.06929643452167511,
|
|
"epoch": 0.6640391735339782,
|
|
"grad_norm": 0.2197265625,
|
|
"learning_rate": 8.424889401674504e-08,
|
|
"logits/chosen": -2.270711660385132,
|
|
"logits/rejected": -2.279158115386963,
|
|
"logps/chosen": -0.3068528175354004,
|
|
"logps/rejected": -0.29276537895202637,
|
|
"loss": 0.6979945302009583,
|
|
"loss/core": 0.6870979070663452,
|
|
"loss/preference_sft": 0.3068528175354004,
|
|
"loss/reference_anchor": 0.18724682927131653,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.9648374319076538,
|
|
"rewards/margins": 1.2810237407684326,
|
|
"rewards/rejected": 0.6838136911392212,
|
|
"step": 695
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22341635823249817,
|
|
"drift/rejected_abs": 0.15612784028053284,
|
|
"epoch": 0.6688164337752299,
|
|
"grad_norm": 16.875,
|
|
"learning_rate": 8.044122066269149e-08,
|
|
"logits/chosen": -2.1035265922546387,
|
|
"logits/rejected": -2.2100517749786377,
|
|
"logps/chosen": -0.2964898943901062,
|
|
"logps/rejected": -0.32455042004585266,
|
|
"loss": 0.7034405469894409,
|
|
"loss/core": 0.6898477673530579,
|
|
"loss/preference_sft": 0.2964898943901062,
|
|
"loss/reference_anchor": 0.2422069013118744,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.2336838245391846,
|
|
"rewards/margins": 0.6977290511131287,
|
|
"rewards/rejected": 1.5359545946121216,
|
|
"step": 700
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3399743437767029,
|
|
"drift/rejected_abs": 0.1532609462738037,
|
|
"epoch": 0.6735936940164815,
|
|
"grad_norm": 1.3984375,
|
|
"learning_rate": 7.670503345834755e-08,
|
|
"logits/chosen": -2.1955857276916504,
|
|
"logits/rejected": -2.152186870574951,
|
|
"logps/chosen": -0.28090688586235046,
|
|
"logps/rejected": -0.26746469736099243,
|
|
"loss": 0.7058395743370056,
|
|
"loss/core": 0.6843441724777222,
|
|
"loss/preference_sft": 0.28090688586235046,
|
|
"loss/reference_anchor": 0.40181976556777954,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.399095058441162,
|
|
"rewards/margins": 1.8677432537078857,
|
|
"rewards/rejected": 1.531352162361145,
|
|
"step": 705
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.35864269733428955,
|
|
"drift/rejected_abs": 0.18870094418525696,
|
|
"epoch": 0.6783709542577332,
|
|
"grad_norm": 12.5,
|
|
"learning_rate": 7.304190758394774e-08,
|
|
"logits/chosen": -1.886185884475708,
|
|
"logits/rejected": -1.873063325881958,
|
|
"logps/chosen": -0.2785230278968811,
|
|
"logps/rejected": -0.2990533411502838,
|
|
"loss": 0.7196978330612183,
|
|
"loss/core": 0.6850727796554565,
|
|
"loss/preference_sft": 0.2785230278968811,
|
|
"loss/reference_anchor": 0.6646484136581421,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.5836358070373535,
|
|
"rewards/margins": 1.6989152431488037,
|
|
"rewards/rejected": 1.884720802307129,
|
|
"step": 710
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18634703755378723,
|
|
"drift/rejected_abs": 0.09215612709522247,
|
|
"epoch": 0.6831482144989849,
|
|
"grad_norm": 0.330078125,
|
|
"learning_rate": 6.945338741699769e-08,
|
|
"logits/chosen": -2.252730131149292,
|
|
"logits/rejected": -2.2337136268615723,
|
|
"logps/chosen": -0.2843503952026367,
|
|
"logps/rejected": -0.3452153205871582,
|
|
"loss": 0.7017387747764587,
|
|
"loss/core": 0.6871799826622009,
|
|
"loss/preference_sft": 0.2843503952026367,
|
|
"loss/reference_anchor": 0.26274123787879944,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.863227128982544,
|
|
"rewards/margins": 1.2492995262145996,
|
|
"rewards/rejected": 0.6139274835586548,
|
|
"step": 715
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20980620384216309,
|
|
"drift/rejected_abs": 0.14841270446777344,
|
|
"epoch": 0.6879254747402365,
|
|
"grad_norm": 0.4921875,
|
|
"learning_rate": 6.594098588116243e-08,
|
|
"logits/chosen": -1.9954087734222412,
|
|
"logits/rejected": -2.003382921218872,
|
|
"logps/chosen": -0.29687029123306274,
|
|
"logps/rejected": -0.30213814973831177,
|
|
"loss": 0.7040995955467224,
|
|
"loss/core": 0.6903706789016724,
|
|
"loss/preference_sft": 0.29687029123306274,
|
|
"loss/reference_anchor": 0.24489395320415497,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.096773624420166,
|
|
"rewards/margins": 0.6157442927360535,
|
|
"rewards/rejected": 1.4810290336608887,
|
|
"step": 720
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1588420420885086,
|
|
"drift/rejected_abs": 0.0730825662612915,
|
|
"epoch": 0.6927027349814882,
|
|
"grad_norm": 0.3203125,
|
|
"learning_rate": 6.25061838084166e-08,
|
|
"logits/chosen": -2.1225600242614746,
|
|
"logits/rejected": -2.238051176071167,
|
|
"logps/chosen": -0.2854083180427551,
|
|
"logps/rejected": -0.277433305978775,
|
|
"loss": 0.6947664022445679,
|
|
"loss/core": 0.6889495253562927,
|
|
"loss/preference_sft": 0.2854083180427551,
|
|
"loss/reference_anchor": 0.08779795467853546,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.5876481533050537,
|
|
"rewards/margins": 0.8577953577041626,
|
|
"rewards/rejected": 0.7298528552055359,
|
|
"step": 725
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.10160217434167862,
|
|
"drift/rejected_abs": 0.08643589913845062,
|
|
"epoch": 0.6974799952227397,
|
|
"grad_norm": 13.8125,
|
|
"learning_rate": 5.915042931472425e-08,
|
|
"logits/chosen": -2.218738317489624,
|
|
"logits/rejected": -2.2675373554229736,
|
|
"logps/chosen": -0.28681716322898865,
|
|
"logps/rejected": -0.2953147888183594,
|
|
"loss": 0.6986241340637207,
|
|
"loss/core": 0.6925312876701355,
|
|
"loss/preference_sft": 0.28681716322898865,
|
|
"loss/reference_anchor": 0.09317411482334137,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.015641689300537,
|
|
"rewards/margins": 0.15372122824192047,
|
|
"rewards/rejected": 0.8619204759597778,
|
|
"step": 730
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14239996671676636,
|
|
"drift/rejected_abs": 0.06578416377305984,
|
|
"epoch": 0.7022572554639914,
|
|
"grad_norm": 1.5625,
|
|
"learning_rate": 5.587513718951165e-08,
|
|
"logits/chosen": -2.047664165496826,
|
|
"logits/rejected": -2.09497332572937,
|
|
"logps/chosen": -0.31937894225120544,
|
|
"logps/rejected": -0.3183499276638031,
|
|
"loss": 0.6945558786392212,
|
|
"loss/core": 0.6893695592880249,
|
|
"loss/preference_sft": 0.31937894225120544,
|
|
"loss/reference_anchor": 0.0717906728386879,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.4239996671676636,
|
|
"rewards/margins": 0.7663625478744507,
|
|
"rewards/rejected": 0.6576371192932129,
|
|
"step": 735
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25790858268737793,
|
|
"drift/rejected_abs": 0.17992377281188965,
|
|
"epoch": 0.707034515705243,
|
|
"grad_norm": 0.34765625,
|
|
"learning_rate": 5.2681688299190455e-08,
|
|
"logits/chosen": -2.1005473136901855,
|
|
"logits/rejected": -2.1174521446228027,
|
|
"logps/chosen": -0.31155890226364136,
|
|
"logps/rejected": -0.31915396451950073,
|
|
"loss": 0.7307167649269104,
|
|
"loss/core": 0.6908460259437561,
|
|
"loss/preference_sft": 0.31155890226364136,
|
|
"loss/reference_anchor": 0.7662578821182251,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.579061985015869,
|
|
"rewards/margins": 0.779824435710907,
|
|
"rewards/rejected": 1.799237608909607,
|
|
"step": 740
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3101605176925659,
|
|
"drift/rejected_abs": 0.1494651436805725,
|
|
"epoch": 0.7118117759464947,
|
|
"grad_norm": 7.6875,
|
|
"learning_rate": 4.957142900498334e-08,
|
|
"logits/chosen": -2.0538952350616455,
|
|
"logits/rejected": -2.0975911617279053,
|
|
"logps/chosen": -0.29252904653549194,
|
|
"logps/rejected": -0.28451254963874817,
|
|
"loss": 0.7017096281051636,
|
|
"loss/core": 0.6854857206344604,
|
|
"loss/preference_sft": 0.29252904653549194,
|
|
"loss/reference_anchor": 0.29522526264190674,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.100473403930664,
|
|
"rewards/margins": 1.6103368997573853,
|
|
"rewards/rejected": 1.4901363849639893,
|
|
"step": 745
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1206597089767456,
|
|
"drift/rejected_abs": 0.12509042024612427,
|
|
"epoch": 0.7165890361877463,
|
|
"grad_norm": 9.8125,
|
|
"learning_rate": 4.654567059529668e-08,
|
|
"logits/chosen": -2.1884284019470215,
|
|
"logits/rejected": -2.180311679840088,
|
|
"logps/chosen": -0.31168606877326965,
|
|
"logps/rejected": -0.30455487966537476,
|
|
"loss": 0.7092859745025635,
|
|
"loss/core": 0.6938542127609253,
|
|
"loss/preference_sft": 0.31168606877326965,
|
|
"loss/reference_anchor": 0.27746719121932983,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.206160306930542,
|
|
"rewards/margins": -0.0409439317882061,
|
|
"rewards/rejected": 1.2471041679382324,
|
|
"step": 750
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23872093856334686,
|
|
"drift/rejected_abs": 0.17086395621299744,
|
|
"epoch": 0.721366296428998,
|
|
"grad_norm": 0.3515625,
|
|
"learning_rate": 4.360568873288009e-08,
|
|
"logits/chosen": -1.9273812770843506,
|
|
"logits/rejected": -1.8365058898925781,
|
|
"logps/chosen": -0.3087690770626068,
|
|
"logps/rejected": -0.29509294033050537,
|
|
"loss": 0.7011326551437378,
|
|
"loss/core": 0.6899515390396118,
|
|
"loss/preference_sft": 0.3087690770626068,
|
|
"loss/reference_anchor": 0.19274473190307617,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.384294033050537,
|
|
"rewards/margins": 0.682456374168396,
|
|
"rewards/rejected": 1.7018375396728516,
|
|
"step": 755
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19956620037555695,
|
|
"drift/rejected_abs": 0.08658763021230698,
|
|
"epoch": 0.7261435566702497,
|
|
"grad_norm": 0.33984375,
|
|
"learning_rate": 4.075272291700557e-08,
|
|
"logits/chosen": -2.1839423179626465,
|
|
"logits/rejected": -2.2009308338165283,
|
|
"logps/chosen": -0.30236124992370605,
|
|
"logps/rejected": -0.29375168681144714,
|
|
"loss": 0.6960939168930054,
|
|
"loss/core": 0.6876411437988281,
|
|
"loss/preference_sft": 0.30236124992370605,
|
|
"loss/reference_anchor": 0.13882040977478027,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.9942079782485962,
|
|
"rewards/margins": 1.1320140361785889,
|
|
"rewards/rejected": 0.8621941804885864,
|
|
"step": 760
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17910103499889374,
|
|
"drift/rejected_abs": 0.10586283355951309,
|
|
"epoch": 0.7309208169115012,
|
|
"grad_norm": 0.69921875,
|
|
"learning_rate": 3.798797596089351e-08,
|
|
"logits/chosen": -2.0851759910583496,
|
|
"logits/rejected": -2.157111644744873,
|
|
"logps/chosen": -0.2764049172401428,
|
|
"logps/rejected": -0.27264276146888733,
|
|
"loss": 0.6968425512313843,
|
|
"loss/core": 0.6895520091056824,
|
|
"loss/preference_sft": 0.2764049172401428,
|
|
"loss/reference_anchor": 0.1181713119149208,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.790848970413208,
|
|
"rewards/margins": 0.7349028587341309,
|
|
"rewards/rejected": 1.0559461116790771,
|
|
"step": 765
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.36868372559547424,
|
|
"drift/rejected_abs": 0.23421208560466766,
|
|
"epoch": 0.7356980771527529,
|
|
"grad_norm": 0.326171875,
|
|
"learning_rate": 3.531261348460554e-08,
|
|
"logits/chosen": -1.9794762134552002,
|
|
"logits/rejected": -1.945230484008789,
|
|
"logps/chosen": -0.31459537148475647,
|
|
"logps/rejected": -0.28590553998947144,
|
|
"loss": 0.7430247664451599,
|
|
"loss/core": 0.6873718500137329,
|
|
"loss/preference_sft": 0.31459537148475647,
|
|
"loss/reference_anchor": 1.0815989971160889,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 3.6851744651794434,
|
|
"rewards/margins": 1.344879388809204,
|
|
"rewards/rejected": 2.3402953147888184,
|
|
"step": 770
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15195751190185547,
|
|
"drift/rejected_abs": 0.09389922767877579,
|
|
"epoch": 0.7404753373940045,
|
|
"grad_norm": 0.2109375,
|
|
"learning_rate": 3.272776342361791e-08,
|
|
"logits/chosen": -2.0958571434020996,
|
|
"logits/rejected": -2.123683452606201,
|
|
"logps/chosen": -0.30696946382522583,
|
|
"logps/rejected": -0.3006220757961273,
|
|
"loss": 0.6954344511032104,
|
|
"loss/core": 0.6903113722801208,
|
|
"loss/preference_sft": 0.30696946382522583,
|
|
"loss/reference_anchor": 0.07176429778337479,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 1.519211769104004,
|
|
"rewards/margins": 0.5804935693740845,
|
|
"rewards/rejected": 0.9387180209159851,
|
|
"step": 775
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18850034475326538,
|
|
"drift/rejected_abs": 0.12492908537387848,
|
|
"epoch": 0.7452525976352562,
|
|
"grad_norm": 0.359375,
|
|
"learning_rate": 3.023451555328252e-08,
|
|
"logits/chosen": -2.181946277618408,
|
|
"logits/rejected": -2.1955080032348633,
|
|
"logps/chosen": -0.30274689197540283,
|
|
"logps/rejected": -0.30594930052757263,
|
|
"loss": 0.6972261667251587,
|
|
"loss/core": 0.6900213956832886,
|
|
"loss/preference_sft": 0.30274689197540283,
|
|
"loss/reference_anchor": 0.11382057517766953,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.8849332332611084,
|
|
"rewards/margins": 0.6389666795730591,
|
|
"rewards/rejected": 1.2459665536880493,
|
|
"step": 780
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3422924876213074,
|
|
"drift/rejected_abs": 0.13462868332862854,
|
|
"epoch": 0.7500298578765078,
|
|
"grad_norm": 0.240234375,
|
|
"learning_rate": 2.7833921029376817e-08,
|
|
"logits/chosen": -1.9921424388885498,
|
|
"logits/rejected": -1.949812650680542,
|
|
"logps/chosen": -0.3121167719364166,
|
|
"logps/rejected": -0.2942814230918884,
|
|
"loss": 0.7160924673080444,
|
|
"loss/core": 0.6839752197265625,
|
|
"loss/preference_sft": 0.3121167719364166,
|
|
"loss/reference_anchor": 0.6111334562301636,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.419778347015381,
|
|
"rewards/margins": 2.079742431640625,
|
|
"rewards/rejected": 1.3400366306304932,
|
|
"step": 785
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26127004623413086,
|
|
"drift/rejected_abs": 0.19187363982200623,
|
|
"epoch": 0.7548071181177595,
|
|
"grad_norm": 11.3125,
|
|
"learning_rate": 2.5526991944935488e-08,
|
|
"logits/chosen": -2.168665647506714,
|
|
"logits/rejected": -1.9961109161376953,
|
|
"logps/chosen": -0.2868359684944153,
|
|
"logps/rejected": -0.3195565342903137,
|
|
"loss": 0.720409095287323,
|
|
"loss/core": 0.6900395750999451,
|
|
"loss/preference_sft": 0.2868359684944153,
|
|
"loss/reference_anchor": 0.5787058472633362,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.6118552684783936,
|
|
"rewards/margins": 0.6948817372322083,
|
|
"rewards/rejected": 1.916973352432251,
|
|
"step": 790
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23737922310829163,
|
|
"drift/rejected_abs": 0.11862516403198242,
|
|
"epoch": 0.7595843783590112,
|
|
"grad_norm": 0.5,
|
|
"learning_rate": 2.3314700903550838e-08,
|
|
"logits/chosen": -1.9713795185089111,
|
|
"logits/rejected": -1.989896535873413,
|
|
"logps/chosen": -0.3081986904144287,
|
|
"logps/rejected": -0.29614824056625366,
|
|
"loss": 0.7151690721511841,
|
|
"loss/core": 0.6874554753303528,
|
|
"loss/preference_sft": 0.3081986904144287,
|
|
"loss/reference_anchor": 0.5234512090682983,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.3726906776428223,
|
|
"rewards/margins": 1.1869637966156006,
|
|
"rewards/rejected": 1.185726523399353,
|
|
"step": 795
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22930972278118134,
|
|
"drift/rejected_abs": 0.11694236099720001,
|
|
"epoch": 0.7643616386002627,
|
|
"grad_norm": 0.31640625,
|
|
"learning_rate": 2.11979806093224e-08,
|
|
"logits/chosen": -2.1524062156677246,
|
|
"logits/rejected": -2.1292905807495117,
|
|
"logps/chosen": -0.28981950879096985,
|
|
"logps/rejected": -0.2884742319583893,
|
|
"loss": 0.696781575679779,
|
|
"loss/core": 0.6877356767654419,
|
|
"loss/preference_sft": 0.28981950879096985,
|
|
"loss/reference_anchor": 0.1519363820552826,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.2911429405212402,
|
|
"rewards/margins": 1.1217482089996338,
|
|
"rewards/rejected": 1.1693947315216064,
|
|
"step": 800
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22585976123809814,
|
|
"drift/rejected_abs": 0.19897663593292236,
|
|
"epoch": 0.7691388988415144,
|
|
"grad_norm": 0.73828125,
|
|
"learning_rate": 1.9177723473627643e-08,
|
|
"logits/chosen": -2.0267646312713623,
|
|
"logits/rejected": -2.014665126800537,
|
|
"logps/chosen": -0.27477264404296875,
|
|
"logps/rejected": -0.2703710198402405,
|
|
"loss": 0.706299901008606,
|
|
"loss/core": 0.6920345425605774,
|
|
"loss/preference_sft": 0.27477264404296875,
|
|
"loss/reference_anchor": 0.2578306198120117,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 2.2579760551452637,
|
|
"rewards/margins": 0.2697838544845581,
|
|
"rewards/rejected": 1.9881923198699951,
|
|
"step": 805
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15302561223506927,
|
|
"drift/rejected_abs": 0.14136184751987457,
|
|
"epoch": 0.773916159082766,
|
|
"grad_norm": 0.404296875,
|
|
"learning_rate": 1.7254781238880828e-08,
|
|
"logits/chosen": -2.2281618118286133,
|
|
"logits/rejected": -2.132458209991455,
|
|
"logps/chosen": -0.30843856930732727,
|
|
"logps/rejected": -0.32327035069465637,
|
|
"loss": 0.7003055810928345,
|
|
"loss/core": 0.6927086114883423,
|
|
"loss/preference_sft": 0.30843856930732727,
|
|
"loss/reference_anchor": 0.12109792232513428,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.5288832187652588,
|
|
"rewards/margins": 0.11678824573755264,
|
|
"rewards/rejected": 1.4120948314666748,
|
|
"step": 810
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2130151093006134,
|
|
"drift/rejected_abs": 0.11095918715000153,
|
|
"epoch": 0.7786934193240177,
|
|
"grad_norm": 1.53125,
|
|
"learning_rate": 1.5429964619437158e-08,
|
|
"logits/chosen": -2.0964717864990234,
|
|
"logits/rejected": -2.1862375736236572,
|
|
"logps/chosen": -0.2831994891166687,
|
|
"logps/rejected": -0.28318652510643005,
|
|
"loss": 0.6956707239151001,
|
|
"loss/core": 0.6880777478218079,
|
|
"loss/preference_sft": 0.2831994891166687,
|
|
"loss/reference_anchor": 0.1235402375459671,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.1292998790740967,
|
|
"rewards/margins": 1.024322509765625,
|
|
"rewards/rejected": 1.1049773693084717,
|
|
"step": 815
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19676700234413147,
|
|
"drift/rejected_abs": 0.0735253393650055,
|
|
"epoch": 0.7834706795652693,
|
|
"grad_norm": 0.2158203125,
|
|
"learning_rate": 1.3704042959795131e-08,
|
|
"logits/chosen": -2.1000118255615234,
|
|
"logits/rejected": -2.133841037750244,
|
|
"logps/chosen": -0.28775379061698914,
|
|
"logps/rejected": -0.3174736201763153,
|
|
"loss": 0.6944074034690857,
|
|
"loss/core": 0.6869372129440308,
|
|
"loss/preference_sft": 0.28775379061698914,
|
|
"loss/reference_anchor": 0.12063012272119522,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.9654747247695923,
|
|
"rewards/margins": 1.2821400165557861,
|
|
"rewards/rejected": 0.6833347082138062,
|
|
"step": 820
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24645988643169403,
|
|
"drift/rejected_abs": 0.1214875802397728,
|
|
"epoch": 0.788247939806521,
|
|
"grad_norm": 1.0546875,
|
|
"learning_rate": 1.2077743910239996e-08,
|
|
"logits/chosen": -2.1431891918182373,
|
|
"logits/rejected": -2.247856616973877,
|
|
"logps/chosen": -0.3019647002220154,
|
|
"logps/rejected": -0.28489747643470764,
|
|
"loss": 0.696218729019165,
|
|
"loss/core": 0.6870028972625732,
|
|
"loss/preference_sft": 0.3019647002220154,
|
|
"loss/reference_anchor": 0.15411940217018127,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.4645986557006836,
|
|
"rewards/margins": 1.2513694763183594,
|
|
"rewards/rejected": 1.2132294178009033,
|
|
"step": 825
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17260222136974335,
|
|
"drift/rejected_abs": 0.1575520932674408,
|
|
"epoch": 0.7930252000477725,
|
|
"grad_norm": 0.52734375,
|
|
"learning_rate": 1.0551753120065621e-08,
|
|
"logits/chosen": -2.17891263961792,
|
|
"logits/rejected": -2.12709379196167,
|
|
"logps/chosen": -0.2786734700202942,
|
|
"logps/rejected": -0.31584739685058594,
|
|
"loss": 0.7015553116798401,
|
|
"loss/core": 0.692550539970398,
|
|
"loss/preference_sft": 0.2786734700202942,
|
|
"loss/reference_anchor": 0.15222761034965515,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.724945306777954,
|
|
"rewards/margins": 0.15078657865524292,
|
|
"rewards/rejected": 1.5741586685180664,
|
|
"step": 830
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24012541770935059,
|
|
"drift/rejected_abs": 0.11180107295513153,
|
|
"epoch": 0.7978024602890242,
|
|
"grad_norm": 1.71875,
|
|
"learning_rate": 9.126713948504228e-09,
|
|
"logits/chosen": -2.094947099685669,
|
|
"logits/rejected": -2.0943989753723145,
|
|
"logps/chosen": -0.2551843822002411,
|
|
"logps/rejected": -0.29870468378067017,
|
|
"loss": 0.6966162919998169,
|
|
"loss/core": 0.685853123664856,
|
|
"loss/preference_sft": 0.2551843822002411,
|
|
"loss/reference_anchor": 0.18974518775939941,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.3995418548583984,
|
|
"rewards/margins": 1.5449695587158203,
|
|
"rewards/rejected": 0.8545724749565125,
|
|
"step": 835
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26994588971138,
|
|
"drift/rejected_abs": 0.10432889312505722,
|
|
"epoch": 0.8025797205302759,
|
|
"grad_norm": 61.0,
|
|
"learning_rate": 7.803227193485334e-09,
|
|
"logits/chosen": -2.03291916847229,
|
|
"logits/rejected": -2.0835347175598145,
|
|
"logps/chosen": -0.33308959007263184,
|
|
"logps/rejected": -0.30082958936691284,
|
|
"loss": 0.7072250843048096,
|
|
"loss/core": 0.6854801177978516,
|
|
"loss/preference_sft": 0.33308959007263184,
|
|
"loss/reference_anchor": 0.4015893340110779,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.696979522705078,
|
|
"rewards/margins": 1.6540577411651611,
|
|
"rewards/rejected": 1.042922019958496,
|
|
"step": 840
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14128538966178894,
|
|
"drift/rejected_abs": 0.06555317342281342,
|
|
"epoch": 0.8073569807715275,
|
|
"grad_norm": 0.296875,
|
|
"learning_rate": 6.581850838338815e-09,
|
|
"logits/chosen": -2.1707913875579834,
|
|
"logits/rejected": -2.2462358474731445,
|
|
"logps/chosen": -0.29429852962493896,
|
|
"logps/rejected": -0.3137997090816498,
|
|
"loss": 0.6939007043838501,
|
|
"loss/core": 0.688424825668335,
|
|
"loss/preference_sft": 0.29429852962493896,
|
|
"loss/reference_anchor": 0.0800880640745163,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.4111610651016235,
|
|
"rewards/margins": 0.9817382097244263,
|
|
"rewards/rejected": 0.42942291498184204,
|
|
"step": 845
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1559392213821411,
|
|
"drift/rejected_abs": 0.09002901613712311,
|
|
"epoch": 0.8121342410127792,
|
|
"grad_norm": 2.640625,
|
|
"learning_rate": 5.463099816548577e-09,
|
|
"logits/chosen": -2.2900779247283936,
|
|
"logits/rejected": -2.3381993770599365,
|
|
"logps/chosen": -0.25206631422042847,
|
|
"logps/rejected": -0.24866075813770294,
|
|
"loss": 0.6935194730758667,
|
|
"loss/core": 0.6898444294929504,
|
|
"loss/preference_sft": 0.25206631422042847,
|
|
"loss/reference_anchor": 0.0482938326895237,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.557403564453125,
|
|
"rewards/margins": 0.6683042645454407,
|
|
"rewards/rejected": 0.8890994191169739,
|
|
"step": 850
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26853224635124207,
|
|
"drift/rejected_abs": 0.12536677718162537,
|
|
"epoch": 0.8169115012540308,
|
|
"grad_norm": 0.890625,
|
|
"learning_rate": 4.447445794656224e-09,
|
|
"logits/chosen": -2.002507209777832,
|
|
"logits/rejected": -2.0403895378112793,
|
|
"logps/chosen": -0.3012729287147522,
|
|
"logps/rejected": -0.2928231358528137,
|
|
"loss": 0.7090455293655396,
|
|
"loss/core": 0.6863538026809692,
|
|
"loss/preference_sft": 0.3012729287147522,
|
|
"loss/reference_anchor": 0.42370709776878357,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.6843512058258057,
|
|
"rewards/margins": 1.4311248064041138,
|
|
"rewards/rejected": 1.253226399421692,
|
|
"step": 855
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4081405997276306,
|
|
"drift/rejected_abs": 0.19295448064804077,
|
|
"epoch": 0.8216887614952825,
|
|
"grad_norm": 0.1943359375,
|
|
"learning_rate": 3.535316973405672e-09,
|
|
"logits/chosen": -1.9501336812973022,
|
|
"logits/rejected": -2.0376601219177246,
|
|
"logps/chosen": -0.3001163601875305,
|
|
"logps/rejected": -0.30008673667907715,
|
|
"loss": 0.7247797250747681,
|
|
"loss/core": 0.6830560564994812,
|
|
"loss/preference_sft": 0.3001163601875305,
|
|
"loss/reference_anchor": 0.8044621348381042,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.080620765686035,
|
|
"rewards/margins": 2.163020610809326,
|
|
"rewards/rejected": 1.9175999164581299,
|
|
"step": 860
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24484309554100037,
|
|
"drift/rejected_abs": 0.15850794315338135,
|
|
"epoch": 0.826466021736534,
|
|
"grad_norm": 0.8359375,
|
|
"learning_rate": 2.7270979072135104e-09,
|
|
"logits/chosen": -1.9584686756134033,
|
|
"logits/rejected": -2.0302765369415283,
|
|
"logps/chosen": -0.29134422540664673,
|
|
"logps/rejected": -0.2941442131996155,
|
|
"loss": 0.7054814696311951,
|
|
"loss/core": 0.6891493797302246,
|
|
"loss/preference_sft": 0.29134422540664673,
|
|
"loss/reference_anchor": 0.29750776290893555,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.447580099105835,
|
|
"rewards/margins": 0.864477813243866,
|
|
"rewards/rejected": 1.5831019878387451,
|
|
"step": 865
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3205713629722595,
|
|
"drift/rejected_abs": 0.09417042136192322,
|
|
"epoch": 0.8312432819777857,
|
|
"grad_norm": 0.59765625,
|
|
"learning_rate": 2.0231293420405195e-09,
|
|
"logits/chosen": -2.1127753257751465,
|
|
"logits/rejected": -2.1212573051452637,
|
|
"logps/chosen": -0.3091801106929779,
|
|
"logps/rejected": -0.2969990670681,
|
|
"loss": 0.7111729979515076,
|
|
"loss/core": 0.6829420924186707,
|
|
"loss/preference_sft": 0.3091801106929779,
|
|
"loss/reference_anchor": 0.533698320388794,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.204364776611328,
|
|
"rewards/margins": 2.2668018341064453,
|
|
"rewards/rejected": 0.9375633001327515,
|
|
"step": 870
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18526910245418549,
|
|
"drift/rejected_abs": 0.10249395668506622,
|
|
"epoch": 0.8360205422190374,
|
|
"grad_norm": 0.396484375,
|
|
"learning_rate": 1.423708071732671e-09,
|
|
"logits/chosen": -2.1826789379119873,
|
|
"logits/rejected": -2.230438232421875,
|
|
"logps/chosen": -0.30893266201019287,
|
|
"logps/rejected": -0.33167764544487,
|
|
"loss": 0.7087309956550598,
|
|
"loss/core": 0.6893488168716431,
|
|
"loss/preference_sft": 0.30893266201019287,
|
|
"loss/reference_anchor": 0.3567507863044739,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.8521311283111572,
|
|
"rewards/margins": 0.8370383381843567,
|
|
"rewards/rejected": 1.0150927305221558,
|
|
"step": 875
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24597208201885223,
|
|
"drift/rejected_abs": 0.18042945861816406,
|
|
"epoch": 0.840797802460289,
|
|
"grad_norm": 1.0625,
|
|
"learning_rate": 9.290868128926377e-10,
|
|
"logits/chosen": -2.1186165809631348,
|
|
"logits/rejected": -2.100987672805786,
|
|
"logps/chosen": -0.2905484139919281,
|
|
"logps/rejected": -0.3378651738166809,
|
|
"loss": 0.7146252393722534,
|
|
"loss/core": 0.6903001666069031,
|
|
"loss/preference_sft": 0.2905484139919281,
|
|
"loss/reference_anchor": 0.4574460983276367,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.4589881896972656,
|
|
"rewards/margins": 0.7572067379951477,
|
|
"rewards/rejected": 1.7017812728881836,
|
|
"step": 880
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2725023329257965,
|
|
"drift/rejected_abs": 0.11308582872152328,
|
|
"epoch": 0.8455750627015407,
|
|
"grad_norm": 0.373046875,
|
|
"learning_rate": 5.394740983341861e-10,
|
|
"logits/chosen": -2.043205976486206,
|
|
"logits/rejected": -2.1195883750915527,
|
|
"logps/chosen": -0.2826032042503357,
|
|
"logps/rejected": -0.27686744928359985,
|
|
"loss": 0.6973305940628052,
|
|
"loss/core": 0.6853419542312622,
|
|
"loss/preference_sft": 0.2826032042503357,
|
|
"loss/reference_anchor": 0.2115131914615631,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.724541187286377,
|
|
"rewards/margins": 1.5997540950775146,
|
|
"rewards/rejected": 1.1247870922088623,
|
|
"step": 885
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2453497350215912,
|
|
"drift/rejected_abs": 0.14477917551994324,
|
|
"epoch": 0.8503523229427923,
|
|
"grad_norm": 5.6875,
|
|
"learning_rate": 2.550341891646435e-10,
|
|
"logits/chosen": -1.9740431308746338,
|
|
"logits/rejected": -2.0952517986297607,
|
|
"logps/chosen": -0.30029481649398804,
|
|
"logps/rejected": -0.2738454043865204,
|
|
"loss": 0.7060577869415283,
|
|
"loss/core": 0.6884250640869141,
|
|
"loss/preference_sft": 0.30029481649398804,
|
|
"loss/reference_anchor": 0.32262665033340454,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.4530909061431885,
|
|
"rewards/margins": 1.009745717048645,
|
|
"rewards/rejected": 1.443345308303833,
|
|
"step": 890
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22996075451374054,
|
|
"drift/rejected_abs": 0.1483142226934433,
|
|
"epoch": 0.855129583184044,
|
|
"grad_norm": 2.015625,
|
|
"learning_rate": 7.588700553209926e-11,
|
|
"logits/chosen": -2.0447440147399902,
|
|
"logits/rejected": -2.0767829418182373,
|
|
"logps/chosen": -0.27262455224990845,
|
|
"logps/rejected": -0.2941727340221405,
|
|
"loss": 0.6983267068862915,
|
|
"loss/core": 0.6891483068466187,
|
|
"loss/preference_sft": 0.27262455224990845,
|
|
"loss/reference_anchor": 0.15630510449409485,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.2990550994873047,
|
|
"rewards/margins": 0.8220510482788086,
|
|
"rewards/rejected": 1.477004051208496,
|
|
"step": 895
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.30940914154052734,
|
|
"drift/rejected_abs": 0.08630160987377167,
|
|
"epoch": 0.8599068434252956,
|
|
"grad_norm": 0.65234375,
|
|
"learning_rate": 2.1080760670144636e-12,
|
|
"logits/chosen": -2.055746555328369,
|
|
"logits/rejected": -2.1760077476501465,
|
|
"logps/chosen": -0.2712702453136444,
|
|
"logps/rejected": -0.268751323223114,
|
|
"loss": 0.7091078758239746,
|
|
"loss/core": 0.6829676032066345,
|
|
"loss/preference_sft": 0.2712702453136444,
|
|
"loss/reference_anchor": 0.4956776201725006,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 3.0939459800720215,
|
|
"rewards/margins": 2.2492384910583496,
|
|
"rewards/rejected": 0.8447073101997375,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.8599068434252956,
|
|
"step": 900,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.7066660939322578,
|
|
"train_runtime": 7291.0823,
|
|
"train_samples_per_second": 1.975,
|
|
"train_steps_per_second": 0.123
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 900,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 900,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|