3644 lines
136 KiB
JSON
3644 lines
136 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 0.8599068434252956,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 900,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.241798996925354,
|
||
|
|
"drift/rejected_abs": 0.09728006273508072,
|
||
|
|
"epoch": 0.004777260241251642,
|
||
|
|
"grad_norm": 30.875,
|
||
|
|
"learning_rate": 1.111111111111111e-08,
|
||
|
|
"logits/chosen": -1.9988330602645874,
|
||
|
|
"logits/rejected": -2.015357494354248,
|
||
|
|
"logps/chosen": -0.279768705368042,
|
||
|
|
"logps/rejected": -0.28206831216812134,
|
||
|
|
"loss": 0.39691096544265747,
|
||
|
|
"loss/core": 0.3805164694786072,
|
||
|
|
"loss/preference_sft": 0.279768705368042,
|
||
|
|
"loss/reference_anchor": 0.19061610102653503,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.4163806438446045,
|
||
|
|
"rewards/margins": 1.4569675922393799,
|
||
|
|
"rewards/rejected": 0.9594132304191589,
|
||
|
|
"step": 5
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.16912893950939178,
|
||
|
|
"drift/rejected_abs": 0.16102473437786102,
|
||
|
|
"epoch": 0.009554520482503284,
|
||
|
|
"grad_norm": 716.0,
|
||
|
|
"learning_rate": 2.5e-08,
|
||
|
|
"logits/chosen": -2.344573497772217,
|
||
|
|
"logits/rejected": -2.3923327922821045,
|
||
|
|
"logps/chosen": -0.28925657272338867,
|
||
|
|
"logps/rejected": -0.28922492265701294,
|
||
|
|
"loss": 1.0646731853485107,
|
||
|
|
"loss/core": 1.0241014957427979,
|
||
|
|
"loss/preference_sft": 0.28925657272338867,
|
||
|
|
"loss/reference_anchor": 0.5120292901992798,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.6911067962646484,
|
||
|
|
"rewards/margins": 0.08549212664365768,
|
||
|
|
"rewards/rejected": 1.6056146621704102,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21334102749824524,
|
||
|
|
"drift/rejected_abs": 0.11430090665817261,
|
||
|
|
"epoch": 0.014331780723754926,
|
||
|
|
"grad_norm": 100.0,
|
||
|
|
"learning_rate": 3.888888888888889e-08,
|
||
|
|
"logits/chosen": -2.2332091331481934,
|
||
|
|
"logits/rejected": -2.1960010528564453,
|
||
|
|
"logps/chosen": -0.27760133147239685,
|
||
|
|
"logps/rejected": -0.28394412994384766,
|
||
|
|
"loss": 0.37293222546577454,
|
||
|
|
"loss/core": 0.35742494463920593,
|
||
|
|
"loss/preference_sft": 0.27760133147239685,
|
||
|
|
"loss/reference_anchor": 0.17900365591049194,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.132291316986084,
|
||
|
|
"rewards/margins": 0.991050124168396,
|
||
|
|
"rewards/rejected": 1.1412410736083984,
|
||
|
|
"step": 15
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.19138672947883606,
|
||
|
|
"drift/rejected_abs": 0.08749718964099884,
|
||
|
|
"epoch": 0.01910904096500657,
|
||
|
|
"grad_norm": 22.5,
|
||
|
|
"learning_rate": 5.2777777777777776e-08,
|
||
|
|
"logits/chosen": -2.1781129837036133,
|
||
|
|
"logits/rejected": -2.219083309173584,
|
||
|
|
"logps/chosen": -0.30154842138290405,
|
||
|
|
"logps/rejected": -0.2870495915412903,
|
||
|
|
"loss": 0.45054665207862854,
|
||
|
|
"loss/core": 0.4320584833621979,
|
||
|
|
"loss/preference_sft": 0.30154842138290405,
|
||
|
|
"loss/reference_anchor": 0.216354101896286,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.9138669967651367,
|
||
|
|
"rewards/margins": 1.0391050577163696,
|
||
|
|
"rewards/rejected": 0.8747620582580566,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.38581329584121704,
|
||
|
|
"drift/rejected_abs": 0.1625959426164627,
|
||
|
|
"epoch": 0.02388630120625821,
|
||
|
|
"grad_norm": 26.5,
|
||
|
|
"learning_rate": 6.666666666666667e-08,
|
||
|
|
"logits/chosen": -2.0747623443603516,
|
||
|
|
"logits/rejected": -2.1344943046569824,
|
||
|
|
"logps/chosen": -0.26991984248161316,
|
||
|
|
"logps/rejected": -0.2599589228630066,
|
||
|
|
"loss": 1.819545030593872,
|
||
|
|
"loss/core": 1.7517907619476318,
|
||
|
|
"loss/preference_sft": 0.26991984248161316,
|
||
|
|
"loss/reference_anchor": 0.8763998746871948,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 3.8559670448303223,
|
||
|
|
"rewards/margins": 2.2310361862182617,
|
||
|
|
"rewards/rejected": 1.6249310970306396,
|
||
|
|
"step": 25
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.22763633728027344,
|
||
|
|
"drift/rejected_abs": 0.11158541589975357,
|
||
|
|
"epoch": 0.028663561447509853,
|
||
|
|
"grad_norm": 252.0,
|
||
|
|
"learning_rate": 8.055555555555555e-08,
|
||
|
|
"logits/chosen": -2.349606513977051,
|
||
|
|
"logits/rejected": -2.331936836242676,
|
||
|
|
"logps/chosen": -0.3003299832344055,
|
||
|
|
"logps/rejected": -0.32463088631629944,
|
||
|
|
"loss": 0.6905651092529297,
|
||
|
|
"loss/core": 0.6633950471878052,
|
||
|
|
"loss/preference_sft": 0.3003299832344055,
|
||
|
|
"loss/reference_anchor": 0.3322353661060333,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.275994300842285,
|
||
|
|
"rewards/margins": 1.2529335021972656,
|
||
|
|
"rewards/rejected": 1.0230610370635986,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.26552093029022217,
|
||
|
|
"drift/rejected_abs": 0.11600001156330109,
|
||
|
|
"epoch": 0.033440821688761495,
|
||
|
|
"grad_norm": 2.28125,
|
||
|
|
"learning_rate": 9.444444444444444e-08,
|
||
|
|
"logits/chosen": -2.203125476837158,
|
||
|
|
"logits/rejected": -2.184936046600342,
|
||
|
|
"logps/chosen": -0.26805487275123596,
|
||
|
|
"logps/rejected": -0.2653547525405884,
|
||
|
|
"loss": 0.8213445544242859,
|
||
|
|
"loss/core": 0.7896862626075745,
|
||
|
|
"loss/preference_sft": 0.26805487275123596,
|
||
|
|
"loss/reference_anchor": 0.3953050374984741,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.6552093029022217,
|
||
|
|
"rewards/margins": 1.496461272239685,
|
||
|
|
"rewards/rejected": 1.1587477922439575,
|
||
|
|
"step": 35
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.44771280884742737,
|
||
|
|
"drift/rejected_abs": 0.16425152122974396,
|
||
|
|
"epoch": 0.03821808193001314,
|
||
|
|
"grad_norm": 81.0,
|
||
|
|
"learning_rate": 1.0833333333333334e-07,
|
||
|
|
"logits/chosen": -2.0018858909606934,
|
||
|
|
"logits/rejected": -2.061411142349243,
|
||
|
|
"logps/chosen": -0.2927972674369812,
|
||
|
|
"logps/rejected": -0.2809644341468811,
|
||
|
|
"loss": 2.044156789779663,
|
||
|
|
"loss/core": 1.9680871963500977,
|
||
|
|
"loss/preference_sft": 0.2927972674369812,
|
||
|
|
"loss/reference_anchor": 0.9849796295166016,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 4.476373672485352,
|
||
|
|
"rewards/margins": 2.8645739555358887,
|
||
|
|
"rewards/rejected": 1.6117998361587524,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21002939343452454,
|
||
|
|
"drift/rejected_abs": 0.08615503460168839,
|
||
|
|
"epoch": 0.04299534217126478,
|
||
|
|
"grad_norm": 8.3125,
|
||
|
|
"learning_rate": 1.2222222222222222e-07,
|
||
|
|
"logits/chosen": -2.2009475231170654,
|
||
|
|
"logits/rejected": -2.3619346618652344,
|
||
|
|
"logps/chosen": -0.2883842885494232,
|
||
|
|
"logps/rejected": -0.2879543900489807,
|
||
|
|
"loss": 0.2927166819572449,
|
||
|
|
"loss/core": 0.28003185987472534,
|
||
|
|
"loss/preference_sft": 0.2883842885494232,
|
||
|
|
"loss/reference_anchor": 0.14029237627983093,
|
||
|
|
"rewards/accuracies": 0.9624999761581421,
|
||
|
|
"rewards/chosen": 2.099714756011963,
|
||
|
|
"rewards/margins": 1.253118634223938,
|
||
|
|
"rewards/rejected": 0.8465961217880249,
|
||
|
|
"step": 45
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.28804001212120056,
|
||
|
|
"drift/rejected_abs": 0.14722736179828644,
|
||
|
|
"epoch": 0.04777260241251642,
|
||
|
|
"grad_norm": 75.5,
|
||
|
|
"learning_rate": 1.3611111111111108e-07,
|
||
|
|
"logits/chosen": -2.1644270420074463,
|
||
|
|
"logits/rejected": -2.1614725589752197,
|
||
|
|
"logps/chosen": -0.2816537022590637,
|
||
|
|
"logps/rejected": -0.2829938530921936,
|
||
|
|
"loss": 0.7515085339546204,
|
||
|
|
"loss/core": 0.7222825884819031,
|
||
|
|
"loss/preference_sft": 0.2816537022590637,
|
||
|
|
"loss/reference_anchor": 0.36151474714279175,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.8799638748168945,
|
||
|
|
"rewards/margins": 1.4102028608322144,
|
||
|
|
"rewards/rejected": 1.4697611331939697,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2258037030696869,
|
||
|
|
"drift/rejected_abs": 0.20605143904685974,
|
||
|
|
"epoch": 0.05254986265376806,
|
||
|
|
"grad_norm": 0.8984375,
|
||
|
|
"learning_rate": 1.5e-07,
|
||
|
|
"logits/chosen": -2.1963486671447754,
|
||
|
|
"logits/rejected": -2.184718608856201,
|
||
|
|
"logps/chosen": -0.28904852271080017,
|
||
|
|
"logps/rejected": -0.28756090998649597,
|
||
|
|
"loss": 1.5850727558135986,
|
||
|
|
"loss/core": 1.5256582498550415,
|
||
|
|
"loss/preference_sft": 0.28904852271080017,
|
||
|
|
"loss/reference_anchor": 0.7632866501808167,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.2579128742218018,
|
||
|
|
"rewards/margins": 0.2070862054824829,
|
||
|
|
"rewards/rejected": 2.0508265495300293,
|
||
|
|
"step": 55
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21603329479694366,
|
||
|
|
"drift/rejected_abs": 0.1505412757396698,
|
||
|
|
"epoch": 0.057327122895019705,
|
||
|
|
"grad_norm": 512.0,
|
||
|
|
"learning_rate": 1.6388888888888888e-07,
|
||
|
|
"logits/chosen": -2.091665744781494,
|
||
|
|
"logits/rejected": -2.120565176010132,
|
||
|
|
"logps/chosen": -0.2742917537689209,
|
||
|
|
"logps/rejected": -0.2904832065105438,
|
||
|
|
"loss": 0.3291396200656891,
|
||
|
|
"loss/core": 0.3152475953102112,
|
||
|
|
"loss/preference_sft": 0.2742917537689209,
|
||
|
|
"loss/reference_anchor": 0.1577974259853363,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.157923698425293,
|
||
|
|
"rewards/margins": 0.668967068195343,
|
||
|
|
"rewards/rejected": 1.4889566898345947,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.29638388752937317,
|
||
|
|
"drift/rejected_abs": 0.11892390251159668,
|
||
|
|
"epoch": 0.06210438313627135,
|
||
|
|
"grad_norm": 7.875,
|
||
|
|
"learning_rate": 1.7777777777777776e-07,
|
||
|
|
"logits/chosen": -1.9620473384857178,
|
||
|
|
"logits/rejected": -2.0643868446350098,
|
||
|
|
"logps/chosen": -0.3099573254585266,
|
||
|
|
"logps/rejected": -0.3055954873561859,
|
||
|
|
"loss": 1.0155407190322876,
|
||
|
|
"loss/core": 0.9765542149543762,
|
||
|
|
"loss/preference_sft": 0.3099573254585266,
|
||
|
|
"loss/reference_anchor": 0.4888220429420471,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.962282419204712,
|
||
|
|
"rewards/margins": 1.783328652381897,
|
||
|
|
"rewards/rejected": 1.178953766822815,
|
||
|
|
"step": 65
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.358979195356369,
|
||
|
|
"drift/rejected_abs": 0.22608236968517303,
|
||
|
|
"epoch": 0.06688164337752299,
|
||
|
|
"grad_norm": 56.25,
|
||
|
|
"learning_rate": 1.9166666666666668e-07,
|
||
|
|
"logits/chosen": -1.9711816310882568,
|
||
|
|
"logits/rejected": -1.9564964771270752,
|
||
|
|
"logps/chosen": -0.38483089208602905,
|
||
|
|
"logps/rejected": -0.2978837192058563,
|
||
|
|
"loss": 1.9197027683258057,
|
||
|
|
"loss/core": 1.84750235080719,
|
||
|
|
"loss/preference_sft": 0.38483089208602905,
|
||
|
|
"loss/reference_anchor": 0.9241858720779419,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 3.5887832641601562,
|
||
|
|
"rewards/margins": 1.4143999814987183,
|
||
|
|
"rewards/rejected": 2.1743829250335693,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2635686993598938,
|
||
|
|
"drift/rejected_abs": 0.15887705981731415,
|
||
|
|
"epoch": 0.07165890361877464,
|
||
|
|
"grad_norm": 8.25,
|
||
|
|
"learning_rate": 2.0555555555555553e-07,
|
||
|
|
"logits/chosen": -2.086611747741699,
|
||
|
|
"logits/rejected": -2.0510897636413574,
|
||
|
|
"logps/chosen": -0.3122991919517517,
|
||
|
|
"logps/rejected": -0.28983455896377563,
|
||
|
|
"loss": 0.7980114817619324,
|
||
|
|
"loss/core": 0.766870379447937,
|
||
|
|
"loss/preference_sft": 0.3122991919517517,
|
||
|
|
"loss/reference_anchor": 0.3839830458164215,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 2.6338839530944824,
|
||
|
|
"rewards/margins": 1.0476503372192383,
|
||
|
|
"rewards/rejected": 1.586233377456665,
|
||
|
|
"step": 75
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.32444751262664795,
|
||
|
|
"drift/rejected_abs": 0.12715654075145721,
|
||
|
|
"epoch": 0.07643616386002627,
|
||
|
|
"grad_norm": 4.15625,
|
||
|
|
"learning_rate": 2.1944444444444442e-07,
|
||
|
|
"logits/chosen": -2.069356918334961,
|
||
|
|
"logits/rejected": -2.190796375274658,
|
||
|
|
"logps/chosen": -0.26029911637306213,
|
||
|
|
"logps/rejected": -0.27222079038619995,
|
||
|
|
"loss": 0.951573371887207,
|
||
|
|
"loss/core": 0.9152650833129883,
|
||
|
|
"loss/preference_sft": 0.26029911637306213,
|
||
|
|
"loss/reference_anchor": 0.4580825865268707,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 3.244004726409912,
|
||
|
|
"rewards/margins": 1.9740562438964844,
|
||
|
|
"rewards/rejected": 1.2699484825134277,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.35097089409828186,
|
||
|
|
"drift/rejected_abs": 0.13202176988124847,
|
||
|
|
"epoch": 0.08121342410127792,
|
||
|
|
"grad_norm": 6.0,
|
||
|
|
"learning_rate": 2.3333333333333333e-07,
|
||
|
|
"logits/chosen": -2.186922073364258,
|
||
|
|
"logits/rejected": -2.2361488342285156,
|
||
|
|
"logps/chosen": -0.30639296770095825,
|
||
|
|
"logps/rejected": -0.2788568139076233,
|
||
|
|
"loss": 1.790135383605957,
|
||
|
|
"loss/core": 1.7231495380401611,
|
||
|
|
"loss/preference_sft": 0.30639296770095825,
|
||
|
|
"loss/reference_anchor": 0.8625059127807617,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 3.509377956390381,
|
||
|
|
"rewards/margins": 2.1900832653045654,
|
||
|
|
"rewards/rejected": 1.3192951679229736,
|
||
|
|
"step": 85
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.17585346102714539,
|
||
|
|
"drift/rejected_abs": 0.08460784703493118,
|
||
|
|
"epoch": 0.08599068434252956,
|
||
|
|
"grad_norm": 1.0,
|
||
|
|
"learning_rate": 2.4722222222222224e-07,
|
||
|
|
"logits/chosen": -2.42102313041687,
|
||
|
|
"logits/rejected": -2.4086685180664062,
|
||
|
|
"logps/chosen": -0.2809022068977356,
|
||
|
|
"logps/rejected": -0.29607102274894714,
|
||
|
|
"loss": 0.32354456186294556,
|
||
|
|
"loss/core": 0.30980855226516724,
|
||
|
|
"loss/preference_sft": 0.2809022068977356,
|
||
|
|
"loss/reference_anchor": 0.1550559550523758,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.7583656311035156,
|
||
|
|
"rewards/margins": 0.9138404726982117,
|
||
|
|
"rewards/rejected": 0.844525158405304,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.320295125246048,
|
||
|
|
"drift/rejected_abs": 0.21105077862739563,
|
||
|
|
"epoch": 0.09076794458378121,
|
||
|
|
"grad_norm": 67.0,
|
||
|
|
"learning_rate": 2.6111111111111113e-07,
|
||
|
|
"logits/chosen": -2.0537590980529785,
|
||
|
|
"logits/rejected": -2.1096103191375732,
|
||
|
|
"logps/chosen": -0.26810017228126526,
|
||
|
|
"logps/rejected": -0.2783377170562744,
|
||
|
|
"loss": 0.9392117261886597,
|
||
|
|
"loss/core": 0.903303325176239,
|
||
|
|
"loss/preference_sft": 0.26810017228126526,
|
||
|
|
"loss/reference_anchor": 0.45196953415870667,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 3.2027251720428467,
|
||
|
|
"rewards/margins": 1.0923994779586792,
|
||
|
|
"rewards/rejected": 2.110325574874878,
|
||
|
|
"step": 95
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.24583487212657928,
|
||
|
|
"drift/rejected_abs": 0.12835685908794403,
|
||
|
|
"epoch": 0.09554520482503284,
|
||
|
|
"grad_norm": 10.8125,
|
||
|
|
"learning_rate": 2.75e-07,
|
||
|
|
"logits/chosen": -2.234044075012207,
|
||
|
|
"logits/rejected": -2.2795557975769043,
|
||
|
|
"logps/chosen": -0.2577240467071533,
|
||
|
|
"logps/rejected": -0.24702732264995575,
|
||
|
|
"loss": 0.37607091665267944,
|
||
|
|
"loss/core": 0.3605777323246002,
|
||
|
|
"loss/preference_sft": 0.2577240467071533,
|
||
|
|
"loss/reference_anchor": 0.18080314993858337,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.4573445320129395,
|
||
|
|
"rewards/margins": 1.1746383905410767,
|
||
|
|
"rewards/rejected": 1.282706379890442,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.19956639409065247,
|
||
|
|
"drift/rejected_abs": 0.08023403584957123,
|
||
|
|
"epoch": 0.10032246506628449,
|
||
|
|
"grad_norm": 302.0,
|
||
|
|
"learning_rate": 2.8888888888888885e-07,
|
||
|
|
"logits/chosen": -2.3368284702301025,
|
||
|
|
"logits/rejected": -2.421706438064575,
|
||
|
|
"logps/chosen": -0.2843855023384094,
|
||
|
|
"logps/rejected": -0.27989473938941956,
|
||
|
|
"loss": 0.41286808252334595,
|
||
|
|
"loss/core": 0.39585790038108826,
|
||
|
|
"loss/preference_sft": 0.2843855023384094,
|
||
|
|
"loss/reference_anchor": 0.19836413860321045,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.9953877925872803,
|
||
|
|
"rewards/margins": 1.1983610391616821,
|
||
|
|
"rewards/rejected": 0.7970272302627563,
|
||
|
|
"step": 105
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.44724979996681213,
|
||
|
|
"drift/rejected_abs": 0.18898524343967438,
|
||
|
|
"epoch": 0.10509972530753613,
|
||
|
|
"grad_norm": 14.5625,
|
||
|
|
"learning_rate": 3.0277777777777773e-07,
|
||
|
|
"logits/chosen": -1.9066753387451172,
|
||
|
|
"logits/rejected": -1.8599035739898682,
|
||
|
|
"logps/chosen": -0.33843153715133667,
|
||
|
|
"logps/rejected": -0.2979224920272827,
|
||
|
|
"loss": 1.8243690729141235,
|
||
|
|
"loss/core": 1.7559093236923218,
|
||
|
|
"loss/preference_sft": 0.33843153715133667,
|
||
|
|
"loss/reference_anchor": 0.8789511919021606,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 4.470961570739746,
|
||
|
|
"rewards/margins": 2.5842795372009277,
|
||
|
|
"rewards/rejected": 1.8866825103759766,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.17774026095867157,
|
||
|
|
"drift/rejected_abs": 0.14382174611091614,
|
||
|
|
"epoch": 0.10987698554878778,
|
||
|
|
"grad_norm": 11.9375,
|
||
|
|
"learning_rate": 3.166666666666666e-07,
|
||
|
|
"logits/chosen": -2.086132764816284,
|
||
|
|
"logits/rejected": -2.030756711959839,
|
||
|
|
"logps/chosen": -0.3001967668533325,
|
||
|
|
"logps/rejected": -0.2998013198375702,
|
||
|
|
"loss": 0.43933549523353577,
|
||
|
|
"loss/core": 0.4212765693664551,
|
||
|
|
"loss/preference_sft": 0.3001967668533325,
|
||
|
|
"loss/reference_anchor": 0.21076564490795135,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.777402639389038,
|
||
|
|
"rewards/margins": 0.34036320447921753,
|
||
|
|
"rewards/rejected": 1.4370393753051758,
|
||
|
|
"step": 115
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.18121027946472168,
|
||
|
|
"drift/rejected_abs": 0.11058615148067474,
|
||
|
|
"epoch": 0.11465424579003941,
|
||
|
|
"grad_norm": 180.0,
|
||
|
|
"learning_rate": 3.3055555555555556e-07,
|
||
|
|
"logits/chosen": -2.0374550819396973,
|
||
|
|
"logits/rejected": -2.0756847858428955,
|
||
|
|
"logps/chosen": -0.301984578371048,
|
||
|
|
"logps/rejected": -0.29793334007263184,
|
||
|
|
"loss": 0.1780211180448532,
|
||
|
|
"loss/core": 0.16938777267932892,
|
||
|
|
"loss/preference_sft": 0.301984578371048,
|
||
|
|
"loss/reference_anchor": 0.08491319417953491,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.8116956949234009,
|
||
|
|
"rewards/margins": 0.7075945734977722,
|
||
|
|
"rewards/rejected": 1.1041011810302734,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2069893628358841,
|
||
|
|
"drift/rejected_abs": 0.10128186643123627,
|
||
|
|
"epoch": 0.11943150603129106,
|
||
|
|
"grad_norm": 6.09375,
|
||
|
|
"learning_rate": 3.4444444444444444e-07,
|
||
|
|
"logits/chosen": -2.108165979385376,
|
||
|
|
"logits/rejected": -2.139435291290283,
|
||
|
|
"logps/chosen": -0.29883089661598206,
|
||
|
|
"logps/rejected": -0.3071492612361908,
|
||
|
|
"loss": 0.2796749472618103,
|
||
|
|
"loss/core": 0.2673766613006592,
|
||
|
|
"loss/preference_sft": 0.29883089661598206,
|
||
|
|
"loss/reference_anchor": 0.1340940147638321,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.067173480987549,
|
||
|
|
"rewards/margins": 1.0553032159805298,
|
||
|
|
"rewards/rejected": 1.0118701457977295,
|
||
|
|
"step": 125
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.18067234754562378,
|
||
|
|
"drift/rejected_abs": 0.08859051764011383,
|
||
|
|
"epoch": 0.1242087662725427,
|
||
|
|
"grad_norm": 11.25,
|
||
|
|
"learning_rate": 3.583333333333333e-07,
|
||
|
|
"logits/chosen": -1.9984080791473389,
|
||
|
|
"logits/rejected": -2.0914056301116943,
|
||
|
|
"logps/chosen": -0.28005415201187134,
|
||
|
|
"logps/rejected": -0.27907031774520874,
|
||
|
|
"loss": 0.15530189871788025,
|
||
|
|
"loss/core": 0.14763937890529633,
|
||
|
|
"loss/preference_sft": 0.28005415201187134,
|
||
|
|
"loss/reference_anchor": 0.0741613507270813,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 1.806658387184143,
|
||
|
|
"rewards/margins": 0.9225584268569946,
|
||
|
|
"rewards/rejected": 0.8841001391410828,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.33673185110092163,
|
||
|
|
"drift/rejected_abs": 0.1371738612651825,
|
||
|
|
"epoch": 0.12898602651379434,
|
||
|
|
"grad_norm": 20.5,
|
||
|
|
"learning_rate": 3.722222222222222e-07,
|
||
|
|
"logits/chosen": -2.0906131267547607,
|
||
|
|
"logits/rejected": -2.16442608833313,
|
||
|
|
"logps/chosen": -0.3028051257133484,
|
||
|
|
"logps/rejected": -0.29675745964050293,
|
||
|
|
"loss": 0.9384252429008484,
|
||
|
|
"loss/core": 0.9022802114486694,
|
||
|
|
"loss/preference_sft": 0.3028051257133484,
|
||
|
|
"loss/reference_anchor": 0.4516521990299225,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.367079973220825,
|
||
|
|
"rewards/margins": 2.038417339324951,
|
||
|
|
"rewards/rejected": 1.3286627531051636,
|
||
|
|
"step": 135
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1627669632434845,
|
||
|
|
"drift/rejected_abs": 0.0710747018456459,
|
||
|
|
"epoch": 0.13376328675504598,
|
||
|
|
"grad_norm": 12.1875,
|
||
|
|
"learning_rate": 3.861111111111111e-07,
|
||
|
|
"logits/chosen": -2.4494571685791016,
|
||
|
|
"logits/rejected": -2.5083508491516113,
|
||
|
|
"logps/chosen": -0.2825891971588135,
|
||
|
|
"logps/rejected": -0.2898746430873871,
|
||
|
|
"loss": 0.12142524868249893,
|
||
|
|
"loss/core": 0.11497412621974945,
|
||
|
|
"loss/preference_sft": 0.2825891971588135,
|
||
|
|
"loss/reference_anchor": 0.057755958288908005,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.6273698806762695,
|
||
|
|
"rewards/margins": 0.9166396856307983,
|
||
|
|
"rewards/rejected": 0.7107301950454712,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.36183470487594604,
|
||
|
|
"drift/rejected_abs": 0.17905983328819275,
|
||
|
|
"epoch": 0.13854054699629761,
|
||
|
|
"grad_norm": 8.125,
|
||
|
|
"learning_rate": 4e-07,
|
||
|
|
"logits/chosen": -1.7783467769622803,
|
||
|
|
"logits/rejected": -1.8096199035644531,
|
||
|
|
"logps/chosen": -0.3000045418739319,
|
||
|
|
"logps/rejected": -0.30179673433303833,
|
||
|
|
"loss": 2.165755271911621,
|
||
|
|
"loss/core": 2.0852818489074707,
|
||
|
|
"loss/preference_sft": 0.3000045418739319,
|
||
|
|
"loss/reference_anchor": 1.0429770946502686,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.617523193359375,
|
||
|
|
"rewards/margins": 1.832786202430725,
|
||
|
|
"rewards/rejected": 1.78473699092865,
|
||
|
|
"step": 145
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.36336830258369446,
|
||
|
|
"drift/rejected_abs": 0.14750418066978455,
|
||
|
|
"epoch": 0.14331780723754928,
|
||
|
|
"grad_norm": 121.0,
|
||
|
|
"learning_rate": 4.1388888888888887e-07,
|
||
|
|
"logits/chosen": -2.2309694290161133,
|
||
|
|
"logits/rejected": -2.280487298965454,
|
||
|
|
"logps/chosen": -0.3181116282939911,
|
||
|
|
"logps/rejected": -0.3068601191043854,
|
||
|
|
"loss": 1.7123243808746338,
|
||
|
|
"loss/core": 1.6480745077133179,
|
||
|
|
"loss/preference_sft": 0.3181116282939911,
|
||
|
|
"loss/reference_anchor": 0.8248558044433594,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.6315560340881348,
|
||
|
|
"rewards/margins": 2.1605687141418457,
|
||
|
|
"rewards/rejected": 1.4709874391555786,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2860521078109741,
|
||
|
|
"drift/rejected_abs": 0.08609304577112198,
|
||
|
|
"epoch": 0.1480950674788009,
|
||
|
|
"grad_norm": 5.125,
|
||
|
|
"learning_rate": 4.2777777777777775e-07,
|
||
|
|
"logits/chosen": -2.0970890522003174,
|
||
|
|
"logits/rejected": -2.1750330924987793,
|
||
|
|
"logps/chosen": -0.28367698192596436,
|
||
|
|
"logps/rejected": -0.29151204228401184,
|
||
|
|
"loss": 0.7529366612434387,
|
||
|
|
"loss/core": 0.7236311435699463,
|
||
|
|
"loss/preference_sft": 0.28367698192596436,
|
||
|
|
"loss/reference_anchor": 0.36237236857414246,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.858731746673584,
|
||
|
|
"rewards/margins": 2.024120330810547,
|
||
|
|
"rewards/rejected": 0.8346115946769714,
|
||
|
|
"step": 155
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.31623023748397827,
|
||
|
|
"drift/rejected_abs": 0.13978847861289978,
|
||
|
|
"epoch": 0.15287232772005255,
|
||
|
|
"grad_norm": 282.0,
|
||
|
|
"learning_rate": 4.4166666666666664e-07,
|
||
|
|
"logits/chosen": -2.0535616874694824,
|
||
|
|
"logits/rejected": -2.027621030807495,
|
||
|
|
"logps/chosen": -0.30931371450424194,
|
||
|
|
"logps/rejected": -0.3156592547893524,
|
||
|
|
"loss": 1.4550539255142212,
|
||
|
|
"loss/core": 1.4001611471176147,
|
||
|
|
"loss/preference_sft": 0.30931371450424194,
|
||
|
|
"loss/reference_anchor": 0.7009710073471069,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 3.1623027324676514,
|
||
|
|
"rewards/margins": 1.764468789100647,
|
||
|
|
"rewards/rejected": 1.397834062576294,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.16005036234855652,
|
||
|
|
"drift/rejected_abs": 0.07912009209394455,
|
||
|
|
"epoch": 0.15764958796130418,
|
||
|
|
"grad_norm": 5.1875,
|
||
|
|
"learning_rate": 4.555555555555555e-07,
|
||
|
|
"logits/chosen": -2.2203352451324463,
|
||
|
|
"logits/rejected": -2.1501989364624023,
|
||
|
|
"logps/chosen": -0.31128016114234924,
|
||
|
|
"logps/rejected": -0.3056836724281311,
|
||
|
|
"loss": 0.18936026096343994,
|
||
|
|
"loss/core": 0.18025028705596924,
|
||
|
|
"loss/preference_sft": 0.31128016114234924,
|
||
|
|
"loss/reference_anchor": 0.09033866971731186,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.60050368309021,
|
||
|
|
"rewards/margins": 0.8113292455673218,
|
||
|
|
"rewards/rejected": 0.7891744375228882,
|
||
|
|
"step": 165
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.26069825887680054,
|
||
|
|
"drift/rejected_abs": 0.2761599123477936,
|
||
|
|
"epoch": 0.16242684820255585,
|
||
|
|
"grad_norm": 378.0,
|
||
|
|
"learning_rate": 4.694444444444444e-07,
|
||
|
|
"logits/chosen": -2.186049461364746,
|
||
|
|
"logits/rejected": -2.1308929920196533,
|
||
|
|
"logps/chosen": -0.273814857006073,
|
||
|
|
"logps/rejected": -0.2607230842113495,
|
||
|
|
"loss": 1.9536501169204712,
|
||
|
|
"loss/core": 1.8810440301895142,
|
||
|
|
"loss/preference_sft": 0.273814857006073,
|
||
|
|
"loss/reference_anchor": 0.9407007098197937,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.6061038970947266,
|
||
|
|
"rewards/margins": -0.15422509610652924,
|
||
|
|
"rewards/rejected": 2.760329008102417,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.25758832693099976,
|
||
|
|
"drift/rejected_abs": 0.12776905298233032,
|
||
|
|
"epoch": 0.16720410844380748,
|
||
|
|
"grad_norm": 406.0,
|
||
|
|
"learning_rate": 4.833333333333333e-07,
|
||
|
|
"logits/chosen": -2.1730058193206787,
|
||
|
|
"logits/rejected": -2.212725877761841,
|
||
|
|
"logps/chosen": -0.2877016067504883,
|
||
|
|
"logps/rejected": -0.30032774806022644,
|
||
|
|
"loss": 1.1656818389892578,
|
||
|
|
"loss/core": 1.1214388608932495,
|
||
|
|
"loss/preference_sft": 0.2877016067504883,
|
||
|
|
"loss/reference_anchor": 0.5611380338668823,
|
||
|
|
"rewards/accuracies": 0.9624999761581421,
|
||
|
|
"rewards/chosen": 2.5753438472747803,
|
||
|
|
"rewards/margins": 1.3018853664398193,
|
||
|
|
"rewards/rejected": 1.273458480834961,
|
||
|
|
"step": 175
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3230380117893219,
|
||
|
|
"drift/rejected_abs": 0.150053471326828,
|
||
|
|
"epoch": 0.17198136868505912,
|
||
|
|
"grad_norm": 668.0,
|
||
|
|
"learning_rate": 4.972222222222222e-07,
|
||
|
|
"logits/chosen": -2.197150945663452,
|
||
|
|
"logits/rejected": -2.108093023300171,
|
||
|
|
"logps/chosen": -0.27259406447410583,
|
||
|
|
"logps/rejected": -0.2611132264137268,
|
||
|
|
"loss": 1.2022207975387573,
|
||
|
|
"loss/core": 1.1567691564559937,
|
||
|
|
"loss/preference_sft": 0.27259406447410583,
|
||
|
|
"loss/reference_anchor": 0.5787622332572937,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 3.2283949851989746,
|
||
|
|
"rewards/margins": 1.730065941810608,
|
||
|
|
"rewards/rejected": 1.4983290433883667,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.10474558919668198,
|
||
|
|
"drift/rejected_abs": 0.06288503110408783,
|
||
|
|
"epoch": 0.17675862892631075,
|
||
|
|
"grad_norm": 3.453125,
|
||
|
|
"learning_rate": 4.999619237890978e-07,
|
||
|
|
"logits/chosen": -2.201859474182129,
|
||
|
|
"logits/rejected": -2.158825397491455,
|
||
|
|
"logps/chosen": -0.28844937682151794,
|
||
|
|
"logps/rejected": -0.2922508418560028,
|
||
|
|
"loss": 0.06655260920524597,
|
||
|
|
"loss/core": 0.06205412745475769,
|
||
|
|
"loss/preference_sft": 0.28844937682151794,
|
||
|
|
"loss/reference_anchor": 0.03113471530377865,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 1.0451735258102417,
|
||
|
|
"rewards/margins": 0.41988277435302734,
|
||
|
|
"rewards/rejected": 0.6252908706665039,
|
||
|
|
"step": 185
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2833901643753052,
|
||
|
|
"drift/rejected_abs": 0.12482712417840958,
|
||
|
|
"epoch": 0.18153588916756241,
|
||
|
|
"grad_norm": 30.5,
|
||
|
|
"learning_rate": 4.998072590601808e-07,
|
||
|
|
"logits/chosen": -2.2913918495178223,
|
||
|
|
"logits/rejected": -2.2682085037231445,
|
||
|
|
"logps/chosen": -0.284027636051178,
|
||
|
|
"logps/rejected": -0.2837992310523987,
|
||
|
|
"loss": 1.1685082912445068,
|
||
|
|
"loss/core": 1.1242001056671143,
|
||
|
|
"loss/preference_sft": 0.284027636051178,
|
||
|
|
"loss/reference_anchor": 0.5623761415481567,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.8339016437530518,
|
||
|
|
"rewards/margins": 1.590225100517273,
|
||
|
|
"rewards/rejected": 1.2436766624450684,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.26705488562583923,
|
||
|
|
"drift/rejected_abs": 0.0977572649717331,
|
||
|
|
"epoch": 0.18631314940881405,
|
||
|
|
"grad_norm": 23.25,
|
||
|
|
"learning_rate": 4.995336996054667e-07,
|
||
|
|
"logits/chosen": -1.982912302017212,
|
||
|
|
"logits/rejected": -2.0029821395874023,
|
||
|
|
"logps/chosen": -0.2908513844013214,
|
||
|
|
"logps/rejected": -0.2896444797515869,
|
||
|
|
"loss": 1.151555061340332,
|
||
|
|
"loss/core": 1.1077957153320312,
|
||
|
|
"loss/preference_sft": 0.2908513844013214,
|
||
|
|
"loss/reference_anchor": 0.5543731451034546,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.66994571685791,
|
||
|
|
"rewards/margins": 1.7028605937957764,
|
||
|
|
"rewards/rejected": 0.9670848846435547,
|
||
|
|
"step": 195
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2615807056427002,
|
||
|
|
"drift/rejected_abs": 0.12196584045886993,
|
||
|
|
"epoch": 0.19109040965006568,
|
||
|
|
"grad_norm": 21.375,
|
||
|
|
"learning_rate": 4.991413756244404e-07,
|
||
|
|
"logits/chosen": -1.8908970355987549,
|
||
|
|
"logits/rejected": -1.9317600727081299,
|
||
|
|
"logps/chosen": -0.3235652446746826,
|
||
|
|
"logps/rejected": -0.316772997379303,
|
||
|
|
"loss": 0.5153834819793701,
|
||
|
|
"loss/core": 0.4943995475769043,
|
||
|
|
"loss/preference_sft": 0.3235652446746826,
|
||
|
|
"loss/reference_anchor": 0.2474292516708374,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.6145291328430176,
|
||
|
|
"rewards/margins": 1.3955014944076538,
|
||
|
|
"rewards/rejected": 1.2190272808074951,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3288207948207855,
|
||
|
|
"drift/rejected_abs": 0.2144632786512375,
|
||
|
|
"epoch": 0.19586766989131732,
|
||
|
|
"grad_norm": 10.6875,
|
||
|
|
"learning_rate": 4.986304738420683e-07,
|
||
|
|
"logits/chosen": -2.1127889156341553,
|
||
|
|
"logits/rejected": -2.164623975753784,
|
||
|
|
"logps/chosen": -0.25030291080474854,
|
||
|
|
"logps/rejected": -0.26794177293777466,
|
||
|
|
"loss": 1.3577306270599365,
|
||
|
|
"loss/core": 1.3068211078643799,
|
||
|
|
"loss/preference_sft": 0.25030291080474854,
|
||
|
|
"loss/reference_anchor": 0.6537634134292603,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 3.287494659423828,
|
||
|
|
"rewards/margins": 1.1692345142364502,
|
||
|
|
"rewards/rejected": 2.118259906768799,
|
||
|
|
"step": 205
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2845247685909271,
|
||
|
|
"drift/rejected_abs": 0.14929194748401642,
|
||
|
|
"epoch": 0.20064493013256898,
|
||
|
|
"grad_norm": 28.625,
|
||
|
|
"learning_rate": 4.980012374199287e-07,
|
||
|
|
"logits/chosen": -2.107903003692627,
|
||
|
|
"logits/rejected": -2.131675958633423,
|
||
|
|
"logps/chosen": -0.3113544285297394,
|
||
|
|
"logps/rejected": -0.3212708830833435,
|
||
|
|
"loss": 1.2182084321975708,
|
||
|
|
"loss/core": 1.1718981266021729,
|
||
|
|
"loss/preference_sft": 0.3113544285297394,
|
||
|
|
"loss/reference_anchor": 0.5863357782363892,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.8425536155700684,
|
||
|
|
"rewards/margins": 1.352899193763733,
|
||
|
|
"rewards/rejected": 1.489654779434204,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.31894436478614807,
|
||
|
|
"drift/rejected_abs": 0.17038756608963013,
|
||
|
|
"epoch": 0.20542219037382062,
|
||
|
|
"grad_norm": 7.25,
|
||
|
|
"learning_rate": 4.972539658404792e-07,
|
||
|
|
"logits/chosen": -2.0803780555725098,
|
||
|
|
"logits/rejected": -2.071502208709717,
|
||
|
|
"logps/chosen": -0.27592846751213074,
|
||
|
|
"logps/rejected": -0.28498896956443787,
|
||
|
|
"loss": 1.6364405155181885,
|
||
|
|
"loss/core": 1.5752842426300049,
|
||
|
|
"loss/preference_sft": 0.27592846751213074,
|
||
|
|
"loss/reference_anchor": 0.78782057762146,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 3.186903953552246,
|
||
|
|
"rewards/margins": 1.491759181022644,
|
||
|
|
"rewards/rejected": 1.6951450109481812,
|
||
|
|
"step": 215
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1886764019727707,
|
||
|
|
"drift/rejected_abs": 0.10738885402679443,
|
||
|
|
"epoch": 0.21019945061507225,
|
||
|
|
"grad_norm": 34.25,
|
||
|
|
"learning_rate": 4.963890147645194e-07,
|
||
|
|
"logits/chosen": -2.0927789211273193,
|
||
|
|
"logits/rejected": -2.0071330070495605,
|
||
|
|
"logps/chosen": -0.3026772439479828,
|
||
|
|
"logps/rejected": -0.3015129268169403,
|
||
|
|
"loss": 0.2290460765361786,
|
||
|
|
"loss/core": 0.21856117248535156,
|
||
|
|
"loss/preference_sft": 0.3026772439479828,
|
||
|
|
"loss/reference_anchor": 0.10953140258789062,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.8838809728622437,
|
||
|
|
"rewards/margins": 0.8116692304611206,
|
||
|
|
"rewards/rejected": 1.072211503982544,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3618369996547699,
|
||
|
|
"drift/rejected_abs": 0.1408586949110031,
|
||
|
|
"epoch": 0.2149767108563239,
|
||
|
|
"grad_norm": 8.875,
|
||
|
|
"learning_rate": 4.95406795861916e-07,
|
||
|
|
"logits/chosen": -2.1790313720703125,
|
||
|
|
"logits/rejected": -2.1682028770446777,
|
||
|
|
"logps/chosen": -0.2909870743751526,
|
||
|
|
"logps/rejected": -0.2756298780441284,
|
||
|
|
"loss": 0.8731632232666016,
|
||
|
|
"loss/core": 0.8394603729248047,
|
||
|
|
"loss/preference_sft": 0.2909870743751526,
|
||
|
|
"loss/reference_anchor": 0.4202722907066345,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 3.6181743144989014,
|
||
|
|
"rewards/margins": 2.2104671001434326,
|
||
|
|
"rewards/rejected": 1.4077072143554688,
|
||
|
|
"step": 225
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.18520915508270264,
|
||
|
|
"drift/rejected_abs": 0.07412123680114746,
|
||
|
|
"epoch": 0.21975397109757555,
|
||
|
|
"grad_norm": 74.0,
|
||
|
|
"learning_rate": 4.943077766156697e-07,
|
||
|
|
"logits/chosen": -2.2688801288604736,
|
||
|
|
"logits/rejected": -2.276559829711914,
|
||
|
|
"logps/chosen": -0.2821432948112488,
|
||
|
|
"logps/rejected": -0.30950599908828735,
|
||
|
|
"loss": 0.19297373294830322,
|
||
|
|
"loss/core": 0.18392691016197205,
|
||
|
|
"loss/preference_sft": 0.2821432948112488,
|
||
|
|
"loss/reference_anchor": 0.09240987151861191,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.8502082824707031,
|
||
|
|
"rewards/margins": 1.1134107112884521,
|
||
|
|
"rewards/rejected": 0.7367974519729614,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1905185729265213,
|
||
|
|
"drift/rejected_abs": 0.13251249492168427,
|
||
|
|
"epoch": 0.2245312313388272,
|
||
|
|
"grad_norm": 23.625,
|
||
|
|
"learning_rate": 4.930924800994191e-07,
|
||
|
|
"logits/chosen": -2.005171298980713,
|
||
|
|
"logits/rejected": -2.0407776832580566,
|
||
|
|
"logps/chosen": -0.29280489683151245,
|
||
|
|
"logps/rejected": -0.28741219639778137,
|
||
|
|
"loss": 0.4691334366798401,
|
||
|
|
"loss/core": 0.45003873109817505,
|
||
|
|
"loss/preference_sft": 0.29280489683151245,
|
||
|
|
"loss/reference_anchor": 0.2253153771162033,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.9031035900115967,
|
||
|
|
"rewards/margins": 0.58293217420578,
|
||
|
|
"rewards/rejected": 1.3201711177825928,
|
||
|
|
"step": 235
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.17637206614017487,
|
||
|
|
"drift/rejected_abs": 0.10210146754980087,
|
||
|
|
"epoch": 0.22930849158007882,
|
||
|
|
"grad_norm": 9.8125,
|
||
|
|
"learning_rate": 4.917614847284858e-07,
|
||
|
|
"logits/chosen": -2.019641399383545,
|
||
|
|
"logits/rejected": -2.0390303134918213,
|
||
|
|
"logps/chosen": -0.294040709733963,
|
||
|
|
"logps/rejected": -0.2948092818260193,
|
||
|
|
"loss": 0.14991113543510437,
|
||
|
|
"loss/core": 0.14234814047813416,
|
||
|
|
"loss/preference_sft": 0.294040709733963,
|
||
|
|
"loss/reference_anchor": 0.0714358538389206,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 1.762891411781311,
|
||
|
|
"rewards/margins": 0.7431442141532898,
|
||
|
|
"rewards/rejected": 1.019747257232666,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3610271215438843,
|
||
|
|
"drift/rejected_abs": 0.11232628673315048,
|
||
|
|
"epoch": 0.23408575182133046,
|
||
|
|
"grad_norm": 304.0,
|
||
|
|
"learning_rate": 4.903154239845797e-07,
|
||
|
|
"logits/chosen": -2.057809829711914,
|
||
|
|
"logits/rejected": -2.0461342334747314,
|
||
|
|
"logps/chosen": -0.2769336998462677,
|
||
|
|
"logps/rejected": -0.29468828439712524,
|
||
|
|
"loss": 1.2440202236175537,
|
||
|
|
"loss/core": 1.1970118284225464,
|
||
|
|
"loss/preference_sft": 0.2769336998462677,
|
||
|
|
"loss/reference_anchor": 0.5990844964981079,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.610271453857422,
|
||
|
|
"rewards/margins": 2.4941940307617188,
|
||
|
|
"rewards/rejected": 1.1160774230957031,
|
||
|
|
"step": 245
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2345919907093048,
|
||
|
|
"drift/rejected_abs": 0.10711641609668732,
|
||
|
|
"epoch": 0.23886301206258212,
|
||
|
|
"grad_norm": 21.625,
|
||
|
|
"learning_rate": 4.887549861142967e-07,
|
||
|
|
"logits/chosen": -1.89980149269104,
|
||
|
|
"logits/rejected": -1.978960633277893,
|
||
|
|
"logps/chosen": -0.2994912564754486,
|
||
|
|
"logps/rejected": -0.29508280754089355,
|
||
|
|
"loss": 0.3056420385837555,
|
||
|
|
"loss/core": 0.2923932373523712,
|
||
|
|
"loss/preference_sft": 0.2994912564754486,
|
||
|
|
"loss/reference_anchor": 0.14670124650001526,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 2.342444896697998,
|
||
|
|
"rewards/margins": 1.2790577411651611,
|
||
|
|
"rewards/rejected": 1.0633875131607056,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.244302898645401,
|
||
|
|
"drift/rejected_abs": 0.1597694456577301,
|
||
|
|
"epoch": 0.24364027230383375,
|
||
|
|
"grad_norm": 13.75,
|
||
|
|
"learning_rate": 4.870809138015498e-07,
|
||
|
|
"logits/chosen": -2.2113137245178223,
|
||
|
|
"logits/rejected": -2.206145763397217,
|
||
|
|
"logps/chosen": -0.2858472466468811,
|
||
|
|
"logps/rejected": -0.28811052441596985,
|
||
|
|
"loss": 0.6415420174598694,
|
||
|
|
"loss/core": 0.6162611246109009,
|
||
|
|
"loss/preference_sft": 0.2858472466468811,
|
||
|
|
"loss/reference_anchor": 0.3084937036037445,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.442288398742676,
|
||
|
|
"rewards/margins": 0.8456929922103882,
|
||
|
|
"rewards/rejected": 1.5965955257415771,
|
||
|
|
"step": 255
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.23896794021129608,
|
||
|
|
"drift/rejected_abs": 0.15801241993904114,
|
||
|
|
"epoch": 0.2484175325450854,
|
||
|
|
"grad_norm": 0.8046875,
|
||
|
|
"learning_rate": 4.852940038140927e-07,
|
||
|
|
"logits/chosen": -2.1472392082214355,
|
||
|
|
"logits/rejected": -2.219313859939575,
|
||
|
|
"logps/chosen": -0.2888643741607666,
|
||
|
|
"logps/rejected": -0.27107033133506775,
|
||
|
|
"loss": 0.5575072765350342,
|
||
|
|
"loss/core": 0.5352469682693481,
|
||
|
|
"loss/preference_sft": 0.2888643741607666,
|
||
|
|
"loss/reference_anchor": 0.2679183781147003,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.3884191513061523,
|
||
|
|
"rewards/margins": 0.8117639422416687,
|
||
|
|
"rewards/rejected": 1.5766551494598389,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1222555860877037,
|
||
|
|
"drift/rejected_abs": 0.11597619950771332,
|
||
|
|
"epoch": 0.25319479278633705,
|
||
|
|
"grad_norm": 1.203125,
|
||
|
|
"learning_rate": 4.833951066243004e-07,
|
||
|
|
"logits/chosen": -2.0287933349609375,
|
||
|
|
"logits/rejected": -2.036017656326294,
|
||
|
|
"logps/chosen": -0.3003501892089844,
|
||
|
|
"logps/rejected": -0.3035464286804199,
|
||
|
|
"loss": 0.44168415665626526,
|
||
|
|
"loss/core": 0.42354241013526917,
|
||
|
|
"loss/preference_sft": 0.3003501892089844,
|
||
|
|
"loss/reference_anchor": 0.21185505390167236,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.2217209339141846,
|
||
|
|
"rewards/margins": 0.06751598417758942,
|
||
|
|
"rewards/rejected": 1.154205083847046,
|
||
|
|
"step": 265
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3274751305580139,
|
||
|
|
"drift/rejected_abs": 0.20972244441509247,
|
||
|
|
"epoch": 0.2579720530275887,
|
||
|
|
"grad_norm": 13.0,
|
||
|
|
"learning_rate": 4.813851260043915e-07,
|
||
|
|
"logits/chosen": -1.9322869777679443,
|
||
|
|
"logits/rejected": -2.0220537185668945,
|
||
|
|
"logps/chosen": -0.2892109155654907,
|
||
|
|
"logps/rejected": -0.2937650978565216,
|
||
|
|
"loss": 1.1844274997711182,
|
||
|
|
"loss/core": 1.1395008563995361,
|
||
|
|
"loss/preference_sft": 0.2892109155654907,
|
||
|
|
"loss/reference_anchor": 0.5701011419296265,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.274138927459717,
|
||
|
|
"rewards/margins": 1.1782522201538086,
|
||
|
|
"rewards/rejected": 2.0958871841430664,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.23182424902915955,
|
||
|
|
"drift/rejected_abs": 0.12205634266138077,
|
||
|
|
"epoch": 0.2627493132688403,
|
||
|
|
"grad_norm": 35.0,
|
||
|
|
"learning_rate": 4.79265018596281e-07,
|
||
|
|
"logits/chosen": -2.081444501876831,
|
||
|
|
"logits/rejected": -2.0351366996765137,
|
||
|
|
"logps/chosen": -0.2973952293395996,
|
||
|
|
"logps/rejected": -0.30073636770248413,
|
||
|
|
"loss": 0.4742198586463928,
|
||
|
|
"loss/core": 0.45490771532058716,
|
||
|
|
"loss/preference_sft": 0.2973952293395996,
|
||
|
|
"loss/reference_anchor": 0.22775478661060333,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.3158555030822754,
|
||
|
|
"rewards/margins": 1.159383773803711,
|
||
|
|
"rewards/rejected": 1.1564717292785645,
|
||
|
|
"step": 275
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.20490439236164093,
|
||
|
|
"drift/rejected_abs": 0.08539918065071106,
|
||
|
|
"epoch": 0.26752657351009196,
|
||
|
|
"grad_norm": 4.1875,
|
||
|
|
"learning_rate": 4.770357934562704e-07,
|
||
|
|
"logits/chosen": -2.0511467456817627,
|
||
|
|
"logits/rejected": -1.9675155878067017,
|
||
|
|
"logps/chosen": -0.2882925868034363,
|
||
|
|
"logps/rejected": -0.3074631094932556,
|
||
|
|
"loss": 0.22447428107261658,
|
||
|
|
"loss/core": 0.21425719559192657,
|
||
|
|
"loss/preference_sft": 0.2882925868034363,
|
||
|
|
"loss/reference_anchor": 0.1073986068367958,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.044360876083374,
|
||
|
|
"rewards/margins": 1.2838475704193115,
|
||
|
|
"rewards/rejected": 0.7605134844779968,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2618315815925598,
|
||
|
|
"drift/rejected_abs": 0.1607695072889328,
|
||
|
|
"epoch": 0.2723038337513436,
|
||
|
|
"grad_norm": 245.0,
|
||
|
|
"learning_rate": 4.746985115747917e-07,
|
||
|
|
"logits/chosen": -2.0746307373046875,
|
||
|
|
"logits/rejected": -2.0701193809509277,
|
||
|
|
"logps/chosen": -0.2960381507873535,
|
||
|
|
"logps/rejected": -0.2952577471733093,
|
||
|
|
"loss": 0.5513629913330078,
|
||
|
|
"loss/core": 0.529272198677063,
|
||
|
|
"loss/preference_sft": 0.2960381507873535,
|
||
|
|
"loss/reference_anchor": 0.2649388313293457,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.616506814956665,
|
||
|
|
"rewards/margins": 1.0125617980957031,
|
||
|
|
"rewards/rejected": 1.6039451360702515,
|
||
|
|
"step": 285
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.14695926010608673,
|
||
|
|
"drift/rejected_abs": 0.14160466194152832,
|
||
|
|
"epoch": 0.27708109399259523,
|
||
|
|
"grad_norm": 676.0,
|
||
|
|
"learning_rate": 4.722542853714341e-07,
|
||
|
|
"logits/chosen": -2.154064655303955,
|
||
|
|
"logits/rejected": -2.165696144104004,
|
||
|
|
"logps/chosen": -0.28789079189300537,
|
||
|
|
"logps/rejected": -0.29165560007095337,
|
||
|
|
"loss": 0.3904039263725281,
|
||
|
|
"loss/core": 0.3742069602012634,
|
||
|
|
"loss/preference_sft": 0.28789079189300537,
|
||
|
|
"loss/reference_anchor": 0.18717019259929657,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.4682142734527588,
|
||
|
|
"rewards/margins": 0.05419105291366577,
|
||
|
|
"rewards/rejected": 1.4140231609344482,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.13077983260154724,
|
||
|
|
"drift/rejected_abs": 0.06749530881643295,
|
||
|
|
"epoch": 0.28185835423384686,
|
||
|
|
"grad_norm": 8.625,
|
||
|
|
"learning_rate": 4.697042781654913e-07,
|
||
|
|
"logits/chosen": -2.5517144203186035,
|
||
|
|
"logits/rejected": -2.51605486869812,
|
||
|
|
"logps/chosen": -0.24481797218322754,
|
||
|
|
"logps/rejected": -0.2569727301597595,
|
||
|
|
"loss": 0.06375555694103241,
|
||
|
|
"loss/core": 0.05966717004776001,
|
||
|
|
"loss/preference_sft": 0.24481797218322754,
|
||
|
|
"loss/reference_anchor": 0.03002994880080223,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 1.307100534439087,
|
||
|
|
"rewards/margins": 0.6367514133453369,
|
||
|
|
"rewards/rejected": 0.6703491806983948,
|
||
|
|
"step": 295
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3725462555885315,
|
||
|
|
"drift/rejected_abs": 0.10222160816192627,
|
||
|
|
"epoch": 0.28663561447509855,
|
||
|
|
"grad_norm": 105.5,
|
||
|
|
"learning_rate": 4.670497036222856e-07,
|
||
|
|
"logits/chosen": -1.96623957157135,
|
||
|
|
"logits/rejected": -1.934470772743225,
|
||
|
|
"logps/chosen": -0.3124147355556488,
|
||
|
|
"logps/rejected": -0.32228654623031616,
|
||
|
|
"loss": 1.024113416671753,
|
||
|
|
"loss/core": 0.9847720861434937,
|
||
|
|
"loss/preference_sft": 0.3124147355556488,
|
||
|
|
"loss/reference_anchor": 0.49330902099609375,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 3.7237000465393066,
|
||
|
|
"rewards/margins": 2.717276096343994,
|
||
|
|
"rewards/rejected": 1.006424069404602,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21607303619384766,
|
||
|
|
"drift/rejected_abs": 0.08962901681661606,
|
||
|
|
"epoch": 0.2914128747163502,
|
||
|
|
"grad_norm": 13.0,
|
||
|
|
"learning_rate": 4.642918251755281e-07,
|
||
|
|
"logits/chosen": -2.1090660095214844,
|
||
|
|
"logits/rejected": -2.171133279800415,
|
||
|
|
"logps/chosen": -0.2694811224937439,
|
||
|
|
"logps/rejected": -0.26096317172050476,
|
||
|
|
"loss": 0.23768694698810577,
|
||
|
|
"loss/core": 0.2271147072315216,
|
||
|
|
"loss/preference_sft": 0.2694811224937439,
|
||
|
|
"loss/reference_anchor": 0.11401481926441193,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.1587884426116943,
|
||
|
|
"rewards/margins": 1.2635397911071777,
|
||
|
|
"rewards/rejected": 0.895248532295227,
|
||
|
|
"step": 305
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2609344720840454,
|
||
|
|
"drift/rejected_abs": 0.2091202735900879,
|
||
|
|
"epoch": 0.2961901349576018,
|
||
|
|
"grad_norm": 83.0,
|
||
|
|
"learning_rate": 4.614319554259933e-07,
|
||
|
|
"logits/chosen": -2.1155569553375244,
|
||
|
|
"logits/rejected": -2.0249547958374023,
|
||
|
|
"logps/chosen": -0.2733913064002991,
|
||
|
|
"logps/rejected": -0.2829728126525879,
|
||
|
|
"loss": 0.948021411895752,
|
||
|
|
"loss/core": 0.9117604494094849,
|
||
|
|
"loss/preference_sft": 0.2733913064002991,
|
||
|
|
"loss/reference_anchor": 0.45614108443260193,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.6087353229522705,
|
||
|
|
"rewards/margins": 0.6165834665298462,
|
||
|
|
"rewards/rejected": 1.9921518564224243,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2914541959762573,
|
||
|
|
"drift/rejected_abs": 0.13409768044948578,
|
||
|
|
"epoch": 0.30096739519885346,
|
||
|
|
"grad_norm": 5.875,
|
||
|
|
"learning_rate": 4.58471455516792e-07,
|
||
|
|
"logits/chosen": -1.945861577987671,
|
||
|
|
"logits/rejected": -2.051717519760132,
|
||
|
|
"logps/chosen": -0.26913851499557495,
|
||
|
|
"logps/rejected": -0.27637916803359985,
|
||
|
|
"loss": 0.48234397172927856,
|
||
|
|
"loss/core": 0.46292680501937866,
|
||
|
|
"loss/preference_sft": 0.26913851499557495,
|
||
|
|
"loss/reference_anchor": 0.23198194801807404,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 2.9132609367370605,
|
||
|
|
"rewards/margins": 1.5744664669036865,
|
||
|
|
"rewards/rejected": 1.3387947082519531,
|
||
|
|
"step": 315
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.15800070762634277,
|
||
|
|
"drift/rejected_abs": 0.12147839367389679,
|
||
|
|
"epoch": 0.3057446554401051,
|
||
|
|
"grad_norm": 3.5625,
|
||
|
|
"learning_rate": 4.5541173448554095e-07,
|
||
|
|
"logits/chosen": -1.986737847328186,
|
||
|
|
"logits/rejected": -2.0025219917297363,
|
||
|
|
"logps/chosen": -0.29054489731788635,
|
||
|
|
"logps/rejected": -0.29405659437179565,
|
||
|
|
"loss": 0.2033577710390091,
|
||
|
|
"loss/core": 0.19389116764068604,
|
||
|
|
"loss/preference_sft": 0.29054489731788635,
|
||
|
|
"loss/reference_anchor": 0.09716695547103882,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 1.5768026113510132,
|
||
|
|
"rewards/margins": 0.3625461161136627,
|
||
|
|
"rewards/rejected": 1.2142564058303833,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.37172552943229675,
|
||
|
|
"drift/rejected_abs": 0.15326622128486633,
|
||
|
|
"epoch": 0.31052191568135673,
|
||
|
|
"grad_norm": 548.0,
|
||
|
|
"learning_rate": 4.5225424859373684e-07,
|
||
|
|
"logits/chosen": -2.2413594722747803,
|
||
|
|
"logits/rejected": -2.1880881786346436,
|
||
|
|
"logps/chosen": -0.28455644845962524,
|
||
|
|
"logps/rejected": -0.2839685380458832,
|
||
|
|
"loss": 1.9195263385772705,
|
||
|
|
"loss/core": 1.848060965538025,
|
||
|
|
"loss/preference_sft": 0.28455644845962524,
|
||
|
|
"loss/reference_anchor": 0.9244130849838257,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 3.716318130493164,
|
||
|
|
"rewards/margins": 2.1848549842834473,
|
||
|
|
"rewards/rejected": 1.5314635038375854,
|
||
|
|
"step": 325
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.20601582527160645,
|
||
|
|
"drift/rejected_abs": 0.11289147287607193,
|
||
|
|
"epoch": 0.31529917592260837,
|
||
|
|
"grad_norm": 2.765625,
|
||
|
|
"learning_rate": 4.4900050063365547e-07,
|
||
|
|
"logits/chosen": -2.2116036415100098,
|
||
|
|
"logits/rejected": -2.1998820304870605,
|
||
|
|
"logps/chosen": -0.27572888135910034,
|
||
|
|
"logps/rejected": -0.2708529829978943,
|
||
|
|
"loss": 0.31742313504219055,
|
||
|
|
"loss/core": 0.30393102765083313,
|
||
|
|
"loss/preference_sft": 0.27572888135910034,
|
||
|
|
"loss/reference_anchor": 0.15232183039188385,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.0594685077667236,
|
||
|
|
"rewards/margins": 0.9315606951713562,
|
||
|
|
"rewards/rejected": 1.1279077529907227,
|
||
|
|
"step": 330
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21230271458625793,
|
||
|
|
"drift/rejected_abs": 0.08433900028467178,
|
||
|
|
"epoch": 0.32007643616386,
|
||
|
|
"grad_norm": 320.0,
|
||
|
|
"learning_rate": 4.4565203921310344e-07,
|
||
|
|
"logits/chosen": -2.03387188911438,
|
||
|
|
"logits/rejected": -2.0167250633239746,
|
||
|
|
"logps/chosen": -0.3023034930229187,
|
||
|
|
"logps/rejected": -0.3277498781681061,
|
||
|
|
"loss": 0.46993932127952576,
|
||
|
|
"loss/core": 0.45074281096458435,
|
||
|
|
"loss/preference_sft": 0.3023034930229187,
|
||
|
|
"loss/reference_anchor": 0.22572243213653564,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.119868755340576,
|
||
|
|
"rewards/margins": 1.523357629776001,
|
||
|
|
"rewards/rejected": 0.5965113043785095,
|
||
|
|
"step": 335
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.20402033627033234,
|
||
|
|
"drift/rejected_abs": 0.08628232777118683,
|
||
|
|
"epoch": 0.3248536964051117,
|
||
|
|
"grad_norm": 154.0,
|
||
|
|
"learning_rate": 4.422104580183649e-07,
|
||
|
|
"logits/chosen": -2.082024335861206,
|
||
|
|
"logits/rejected": -2.2043795585632324,
|
||
|
|
"logps/chosen": -0.28011229634284973,
|
||
|
|
"logps/rejected": -0.28280124068260193,
|
||
|
|
"loss": 0.32376912236213684,
|
||
|
|
"loss/core": 0.3100174069404602,
|
||
|
|
"loss/preference_sft": 0.28011229634284973,
|
||
|
|
"loss/reference_anchor": 0.15534545481204987,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.039736032485962,
|
||
|
|
"rewards/margins": 1.180772066116333,
|
||
|
|
"rewards/rejected": 0.8589638471603394,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.16444484889507294,
|
||
|
|
"drift/rejected_abs": 0.08357802778482437,
|
||
|
|
"epoch": 0.3296309566463633,
|
||
|
|
"grad_norm": 8.875,
|
||
|
|
"learning_rate": 4.3867739505569303e-07,
|
||
|
|
"logits/chosen": -2.301851987838745,
|
||
|
|
"logits/rejected": -2.362597942352295,
|
||
|
|
"logps/chosen": -0.2823006212711334,
|
||
|
|
"logps/rejected": -0.2937981188297272,
|
||
|
|
"loss": 0.2071170061826706,
|
||
|
|
"loss/core": 0.19756881892681122,
|
||
|
|
"loss/preference_sft": 0.2823006212711334,
|
||
|
|
"loss/reference_anchor": 0.09907897561788559,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.6424602270126343,
|
||
|
|
"rewards/margins": 0.8107796907424927,
|
||
|
|
"rewards/rejected": 0.8316806554794312,
|
||
|
|
"step": 345
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.19968092441558838,
|
||
|
|
"drift/rejected_abs": 0.09401963651180267,
|
||
|
|
"epoch": 0.33440821688761496,
|
||
|
|
"grad_norm": 7.9375,
|
||
|
|
"learning_rate": 4.35054531871708e-07,
|
||
|
|
"logits/chosen": -2.045243978500366,
|
||
|
|
"logits/rejected": -2.024528980255127,
|
||
|
|
"logps/chosen": -0.26065462827682495,
|
||
|
|
"logps/rejected": -0.2573546767234802,
|
||
|
|
"loss": 0.21272404491901398,
|
||
|
|
"loss/core": 0.2031305581331253,
|
||
|
|
"loss/preference_sft": 0.26065462827682495,
|
||
|
|
"loss/reference_anchor": 0.10184787213802338,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.995410680770874,
|
||
|
|
"rewards/margins": 1.057207703590393,
|
||
|
|
"rewards/rejected": 0.9382032155990601,
|
||
|
|
"step": 350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1851430982351303,
|
||
|
|
"drift/rejected_abs": 0.1371186524629593,
|
||
|
|
"epoch": 0.3391854771288666,
|
||
|
|
"grad_norm": 9.6875,
|
||
|
|
"learning_rate": 4.3134359275307185e-07,
|
||
|
|
"logits/chosen": -2.0893471240997314,
|
||
|
|
"logits/rejected": -2.2154877185821533,
|
||
|
|
"logps/chosen": -0.2807328701019287,
|
||
|
|
"logps/rejected": -0.29229921102523804,
|
||
|
|
"loss": 0.4610310196876526,
|
||
|
|
"loss/core": 0.4423205256462097,
|
||
|
|
"loss/preference_sft": 0.2807328701019287,
|
||
|
|
"loss/reference_anchor": 0.22139939665794373,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 1.850562334060669,
|
||
|
|
"rewards/margins": 0.49644041061401367,
|
||
|
|
"rewards/rejected": 1.3541219234466553,
|
||
|
|
"step": 355
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2644307017326355,
|
||
|
|
"drift/rejected_abs": 0.12655285000801086,
|
||
|
|
"epoch": 0.34396273737011823,
|
||
|
|
"grad_norm": 140.0,
|
||
|
|
"learning_rate": 4.2754634390582133e-07,
|
||
|
|
"logits/chosen": -2.113999605178833,
|
||
|
|
"logits/rejected": -2.1762969493865967,
|
||
|
|
"logps/chosen": -0.2791525721549988,
|
||
|
|
"logps/rejected": -0.28297150135040283,
|
||
|
|
"loss": 0.6517452001571655,
|
||
|
|
"loss/core": 0.626146674156189,
|
||
|
|
"loss/preference_sft": 0.2791525721549988,
|
||
|
|
"loss/reference_anchor": 0.31339868903160095,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.643383026123047,
|
||
|
|
"rewards/margins": 1.3792022466659546,
|
||
|
|
"rewards/rejected": 1.2641807794570923,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.28687232732772827,
|
||
|
|
"drift/rejected_abs": 0.07955290377140045,
|
||
|
|
"epoch": 0.34873999761136987,
|
||
|
|
"grad_norm": 470.0,
|
||
|
|
"learning_rate": 4.236645926147493e-07,
|
||
|
|
"logits/chosen": -2.053063154220581,
|
||
|
|
"logits/rejected": -2.159447193145752,
|
||
|
|
"logps/chosen": -0.2858540415763855,
|
||
|
|
"logps/rejected": -0.28544557094573975,
|
||
|
|
"loss": 1.0082862377166748,
|
||
|
|
"loss/core": 0.9697242975234985,
|
||
|
|
"loss/preference_sft": 0.2858540415763855,
|
||
|
|
"loss/reference_anchor": 0.4855729043483734,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.865591049194336,
|
||
|
|
"rewards/margins": 2.0717544555664062,
|
||
|
|
"rewards/rejected": 0.793836772441864,
|
||
|
|
"step": 365
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.16405682265758514,
|
||
|
|
"drift/rejected_abs": 0.1052694097161293,
|
||
|
|
"epoch": 0.3535172578526215,
|
||
|
|
"grad_norm": 22.375,
|
||
|
|
"learning_rate": 4.1970018638323547e-07,
|
||
|
|
"logits/chosen": -2.2507824897766113,
|
||
|
|
"logits/rejected": -2.2962427139282227,
|
||
|
|
"logps/chosen": -0.28528791666030884,
|
||
|
|
"logps/rejected": -0.28648024797439575,
|
||
|
|
"loss": 0.18379460275173187,
|
||
|
|
"loss/core": 0.17507164180278778,
|
||
|
|
"loss/preference_sft": 0.28528791666030884,
|
||
|
|
"loss/reference_anchor": 0.08777736127376556,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.639823317527771,
|
||
|
|
"rewards/margins": 0.5898867249488831,
|
||
|
|
"rewards/rejected": 1.0499365329742432,
|
||
|
|
"step": 370
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.30081304907798767,
|
||
|
|
"drift/rejected_abs": 0.22256799042224884,
|
||
|
|
"epoch": 0.35829451809387314,
|
||
|
|
"grad_norm": 8.8125,
|
||
|
|
"learning_rate": 4.1565501205393436e-07,
|
||
|
|
"logits/chosen": -2.0259885787963867,
|
||
|
|
"logits/rejected": -2.004533290863037,
|
||
|
|
"logps/chosen": -0.2593756318092346,
|
||
|
|
"logps/rejected": -0.2595575451850891,
|
||
|
|
"loss": 0.8363986015319824,
|
||
|
|
"loss/core": 0.8042640686035156,
|
||
|
|
"loss/preference_sft": 0.2593756318092346,
|
||
|
|
"loss/reference_anchor": 0.4025222659111023,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 3.007532835006714,
|
||
|
|
"rewards/margins": 0.8016997575759888,
|
||
|
|
"rewards/rejected": 2.2058329582214355,
|
||
|
|
"step": 375
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2039259374141693,
|
||
|
|
"drift/rejected_abs": 0.16159279644489288,
|
||
|
|
"epoch": 0.36307177833512483,
|
||
|
|
"grad_norm": 17.125,
|
||
|
|
"learning_rate": 4.1153099491074093e-07,
|
||
|
|
"logits/chosen": -2.1002986431121826,
|
||
|
|
"logits/rejected": -2.1101698875427246,
|
||
|
|
"logps/chosen": -0.28150156140327454,
|
||
|
|
"logps/rejected": -0.29337161779403687,
|
||
|
|
"loss": 0.7919434905052185,
|
||
|
|
"loss/core": 0.7612730264663696,
|
||
|
|
"loss/preference_sft": 0.28150156140327454,
|
||
|
|
"loss/reference_anchor": 0.3807896077632904,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.036198854446411,
|
||
|
|
"rewards/margins": 0.47472089529037476,
|
||
|
|
"rewards/rejected": 1.5614778995513916,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21656015515327454,
|
||
|
|
"drift/rejected_abs": 0.1512974351644516,
|
||
|
|
"epoch": 0.36784903857637646,
|
||
|
|
"grad_norm": 30.0,
|
||
|
|
"learning_rate": 4.0733009776245937e-07,
|
||
|
|
"logits/chosen": -2.0933432579040527,
|
||
|
|
"logits/rejected": -2.113175630569458,
|
||
|
|
"logps/chosen": -0.27162864804267883,
|
||
|
|
"logps/rejected": -0.2761984169483185,
|
||
|
|
"loss": 0.38956135511398315,
|
||
|
|
"loss/core": 0.37349286675453186,
|
||
|
|
"loss/preference_sft": 0.27162864804267883,
|
||
|
|
"loss/reference_anchor": 0.18708345293998718,
|
||
|
|
"rewards/accuracies": 0.8125,
|
||
|
|
"rewards/chosen": 2.1639719009399414,
|
||
|
|
"rewards/margins": 0.6549761295318604,
|
||
|
|
"rewards/rejected": 1.5089956521987915,
|
||
|
|
"step": 385
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.17612993717193604,
|
||
|
|
"drift/rejected_abs": 0.08776899427175522,
|
||
|
|
"epoch": 0.3726262988176281,
|
||
|
|
"grad_norm": 2.015625,
|
||
|
|
"learning_rate": 4.0305432000861226e-07,
|
||
|
|
"logits/chosen": -1.8524726629257202,
|
||
|
|
"logits/rejected": -1.9306892156600952,
|
||
|
|
"logps/chosen": -0.3076761066913605,
|
||
|
|
"logps/rejected": -0.3044106662273407,
|
||
|
|
"loss": 0.21172335743904114,
|
||
|
|
"loss/core": 0.20182469487190247,
|
||
|
|
"loss/preference_sft": 0.3076761066913605,
|
||
|
|
"loss/reference_anchor": 0.10121461004018784,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.7569738626480103,
|
||
|
|
"rewards/margins": 0.8802940249443054,
|
||
|
|
"rewards/rejected": 0.8766800165176392,
|
||
|
|
"step": 390
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.16845259070396423,
|
||
|
|
"drift/rejected_abs": 0.07968206703662872,
|
||
|
|
"epoch": 0.37740355905887973,
|
||
|
|
"grad_norm": 18.375,
|
||
|
|
"learning_rate": 3.9870569668783533e-07,
|
||
|
|
"logits/chosen": -2.2020130157470703,
|
||
|
|
"logits/rejected": -2.3044018745422363,
|
||
|
|
"logps/chosen": -0.2947576642036438,
|
||
|
|
"logps/rejected": -0.2907804548740387,
|
||
|
|
"loss": 0.1688641607761383,
|
||
|
|
"loss/core": 0.1606094241142273,
|
||
|
|
"loss/preference_sft": 0.2947576642036438,
|
||
|
|
"loss/reference_anchor": 0.08058713376522064,
|
||
|
|
"rewards/accuracies": 0.9750000238418579,
|
||
|
|
"rewards/chosen": 1.6845258474349976,
|
||
|
|
"rewards/margins": 0.8892566561698914,
|
||
|
|
"rewards/rejected": 0.7952693104743958,
|
||
|
|
"step": 395
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2936556935310364,
|
||
|
|
"drift/rejected_abs": 0.08415323495864868,
|
||
|
|
"epoch": 0.38218081930013137,
|
||
|
|
"grad_norm": 53.25,
|
||
|
|
"learning_rate": 3.942862975093084e-07,
|
||
|
|
"logits/chosen": -2.0824859142303467,
|
||
|
|
"logits/rejected": -2.1172521114349365,
|
||
|
|
"logps/chosen": -0.2918018698692322,
|
||
|
|
"logps/rejected": -0.2934037148952484,
|
||
|
|
"loss": 1.0916287899017334,
|
||
|
|
"loss/core": 1.0500308275222778,
|
||
|
|
"loss/preference_sft": 0.2918018698692322,
|
||
|
|
"loss/reference_anchor": 0.5254592299461365,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.9353275299072266,
|
||
|
|
"rewards/margins": 2.0966556072235107,
|
||
|
|
"rewards/rejected": 0.8386720418930054,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.23455221951007843,
|
||
|
|
"drift/rejected_abs": 0.12242819368839264,
|
||
|
|
"epoch": 0.386958079541383,
|
||
|
|
"grad_norm": 6.0,
|
||
|
|
"learning_rate": 3.8979822586768666e-07,
|
||
|
|
"logits/chosen": -2.113771677017212,
|
||
|
|
"logits/rejected": -2.168362617492676,
|
||
|
|
"logps/chosen": -0.3119937777519226,
|
||
|
|
"logps/rejected": -0.30502599477767944,
|
||
|
|
"loss": 0.7244704961776733,
|
||
|
|
"loss/core": 0.69599449634552,
|
||
|
|
"loss/preference_sft": 0.3119937777519226,
|
||
|
|
"loss/reference_anchor": 0.3484812080860138,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.345109224319458,
|
||
|
|
"rewards/margins": 1.12223219871521,
|
||
|
|
"rewards/rejected": 1.222877025604248,
|
||
|
|
"step": 405
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.20881287753582,
|
||
|
|
"drift/rejected_abs": 0.11809121072292328,
|
||
|
|
"epoch": 0.39173533978263464,
|
||
|
|
"grad_norm": 21.875,
|
||
|
|
"learning_rate": 3.8524361784199847e-07,
|
||
|
|
"logits/chosen": -2.228954315185547,
|
||
|
|
"logits/rejected": -2.2588367462158203,
|
||
|
|
"logps/chosen": -0.2771432101726532,
|
||
|
|
"logps/rejected": -0.28896427154541016,
|
||
|
|
"loss": 0.2878721058368683,
|
||
|
|
"loss/core": 0.27545636892318726,
|
||
|
|
"loss/preference_sft": 0.2771432101726532,
|
||
|
|
"loss/reference_anchor": 0.13782820105552673,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.0866684913635254,
|
||
|
|
"rewards/margins": 0.9072427749633789,
|
||
|
|
"rewards/rejected": 1.179425597190857,
|
||
|
|
"step": 410
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.09865269809961319,
|
||
|
|
"drift/rejected_abs": 0.0373404435813427,
|
||
|
|
"epoch": 0.3965126000238863,
|
||
|
|
"grad_norm": 13.25,
|
||
|
|
"learning_rate": 3.806246411789872e-07,
|
||
|
|
"logits/chosen": -2.0978753566741943,
|
||
|
|
"logits/rejected": -2.176331043243408,
|
||
|
|
"logps/chosen": -0.33135882019996643,
|
||
|
|
"logps/rejected": -0.32153475284576416,
|
||
|
|
"loss": 0.06347070634365082,
|
||
|
|
"loss/core": 0.0587652213871479,
|
||
|
|
"loss/preference_sft": 0.33135882019996643,
|
||
|
|
"loss/reference_anchor": 0.029603928327560425,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 0.9844989776611328,
|
||
|
|
"rewards/margins": 0.61436527967453,
|
||
|
|
"rewards/rejected": 0.37013378739356995,
|
||
|
|
"step": 415
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.20410975813865662,
|
||
|
|
"drift/rejected_abs": 0.07133093476295471,
|
||
|
|
"epoch": 0.40128986026513797,
|
||
|
|
"grad_norm": 92.5,
|
||
|
|
"learning_rate": 3.7594349426138154e-07,
|
||
|
|
"logits/chosen": -2.3295199871063232,
|
||
|
|
"logits/rejected": -2.2967400550842285,
|
||
|
|
"logps/chosen": -0.28380584716796875,
|
||
|
|
"logps/rejected": -0.2998279929161072,
|
||
|
|
"loss": 0.35089415311813354,
|
||
|
|
"loss/core": 0.3361276388168335,
|
||
|
|
"loss/preference_sft": 0.28380584716796875,
|
||
|
|
"loss/reference_anchor": 0.16850566864013672,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.040860176086426,
|
||
|
|
"rewards/margins": 1.3300426006317139,
|
||
|
|
"rewards/rejected": 0.7108174562454224,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.22340837121009827,
|
||
|
|
"drift/rejected_abs": 0.16072794795036316,
|
||
|
|
"epoch": 0.4060671205063896,
|
||
|
|
"grad_norm": 186.0,
|
||
|
|
"learning_rate": 3.712024050615843e-07,
|
||
|
|
"logits/chosen": -1.9938865900039673,
|
||
|
|
"logits/rejected": -2.0516488552093506,
|
||
|
|
"logps/chosen": -0.2920078635215759,
|
||
|
|
"logps/rejected": -0.2692197859287262,
|
||
|
|
"loss": 0.8027600049972534,
|
||
|
|
"loss/core": 0.7715979814529419,
|
||
|
|
"loss/preference_sft": 0.2920078635215759,
|
||
|
|
"loss/reference_anchor": 0.38629359006881714,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.2311575412750244,
|
||
|
|
"rewards/margins": 0.626264214515686,
|
||
|
|
"rewards/rejected": 1.6048933267593384,
|
||
|
|
"step": 425
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.4396797716617584,
|
||
|
|
"drift/rejected_abs": 0.1412547528743744,
|
||
|
|
"epoch": 0.41084438074764124,
|
||
|
|
"grad_norm": 696.0,
|
||
|
|
"learning_rate": 3.664036300812778e-07,
|
||
|
|
"logits/chosen": -2.0478415489196777,
|
||
|
|
"logits/rejected": -2.0716514587402344,
|
||
|
|
"logps/chosen": -0.2803536057472229,
|
||
|
|
"logps/rejected": -0.26729705929756165,
|
||
|
|
"loss": 1.897403359413147,
|
||
|
|
"loss/core": 1.8267196416854858,
|
||
|
|
"loss/preference_sft": 0.2803536057472229,
|
||
|
|
"loss/reference_anchor": 0.914415717124939,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 4.394786834716797,
|
||
|
|
"rewards/margins": 2.983668804168701,
|
||
|
|
"rewards/rejected": 1.4111182689666748,
|
||
|
|
"step": 430
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1620672643184662,
|
||
|
|
"drift/rejected_abs": 0.10048636049032211,
|
||
|
|
"epoch": 0.41562164098889287,
|
||
|
|
"grad_norm": 8.6875,
|
||
|
|
"learning_rate": 3.615494532774522e-07,
|
||
|
|
"logits/chosen": -2.3244786262512207,
|
||
|
|
"logits/rejected": -2.30962872505188,
|
||
|
|
"logps/chosen": -0.27878594398498535,
|
||
|
|
"logps/rejected": -0.2934947609901428,
|
||
|
|
"loss": 0.2484576255083084,
|
||
|
|
"loss/core": 0.23745962977409363,
|
||
|
|
"loss/preference_sft": 0.27878594398498535,
|
||
|
|
"loss/reference_anchor": 0.11876034736633301,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.6202609539031982,
|
||
|
|
"rewards/margins": 0.6176348328590393,
|
||
|
|
"rewards/rejected": 1.0026261806488037,
|
||
|
|
"step": 435
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.29970285296440125,
|
||
|
|
"drift/rejected_abs": 0.13486061990261078,
|
||
|
|
"epoch": 0.4203989012301445,
|
||
|
|
"grad_norm": 490.0,
|
||
|
|
"learning_rate": 3.5664218497536456e-07,
|
||
|
|
"logits/chosen": -2.0376389026641846,
|
||
|
|
"logits/rejected": -2.108769416809082,
|
||
|
|
"logps/chosen": -0.304705411195755,
|
||
|
|
"logps/rejected": -0.3040980398654938,
|
||
|
|
"loss": 1.0282645225524902,
|
||
|
|
"loss/core": 0.9888750314712524,
|
||
|
|
"loss/preference_sft": 0.304705411195755,
|
||
|
|
"loss/reference_anchor": 0.4947218894958496,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.992257595062256,
|
||
|
|
"rewards/margins": 1.6467006206512451,
|
||
|
|
"rewards/rejected": 1.3455569744110107,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2948855757713318,
|
||
|
|
"drift/rejected_abs": 0.1586662232875824,
|
||
|
|
"epoch": 0.42517616147139614,
|
||
|
|
"grad_norm": 40.0,
|
||
|
|
"learning_rate": 3.516841607689501e-07,
|
||
|
|
"logits/chosen": -1.9140384197235107,
|
||
|
|
"logits/rejected": -1.8901231288909912,
|
||
|
|
"logps/chosen": -0.29691654443740845,
|
||
|
|
"logps/rejected": -0.2970215678215027,
|
||
|
|
"loss": 0.5381649732589722,
|
||
|
|
"loss/core": 0.5165340900421143,
|
||
|
|
"loss/preference_sft": 0.29691654443740845,
|
||
|
|
"loss/reference_anchor": 0.2587199807167053,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.9474780559539795,
|
||
|
|
"rewards/margins": 1.3833874464035034,
|
||
|
|
"rewards/rejected": 1.5640907287597656,
|
||
|
|
"step": 445
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3455376923084259,
|
||
|
|
"drift/rejected_abs": 0.15802158415317535,
|
||
|
|
"epoch": 0.4299534217126478,
|
||
|
|
"grad_norm": 426.0,
|
||
|
|
"learning_rate": 3.4667774040920515e-07,
|
||
|
|
"logits/chosen": -1.9539282321929932,
|
||
|
|
"logits/rejected": -1.9721944332122803,
|
||
|
|
"logps/chosen": -0.29860302805900574,
|
||
|
|
"logps/rejected": -0.29259243607521057,
|
||
|
|
"loss": 0.9283429980278015,
|
||
|
|
"loss/core": 0.8925935626029968,
|
||
|
|
"loss/preference_sft": 0.29860302805900574,
|
||
|
|
"loss/reference_anchor": 0.4467983841896057,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 3.454946994781494,
|
||
|
|
"rewards/margins": 1.8800350427627563,
|
||
|
|
"rewards/rejected": 1.5749118328094482,
|
||
|
|
"step": 450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3471214175224304,
|
||
|
|
"drift/rejected_abs": 0.14864817261695862,
|
||
|
|
"epoch": 0.4347306819538994,
|
||
|
|
"grad_norm": 200.0,
|
||
|
|
"learning_rate": 3.416253066810743e-07,
|
||
|
|
"logits/chosen": -1.918105125427246,
|
||
|
|
"logits/rejected": -2.1081409454345703,
|
||
|
|
"logps/chosen": -0.3223501741886139,
|
||
|
|
"logps/rejected": -0.31352201104164124,
|
||
|
|
"loss": 1.5264394283294678,
|
||
|
|
"loss/core": 1.4688888788223267,
|
||
|
|
"loss/preference_sft": 0.3223501741886139,
|
||
|
|
"loss/reference_anchor": 0.7351065278053284,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 3.4693026542663574,
|
||
|
|
"rewards/margins": 1.9863710403442383,
|
||
|
|
"rewards/rejected": 1.4829317331314087,
|
||
|
|
"step": 455
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.36486533284187317,
|
||
|
|
"drift/rejected_abs": 0.1869690865278244,
|
||
|
|
"epoch": 0.4395079421951511,
|
||
|
|
"grad_norm": 33.75,
|
||
|
|
"learning_rate": 3.3652926426937325e-07,
|
||
|
|
"logits/chosen": -2.0386669635772705,
|
||
|
|
"logits/rejected": -2.027118444442749,
|
||
|
|
"logps/chosen": -0.29761844873428345,
|
||
|
|
"logps/rejected": -0.3065989017486572,
|
||
|
|
"loss": 1.2015959024429321,
|
||
|
|
"loss/core": 1.155981421470642,
|
||
|
|
"loss/preference_sft": 0.29761844873428345,
|
||
|
|
"loss/reference_anchor": 0.5784320831298828,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 3.648583173751831,
|
||
|
|
"rewards/margins": 1.7795528173446655,
|
||
|
|
"rewards/rejected": 1.869030237197876,
|
||
|
|
"step": 460
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2300829440355301,
|
||
|
|
"drift/rejected_abs": 0.09263820201158524,
|
||
|
|
"epoch": 0.44428520243640274,
|
||
|
|
"grad_norm": 11.0,
|
||
|
|
"learning_rate": 3.3139203861428914e-07,
|
||
|
|
"logits/chosen": -1.8710740804672241,
|
||
|
|
"logits/rejected": -1.9699445962905884,
|
||
|
|
"logps/chosen": -0.29734542965888977,
|
||
|
|
"logps/rejected": -0.29221880435943604,
|
||
|
|
"loss": 0.24437861144542694,
|
||
|
|
"loss/core": 0.23336219787597656,
|
||
|
|
"loss/preference_sft": 0.29734542965888977,
|
||
|
|
"loss/reference_anchor": 0.1171511560678482,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 2.299163818359375,
|
||
|
|
"rewards/margins": 1.3750882148742676,
|
||
|
|
"rewards/rejected": 0.9240756034851074,
|
||
|
|
"step": 465
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.297715425491333,
|
||
|
|
"drift/rejected_abs": 0.12405421584844589,
|
||
|
|
"epoch": 0.4490624626776544,
|
||
|
|
"grad_norm": 1064.0,
|
||
|
|
"learning_rate": 3.262160747570027e-07,
|
||
|
|
"logits/chosen": -2.1081130504608154,
|
||
|
|
"logits/rejected": -2.1759676933288574,
|
||
|
|
"logps/chosen": -0.282444030046463,
|
||
|
|
"logps/rejected": -0.26066750288009644,
|
||
|
|
"loss": 1.1672271490097046,
|
||
|
|
"loss/core": 1.122953176498413,
|
||
|
|
"loss/preference_sft": 0.282444030046463,
|
||
|
|
"loss/reference_anchor": 0.5620765686035156,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 2.976147174835205,
|
||
|
|
"rewards/margins": 1.8132495880126953,
|
||
|
|
"rewards/rejected": 1.1628977060317993,
|
||
|
|
"step": 470
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1827501803636551,
|
||
|
|
"drift/rejected_abs": 0.14713791012763977,
|
||
|
|
"epoch": 0.453839722918906,
|
||
|
|
"grad_norm": 82.5,
|
||
|
|
"learning_rate": 3.210038361759807e-07,
|
||
|
|
"logits/chosen": -2.0727429389953613,
|
||
|
|
"logits/rejected": -2.012450933456421,
|
||
|
|
"logps/chosen": -0.2958831191062927,
|
||
|
|
"logps/rejected": -0.32066217064857483,
|
||
|
|
"loss": 0.4937218129634857,
|
||
|
|
"loss/core": 0.47373858094215393,
|
||
|
|
"loss/preference_sft": 0.2958831191062927,
|
||
|
|
"loss/reference_anchor": 0.23685435950756073,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.8267005681991577,
|
||
|
|
"rewards/margins": 0.35839390754699707,
|
||
|
|
"rewards/rejected": 1.4683068990707397,
|
||
|
|
"step": 475
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.23704016208648682,
|
||
|
|
"drift/rejected_abs": 0.10950712114572525,
|
||
|
|
"epoch": 0.45861698316015764,
|
||
|
|
"grad_norm": 6.28125,
|
||
|
|
"learning_rate": 3.1575780361449364e-07,
|
||
|
|
"logits/chosen": -2.2403244972229004,
|
||
|
|
"logits/rejected": -2.296858310699463,
|
||
|
|
"logps/chosen": -0.2744103670120239,
|
||
|
|
"logps/rejected": -0.27569180727005005,
|
||
|
|
"loss": 0.5795036554336548,
|
||
|
|
"loss/core": 0.5565477013587952,
|
||
|
|
"loss/preference_sft": 0.2744103670120239,
|
||
|
|
"loss/reference_anchor": 0.2786380648612976,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.36838960647583,
|
||
|
|
"rewards/margins": 1.3759324550628662,
|
||
|
|
"rewards/rejected": 0.9924572110176086,
|
||
|
|
"step": 480
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.17343619465827942,
|
||
|
|
"drift/rejected_abs": 0.08826018869876862,
|
||
|
|
"epoch": 0.4633942434014093,
|
||
|
|
"grad_norm": 13.4375,
|
||
|
|
"learning_rate": 3.104804738999169e-07,
|
||
|
|
"logits/chosen": -2.1836743354797363,
|
||
|
|
"logits/rejected": -2.1913857460021973,
|
||
|
|
"logps/chosen": -0.2924710214138031,
|
||
|
|
"logps/rejected": -0.2964377999305725,
|
||
|
|
"loss": 0.16332216560840607,
|
||
|
|
"loss/core": 0.15528026223182678,
|
||
|
|
"loss/preference_sft": 0.2924710214138031,
|
||
|
|
"loss/reference_anchor": 0.07797853648662567,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.7339932918548584,
|
||
|
|
"rewards/margins": 0.8536950349807739,
|
||
|
|
"rewards/rejected": 0.8802981376647949,
|
||
|
|
"step": 485
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2068719118833542,
|
||
|
|
"drift/rejected_abs": 0.1048976331949234,
|
||
|
|
"epoch": 0.4681715036426609,
|
||
|
|
"grad_norm": 21.25,
|
||
|
|
"learning_rate": 3.0517435875537536e-07,
|
||
|
|
"logits/chosen": -2.1321020126342773,
|
||
|
|
"logits/rejected": -2.1710128784179688,
|
||
|
|
"logps/chosen": -0.3080752491950989,
|
||
|
|
"logps/rejected": -0.2949379086494446,
|
||
|
|
"loss": 0.30973607301712036,
|
||
|
|
"loss/core": 0.2963031232357025,
|
||
|
|
"loss/preference_sft": 0.3080752491950989,
|
||
|
|
"loss/reference_anchor": 0.1482984721660614,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.0682833194732666,
|
||
|
|
"rewards/margins": 1.0205515623092651,
|
||
|
|
"rewards/rejected": 1.047731637954712,
|
||
|
|
"step": 490
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.26098620891571045,
|
||
|
|
"drift/rejected_abs": 0.0885407105088234,
|
||
|
|
"epoch": 0.47294876388391255,
|
||
|
|
"grad_norm": 20.875,
|
||
|
|
"learning_rate": 2.998419836042993e-07,
|
||
|
|
"logits/chosen": -2.2028145790100098,
|
||
|
|
"logits/rejected": -2.297614097595215,
|
||
|
|
"logps/chosen": -0.29816746711730957,
|
||
|
|
"logps/rejected": -0.2916865348815918,
|
||
|
|
"loss": 0.7447795271873474,
|
||
|
|
"loss/core": 0.7156817317008972,
|
||
|
|
"loss/preference_sft": 0.29816746711730957,
|
||
|
|
"loss/reference_anchor": 0.3581535518169403,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.6079015731811523,
|
||
|
|
"rewards/margins": 1.724554419517517,
|
||
|
|
"rewards/rejected": 0.8833469152450562,
|
||
|
|
"step": 495
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.41630882024765015,
|
||
|
|
"drift/rejected_abs": 0.20278768241405487,
|
||
|
|
"epoch": 0.47772602412516424,
|
||
|
|
"grad_norm": 123.0,
|
||
|
|
"learning_rate": 2.9448588636846043e-07,
|
||
|
|
"logits/chosen": -1.9985599517822266,
|
||
|
|
"logits/rejected": -1.949119210243225,
|
||
|
|
"logps/chosen": -0.28114375472068787,
|
||
|
|
"logps/rejected": -0.30273741483688354,
|
||
|
|
"loss": 2.383561134338379,
|
||
|
|
"loss/core": 2.295335292816162,
|
||
|
|
"loss/preference_sft": 0.28114375472068787,
|
||
|
|
"loss/reference_anchor": 1.1482231616973877,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 4.163088798522949,
|
||
|
|
"rewards/margins": 2.138087511062622,
|
||
|
|
"rewards/rejected": 2.025001049041748,
|
||
|
|
"step": 500
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3645007014274597,
|
||
|
|
"drift/rejected_abs": 0.18596860766410828,
|
||
|
|
"epoch": 0.4825032843664159,
|
||
|
|
"grad_norm": 354.0,
|
||
|
|
"learning_rate": 2.8910861626005773e-07,
|
||
|
|
"logits/chosen": -2.129087209701538,
|
||
|
|
"logits/rejected": -2.1622016429901123,
|
||
|
|
"logps/chosen": -0.28229135274887085,
|
||
|
|
"logps/rejected": -0.28652527928352356,
|
||
|
|
"loss": 1.6089633703231812,
|
||
|
|
"loss/core": 1.548729658126831,
|
||
|
|
"loss/preference_sft": 0.28229135274887085,
|
||
|
|
"loss/reference_anchor": 0.7748891115188599,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 3.643630266189575,
|
||
|
|
"rewards/margins": 1.8779332637786865,
|
||
|
|
"rewards/rejected": 1.7656971216201782,
|
||
|
|
"step": 505
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.20183968544006348,
|
||
|
|
"drift/rejected_abs": 0.0795665830373764,
|
||
|
|
"epoch": 0.4872805446076675,
|
||
|
|
"grad_norm": 0.28515625,
|
||
|
|
"learning_rate": 2.8371273256843074e-07,
|
||
|
|
"logits/chosen": -2.1265504360198975,
|
||
|
|
"logits/rejected": -2.259223699569702,
|
||
|
|
"logps/chosen": -0.31546053290367126,
|
||
|
|
"logps/rejected": -0.29201287031173706,
|
||
|
|
"loss": 0.4467145502567291,
|
||
|
|
"loss/core": 0.42825937271118164,
|
||
|
|
"loss/preference_sft": 0.31546053290367126,
|
||
|
|
"loss/reference_anchor": 0.21452267467975616,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.015305995941162,
|
||
|
|
"rewards/margins": 1.221923589706421,
|
||
|
|
"rewards/rejected": 0.793382465839386,
|
||
|
|
"step": 510
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.25470539927482605,
|
||
|
|
"drift/rejected_abs": 0.16728895902633667,
|
||
|
|
"epoch": 0.49205780484891914,
|
||
|
|
"grad_norm": 38.75,
|
||
|
|
"learning_rate": 2.783008034419767e-07,
|
||
|
|
"logits/chosen": -2.011662721633911,
|
||
|
|
"logits/rejected": -2.014399528503418,
|
||
|
|
"logps/chosen": -0.2826010584831238,
|
||
|
|
"logps/rejected": -0.306089848279953,
|
||
|
|
"loss": 0.4587891697883606,
|
||
|
|
"loss/core": 0.44014644622802734,
|
||
|
|
"loss/preference_sft": 0.2826010584831238,
|
||
|
|
"loss/reference_anchor": 0.22030992805957794,
|
||
|
|
"rewards/accuracies": 0.824999988079071,
|
||
|
|
"rewards/chosen": 2.545462131500244,
|
||
|
|
"rewards/margins": 0.8881236910820007,
|
||
|
|
"rewards/rejected": 1.6573383808135986,
|
||
|
|
"step": 515
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3071853816509247,
|
||
|
|
"drift/rejected_abs": 0.09579263627529144,
|
||
|
|
"epoch": 0.4968350650901708,
|
||
|
|
"grad_norm": 55.5,
|
||
|
|
"learning_rate": 2.7287540466585064e-07,
|
||
|
|
"logits/chosen": -2.0407145023345947,
|
||
|
|
"logits/rejected": -2.208981990814209,
|
||
|
|
"logps/chosen": -0.32281801104545593,
|
||
|
|
"logps/rejected": -0.3359902501106262,
|
||
|
|
"loss": 0.8434049487113953,
|
||
|
|
"loss/core": 0.8105605244636536,
|
||
|
|
"loss/preference_sft": 0.32281801104545593,
|
||
|
|
"loss/reference_anchor": 0.40564513206481934,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 3.066298007965088,
|
||
|
|
"rewards/margins": 2.121892213821411,
|
||
|
|
"rewards/rejected": 0.9444060325622559,
|
||
|
|
"step": 520
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.41961222887039185,
|
||
|
|
"drift/rejected_abs": 0.15004144608974457,
|
||
|
|
"epoch": 0.5016123253314224,
|
||
|
|
"grad_norm": 1408.0,
|
||
|
|
"learning_rate": 2.674391184360313e-07,
|
||
|
|
"logits/chosen": -2.1737782955169678,
|
||
|
|
"logits/rejected": -2.141087293624878,
|
||
|
|
"logps/chosen": -0.31184646487236023,
|
||
|
|
"logps/rejected": -0.29689449071884155,
|
||
|
|
"loss": 1.9573285579681396,
|
||
|
|
"loss/core": 1.8842939138412476,
|
||
|
|
"loss/preference_sft": 0.31184646487236023,
|
||
|
|
"loss/reference_anchor": 0.9426106214523315,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 4.192916393280029,
|
||
|
|
"rewards/margins": 2.6963582038879395,
|
||
|
|
"rewards/rejected": 1.4965574741363525,
|
||
|
|
"step": 525
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3052569031715393,
|
||
|
|
"drift/rejected_abs": 0.1496833860874176,
|
||
|
|
"epoch": 0.5063895855726741,
|
||
|
|
"grad_norm": 0.41796875,
|
||
|
|
"learning_rate": 2.6199453213033593e-07,
|
||
|
|
"logits/chosen": -2.0180935859680176,
|
||
|
|
"logits/rejected": -1.9813159704208374,
|
||
|
|
"logps/chosen": -0.28599947690963745,
|
||
|
|
"logps/rejected": -0.28876030445098877,
|
||
|
|
"loss": 0.8264201283454895,
|
||
|
|
"loss/core": 0.7944486141204834,
|
||
|
|
"loss/preference_sft": 0.28599947690963745,
|
||
|
|
"loss/reference_anchor": 0.3976878523826599,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.0507431030273438,
|
||
|
|
"rewards/margins": 1.5630565881729126,
|
||
|
|
"rewards/rejected": 1.4876863956451416,
|
||
|
|
"step": 530
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.31068387627601624,
|
||
|
|
"drift/rejected_abs": 0.10734076797962189,
|
||
|
|
"epoch": 0.5111668458139257,
|
||
|
|
"grad_norm": 580.0,
|
||
|
|
"learning_rate": 2.565442370769683e-07,
|
||
|
|
"logits/chosen": -2.1644198894500732,
|
||
|
|
"logits/rejected": -2.085977554321289,
|
||
|
|
"logps/chosen": -0.2844741642475128,
|
||
|
|
"logps/rejected": -0.27978286147117615,
|
||
|
|
"loss": 0.9638272523880005,
|
||
|
|
"loss/core": 0.9268702268600464,
|
||
|
|
"loss/preference_sft": 0.2844741642475128,
|
||
|
|
"loss/reference_anchor": 0.46431416273117065,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 3.104759931564331,
|
||
|
|
"rewards/margins": 2.0328688621520996,
|
||
|
|
"rewards/rejected": 1.0718910694122314,
|
||
|
|
"step": 535
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.15837408602237701,
|
||
|
|
"drift/rejected_abs": 0.15098315477371216,
|
||
|
|
"epoch": 0.5159441060551774,
|
||
|
|
"grad_norm": 1.5,
|
||
|
|
"learning_rate": 2.510908273211866e-07,
|
||
|
|
"logits/chosen": -2.086716413497925,
|
||
|
|
"logits/rejected": -2.065261125564575,
|
||
|
|
"logps/chosen": -0.27920594811439514,
|
||
|
|
"logps/rejected": -0.2877506613731384,
|
||
|
|
"loss": 0.33345627784729004,
|
||
|
|
"loss/core": 0.31936904788017273,
|
||
|
|
"loss/preference_sft": 0.27920594811439514,
|
||
|
|
"loss/reference_anchor": 0.15990903973579407,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.5837407112121582,
|
||
|
|
"rewards/margins": 0.07735572755336761,
|
||
|
|
"rewards/rejected": 1.5063849687576294,
|
||
|
|
"step": 540
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2585676312446594,
|
||
|
|
"drift/rejected_abs": 0.08764716237783432,
|
||
|
|
"epoch": 0.520721366296429,
|
||
|
|
"grad_norm": 1264.0,
|
||
|
|
"learning_rate": 2.456368983906791e-07,
|
||
|
|
"logits/chosen": -2.305999994277954,
|
||
|
|
"logits/rejected": -2.1507370471954346,
|
||
|
|
"logps/chosen": -0.28886252641677856,
|
||
|
|
"logps/rejected": -0.2935750484466553,
|
||
|
|
"loss": 0.9189807772636414,
|
||
|
|
"loss/core": 0.8836311101913452,
|
||
|
|
"loss/preference_sft": 0.28886252641677856,
|
||
|
|
"loss/reference_anchor": 0.44244223833084106,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.5849061012268066,
|
||
|
|
"rewards/margins": 1.7120176553726196,
|
||
|
|
"rewards/rejected": 0.8728883862495422,
|
||
|
|
"step": 545
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.27887991070747375,
|
||
|
|
"drift/rejected_abs": 0.1610538214445114,
|
||
|
|
"epoch": 0.5254986265376806,
|
||
|
|
"grad_norm": 12.75,
|
||
|
|
"learning_rate": 2.401850460602329e-07,
|
||
|
|
"logits/chosen": -1.8745006322860718,
|
||
|
|
"logits/rejected": -1.8935530185699463,
|
||
|
|
"logps/chosen": -0.2741144299507141,
|
||
|
|
"logps/rejected": -0.2991863191127777,
|
||
|
|
"loss": 0.8370600938796997,
|
||
|
|
"loss/core": 0.8047995567321777,
|
||
|
|
"loss/preference_sft": 0.2741144299507141,
|
||
|
|
"loss/reference_anchor": 0.40272873640060425,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.7885258197784424,
|
||
|
|
"rewards/margins": 1.1791884899139404,
|
||
|
|
"rewards/rejected": 1.6093372106552124,
|
||
|
|
"step": 550
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3264530301094055,
|
||
|
|
"drift/rejected_abs": 0.09586937725543976,
|
||
|
|
"epoch": 0.5302758867789323,
|
||
|
|
"grad_norm": 5.8125,
|
||
|
|
"learning_rate": 2.3473786511628575e-07,
|
||
|
|
"logits/chosen": -2.0708940029144287,
|
||
|
|
"logits/rejected": -2.1273093223571777,
|
||
|
|
"logps/chosen": -0.3148099184036255,
|
||
|
|
"logps/rejected": -0.2786844074726105,
|
||
|
|
"loss": 0.81494140625,
|
||
|
|
"loss/core": 0.7831686735153198,
|
||
|
|
"loss/preference_sft": 0.3148099184036255,
|
||
|
|
"loss/reference_anchor": 0.39215487241744995,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 3.2636561393737793,
|
||
|
|
"rewards/margins": 2.305521011352539,
|
||
|
|
"rewards/rejected": 0.9581351280212402,
|
||
|
|
"step": 555
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.28125500679016113,
|
||
|
|
"drift/rejected_abs": 0.15266069769859314,
|
||
|
|
"epoch": 0.5350531470201839,
|
||
|
|
"grad_norm": 50.5,
|
||
|
|
"learning_rate": 2.2929794812194894e-07,
|
||
|
|
"logits/chosen": -2.1583635807037354,
|
||
|
|
"logits/rejected": -2.1458077430725098,
|
||
|
|
"logps/chosen": -0.27610182762145996,
|
||
|
|
"logps/rejected": -0.28124934434890747,
|
||
|
|
"loss": 0.7136462926864624,
|
||
|
|
"loss/core": 0.6858313083648682,
|
||
|
|
"loss/preference_sft": 0.27610182762145996,
|
||
|
|
"loss/reference_anchor": 0.3432568311691284,
|
||
|
|
"rewards/accuracies": 0.949999988079071,
|
||
|
|
"rewards/chosen": 2.8115084171295166,
|
||
|
|
"rewards/margins": 1.2856230735778809,
|
||
|
|
"rewards/rejected": 1.5258853435516357,
|
||
|
|
"step": 560
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.27818697690963745,
|
||
|
|
"drift/rejected_abs": 0.16565275192260742,
|
||
|
|
"epoch": 0.5398304072614356,
|
||
|
|
"grad_norm": 19.5,
|
||
|
|
"learning_rate": 2.2386788418308665e-07,
|
||
|
|
"logits/chosen": -2.056506395339966,
|
||
|
|
"logits/rejected": -2.082536220550537,
|
||
|
|
"logps/chosen": -0.28751203417778015,
|
||
|
|
"logps/rejected": -0.3056444227695465,
|
||
|
|
"loss": 1.35868239402771,
|
||
|
|
"loss/core": 1.3074853420257568,
|
||
|
|
"loss/preference_sft": 0.28751203417778015,
|
||
|
|
"loss/reference_anchor": 0.6538757085800171,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.7808547019958496,
|
||
|
|
"rewards/margins": 1.124733805656433,
|
||
|
|
"rewards/rejected": 1.6561208963394165,
|
||
|
|
"step": 565
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3337910771369934,
|
||
|
|
"drift/rejected_abs": 0.16797366738319397,
|
||
|
|
"epoch": 0.5446076675026872,
|
||
|
|
"grad_norm": 71.5,
|
||
|
|
"learning_rate": 2.184502577160426e-07,
|
||
|
|
"logits/chosen": -2.0548603534698486,
|
||
|
|
"logits/rejected": -2.1045546531677246,
|
||
|
|
"logps/chosen": -0.2894991338253021,
|
||
|
|
"logps/rejected": -0.2895090579986572,
|
||
|
|
"loss": 0.9317213892936707,
|
||
|
|
"loss/core": 0.8959233164787292,
|
||
|
|
"loss/preference_sft": 0.2894991338253021,
|
||
|
|
"loss/reference_anchor": 0.4483569264411926,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 3.335692882537842,
|
||
|
|
"rewards/margins": 1.657538652420044,
|
||
|
|
"rewards/rejected": 1.6781543493270874,
|
||
|
|
"step": 570
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.18387334048748016,
|
||
|
|
"drift/rejected_abs": 0.06442725658416748,
|
||
|
|
"epoch": 0.5493849277439389,
|
||
|
|
"grad_norm": 8.9375,
|
||
|
|
"learning_rate": 2.1304764721759732e-07,
|
||
|
|
"logits/chosen": -2.1352288722991943,
|
||
|
|
"logits/rejected": -2.1651110649108887,
|
||
|
|
"logps/chosen": -0.30129939317703247,
|
||
|
|
"logps/rejected": -0.32308152318000793,
|
||
|
|
"loss": 0.345123291015625,
|
||
|
|
"loss/core": 0.3304422199726105,
|
||
|
|
"loss/preference_sft": 0.30129939317703247,
|
||
|
|
"loss/reference_anchor": 0.16561798751354218,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.8363609313964844,
|
||
|
|
"rewards/margins": 1.274545669555664,
|
||
|
|
"rewards/rejected": 0.5618152022361755,
|
||
|
|
"step": 575
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.14127376675605774,
|
||
|
|
"drift/rejected_abs": 0.0939093828201294,
|
||
|
|
"epoch": 0.5541621879851905,
|
||
|
|
"grad_norm": 0.49609375,
|
||
|
|
"learning_rate": 2.0766262403774385e-07,
|
||
|
|
"logits/chosen": -2.4066150188446045,
|
||
|
|
"logits/rejected": -2.462902545928955,
|
||
|
|
"logps/chosen": -0.26729652285575867,
|
||
|
|
"logps/rejected": -0.2831682562828064,
|
||
|
|
"loss": 0.17010077834129333,
|
||
|
|
"loss/core": 0.1620091199874878,
|
||
|
|
"loss/preference_sft": 0.26729652285575867,
|
||
|
|
"loss/reference_anchor": 0.08115912973880768,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.4122838973999023,
|
||
|
|
"rewards/margins": 0.4755134582519531,
|
||
|
|
"rewards/rejected": 0.9367703199386597,
|
||
|
|
"step": 580
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.22129257023334503,
|
||
|
|
"drift/rejected_abs": 0.13631665706634521,
|
||
|
|
"epoch": 0.5589394482264421,
|
||
|
|
"grad_norm": 35.25,
|
||
|
|
"learning_rate": 2.0229775115586378e-07,
|
||
|
|
"logits/chosen": -2.1155853271484375,
|
||
|
|
"logits/rejected": -2.0802512168884277,
|
||
|
|
"logps/chosen": -0.2712664008140564,
|
||
|
|
"logps/rejected": -0.28381767868995667,
|
||
|
|
"loss": 1.4533600807189941,
|
||
|
|
"loss/core": 1.3988597393035889,
|
||
|
|
"loss/preference_sft": 0.2712664008140564,
|
||
|
|
"loss/reference_anchor": 0.6995447874069214,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.212214469909668,
|
||
|
|
"rewards/margins": 0.8542079925537109,
|
||
|
|
"rewards/rejected": 1.358006477355957,
|
||
|
|
"step": 585
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3607145845890045,
|
||
|
|
"drift/rejected_abs": 0.10202290117740631,
|
||
|
|
"epoch": 0.5637167084676937,
|
||
|
|
"grad_norm": 4.625,
|
||
|
|
"learning_rate": 1.9695558196088844e-07,
|
||
|
|
"logits/chosen": -2.002814292907715,
|
||
|
|
"logits/rejected": -2.099381923675537,
|
||
|
|
"logps/chosen": -0.28898441791534424,
|
||
|
|
"logps/rejected": -0.28851771354675293,
|
||
|
|
"loss": 0.9255863428115845,
|
||
|
|
"loss/core": 0.8899755477905273,
|
||
|
|
"loss/preference_sft": 0.28898441791534424,
|
||
|
|
"loss/reference_anchor": 0.44591063261032104,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 3.606508731842041,
|
||
|
|
"rewards/margins": 2.5896670818328857,
|
||
|
|
"rewards/rejected": 1.0168412923812866,
|
||
|
|
"step": 590
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1707986295223236,
|
||
|
|
"drift/rejected_abs": 0.08907818049192429,
|
||
|
|
"epoch": 0.5684939687089454,
|
||
|
|
"grad_norm": 4.28125,
|
||
|
|
"learning_rate": 1.9163865903602372e-07,
|
||
|
|
"logits/chosen": -2.0221307277679443,
|
||
|
|
"logits/rejected": -2.0168774127960205,
|
||
|
|
"logps/chosen": -0.2755976915359497,
|
||
|
|
"logps/rejected": -0.27291139960289,
|
||
|
|
"loss": 0.11963596194982529,
|
||
|
|
"loss/core": 0.1133008748292923,
|
||
|
|
"loss/preference_sft": 0.2755976915359497,
|
||
|
|
"loss/reference_anchor": 0.05690792202949524,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.7069168090820312,
|
||
|
|
"rewards/margins": 0.8279293775558472,
|
||
|
|
"rewards/rejected": 0.8789873123168945,
|
||
|
|
"step": 595
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.18752028048038483,
|
||
|
|
"drift/rejected_abs": 0.10666258633136749,
|
||
|
|
"epoch": 0.5732712289501971,
|
||
|
|
"grad_norm": 24.875,
|
||
|
|
"learning_rate": 1.8634951294861806e-07,
|
||
|
|
"logits/chosen": -2.253175735473633,
|
||
|
|
"logits/rejected": -2.2427711486816406,
|
||
|
|
"logps/chosen": -0.2663193345069885,
|
||
|
|
"logps/rejected": -0.28532105684280396,
|
||
|
|
"loss": 0.31239908933639526,
|
||
|
|
"loss/core": 0.2991681396961212,
|
||
|
|
"loss/preference_sft": 0.2663193345069885,
|
||
|
|
"loss/reference_anchor": 0.14978043735027313,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.8731563091278076,
|
||
|
|
"rewards/margins": 0.82030189037323,
|
||
|
|
"rewards/rejected": 1.052854299545288,
|
||
|
|
"step": 600
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.41676053404808044,
|
||
|
|
"drift/rejected_abs": 0.11940282583236694,
|
||
|
|
"epoch": 0.5780484891914487,
|
||
|
|
"grad_norm": 133.0,
|
||
|
|
"learning_rate": 1.810906610457502e-07,
|
||
|
|
"logits/chosen": -2.0973143577575684,
|
||
|
|
"logits/rejected": -2.171078681945801,
|
||
|
|
"logps/chosen": -0.2819843590259552,
|
||
|
|
"logps/rejected": -0.27706506848335266,
|
||
|
|
"loss": 1.816592812538147,
|
||
|
|
"loss/core": 1.7488460540771484,
|
||
|
|
"loss/preference_sft": 0.2819843590259552,
|
||
|
|
"loss/reference_anchor": 0.875086784362793,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 4.165308952331543,
|
||
|
|
"rewards/margins": 2.9730935096740723,
|
||
|
|
"rewards/rejected": 1.1922153234481812,
|
||
|
|
"step": 605
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.4254422187805176,
|
||
|
|
"drift/rejected_abs": 0.19570092856884003,
|
||
|
|
"epoch": 0.5828257494327004,
|
||
|
|
"grad_norm": 106.0,
|
||
|
|
"learning_rate": 1.7586460625610726e-07,
|
||
|
|
"logits/chosen": -2.0943334102630615,
|
||
|
|
"logits/rejected": -2.1102890968322754,
|
||
|
|
"logps/chosen": -0.2782813310623169,
|
||
|
|
"logps/rejected": -0.27373403310775757,
|
||
|
|
"loss": 1.9183744192123413,
|
||
|
|
"loss/core": 1.8469626903533936,
|
||
|
|
"loss/preference_sft": 0.2782813310623169,
|
||
|
|
"loss/reference_anchor": 0.9243267178535461,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 4.251742362976074,
|
||
|
|
"rewards/margins": 2.3000569343566895,
|
||
|
|
"rewards/rejected": 1.9516862630844116,
|
||
|
|
"step": 610
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.27416449785232544,
|
||
|
|
"drift/rejected_abs": 0.14728380739688873,
|
||
|
|
"epoch": 0.587603009673952,
|
||
|
|
"grad_norm": 13.5,
|
||
|
|
"learning_rate": 1.70673835898727e-07,
|
||
|
|
"logits/chosen": -1.9468109607696533,
|
||
|
|
"logits/rejected": -2.0125274658203125,
|
||
|
|
"logps/chosen": -0.3140425682067871,
|
||
|
|
"logps/rejected": -0.30548495054244995,
|
||
|
|
"loss": 0.6002452373504639,
|
||
|
|
"loss/core": 0.5762513279914856,
|
||
|
|
"loss/preference_sft": 0.3140425682067871,
|
||
|
|
"loss/reference_anchor": 0.2885145843029022,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.7396042346954346,
|
||
|
|
"rewards/margins": 1.2685816287994385,
|
||
|
|
"rewards/rejected": 1.471022367477417,
|
||
|
|
"step": 615
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.16883981227874756,
|
||
|
|
"drift/rejected_abs": 0.14558133482933044,
|
||
|
|
"epoch": 0.5923802699152036,
|
||
|
|
"grad_norm": 7.15625,
|
||
|
|
"learning_rate": 1.6552082049916824e-07,
|
||
|
|
"logits/chosen": -2.119967460632324,
|
||
|
|
"logits/rejected": -2.1456103324890137,
|
||
|
|
"logps/chosen": -0.2863292992115021,
|
||
|
|
"logps/rejected": -0.29261788725852966,
|
||
|
|
"loss": 0.40366506576538086,
|
||
|
|
"loss/core": 0.38699546456336975,
|
||
|
|
"loss/preference_sft": 0.2863292992115021,
|
||
|
|
"loss/reference_anchor": 0.1936284750699997,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.6874778270721436,
|
||
|
|
"rewards/margins": 0.236982062458992,
|
||
|
|
"rewards/rejected": 1.450495719909668,
|
||
|
|
"step": 620
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.388235867023468,
|
||
|
|
"drift/rejected_abs": 0.17607903480529785,
|
||
|
|
"epoch": 0.5971575301564552,
|
||
|
|
"grad_norm": 1256.0,
|
||
|
|
"learning_rate": 1.6040801261367493e-07,
|
||
|
|
"logits/chosen": -1.9117774963378906,
|
||
|
|
"logits/rejected": -1.9400612115859985,
|
||
|
|
"logps/chosen": -0.3054983913898468,
|
||
|
|
"logps/rejected": -0.3448588252067566,
|
||
|
|
"loss": 2.1722350120544434,
|
||
|
|
"loss/core": 2.0914366245269775,
|
||
|
|
"loss/preference_sft": 0.3054983913898468,
|
||
|
|
"loss/reference_anchor": 1.0467677116394043,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 3.8796839714050293,
|
||
|
|
"rewards/margins": 2.4553399085998535,
|
||
|
|
"rewards/rejected": 1.4243443012237549,
|
||
|
|
"step": 625
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.29388323426246643,
|
||
|
|
"drift/rejected_abs": 0.12398219108581543,
|
||
|
|
"epoch": 0.6019347903977069,
|
||
|
|
"grad_norm": 204.0,
|
||
|
|
"learning_rate": 1.5533784566189175e-07,
|
||
|
|
"logits/chosen": -1.9094196557998657,
|
||
|
|
"logits/rejected": -2.0244593620300293,
|
||
|
|
"logps/chosen": -0.27296558022499084,
|
||
|
|
"logps/rejected": -0.2840172350406647,
|
||
|
|
"loss": 0.9247919321060181,
|
||
|
|
"loss/core": 0.8893438577651978,
|
||
|
|
"loss/preference_sft": 0.27296558022499084,
|
||
|
|
"loss/reference_anchor": 0.4453449249267578,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.938032388687134,
|
||
|
|
"rewards/margins": 1.7004245519638062,
|
||
|
|
"rewards/rejected": 1.237607479095459,
|
||
|
|
"step": 630
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.23004408180713654,
|
||
|
|
"drift/rejected_abs": 0.10162544250488281,
|
||
|
|
"epoch": 0.6067120506389586,
|
||
|
|
"grad_norm": 300.0,
|
||
|
|
"learning_rate": 1.5031273276868845e-07,
|
||
|
|
"logits/chosen": -1.9615974426269531,
|
||
|
|
"logits/rejected": -2.0472800731658936,
|
||
|
|
"logps/chosen": -0.3058452904224396,
|
||
|
|
"logps/rejected": -0.3000328540802002,
|
||
|
|
"loss": 0.45604729652404785,
|
||
|
|
"loss/core": 0.43732786178588867,
|
||
|
|
"loss/preference_sft": 0.3058452904224396,
|
||
|
|
"loss/reference_anchor": 0.21900811791419983,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.2964746952056885,
|
||
|
|
"rewards/margins": 1.2814719676971436,
|
||
|
|
"rewards/rejected": 1.015002727508545,
|
||
|
|
"step": 635
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.19580507278442383,
|
||
|
|
"drift/rejected_abs": 0.07966174185276031,
|
||
|
|
"epoch": 0.6114893108802102,
|
||
|
|
"grad_norm": 2.625,
|
||
|
|
"learning_rate": 1.4533506561564305e-07,
|
||
|
|
"logits/chosen": -2.188485622406006,
|
||
|
|
"logits/rejected": -2.2632412910461426,
|
||
|
|
"logps/chosen": -0.30155614018440247,
|
||
|
|
"logps/rejected": -0.2989227771759033,
|
||
|
|
"loss": 0.4078275263309479,
|
||
|
|
"loss/core": 0.39089325070381165,
|
||
|
|
"loss/preference_sft": 0.30155614018440247,
|
||
|
|
"loss/reference_anchor": 0.19563566148281097,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 1.9569017887115479,
|
||
|
|
"rewards/margins": 1.161104440689087,
|
||
|
|
"rewards/rejected": 0.7957972288131714,
|
||
|
|
"step": 640
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1673978865146637,
|
||
|
|
"drift/rejected_abs": 0.11365523189306259,
|
||
|
|
"epoch": 0.6162665711214619,
|
||
|
|
"grad_norm": 15.8125,
|
||
|
|
"learning_rate": 1.404072133027306e-07,
|
||
|
|
"logits/chosen": -2.133847713470459,
|
||
|
|
"logits/rejected": -2.1120846271514893,
|
||
|
|
"logps/chosen": -0.2908037602901459,
|
||
|
|
"logps/rejected": -0.30515560507774353,
|
||
|
|
"loss": 0.2214515656232834,
|
||
|
|
"loss/core": 0.21132436394691467,
|
||
|
|
"loss/preference_sft": 0.2908037602901459,
|
||
|
|
"loss/reference_anchor": 0.10594894737005234,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.6722071170806885,
|
||
|
|
"rewards/margins": 0.5402682423591614,
|
||
|
|
"rewards/rejected": 1.1319388151168823,
|
||
|
|
"step": 645
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.13293756544589996,
|
||
|
|
"drift/rejected_abs": 0.08515778928995132,
|
||
|
|
"epoch": 0.6210438313627135,
|
||
|
|
"grad_norm": 4.21875,
|
||
|
|
"learning_rate": 1.3553152122076078e-07,
|
||
|
|
"logits/chosen": -2.2003543376922607,
|
||
|
|
"logits/rejected": -2.1520040035247803,
|
||
|
|
"logps/chosen": -0.2900927662849426,
|
||
|
|
"logps/rejected": -0.30046719312667847,
|
||
|
|
"loss": 0.15672235190868378,
|
||
|
|
"loss/core": 0.14894674718379974,
|
||
|
|
"loss/preference_sft": 0.2900927662849426,
|
||
|
|
"loss/reference_anchor": 0.07466545701026917,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 1.327453851699829,
|
||
|
|
"rewards/margins": 0.5011469125747681,
|
||
|
|
"rewards/rejected": 0.8263068199157715,
|
||
|
|
"step": 650
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.20016181468963623,
|
||
|
|
"drift/rejected_abs": 0.09695510566234589,
|
||
|
|
"epoch": 0.6258210916039652,
|
||
|
|
"grad_norm": 56.0,
|
||
|
|
"learning_rate": 1.3071030993509787e-07,
|
||
|
|
"logits/chosen": -2.0374953746795654,
|
||
|
|
"logits/rejected": -2.099519729614258,
|
||
|
|
"logps/chosen": -0.2827564477920532,
|
||
|
|
"logps/rejected": -0.29163193702697754,
|
||
|
|
"loss": 0.2658204436302185,
|
||
|
|
"loss/core": 0.2541511654853821,
|
||
|
|
"loss/preference_sft": 0.2827564477920532,
|
||
|
|
"loss/reference_anchor": 0.12731416523456573,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.9995691776275635,
|
||
|
|
"rewards/margins": 1.030291199684143,
|
||
|
|
"rewards/rejected": 0.9692778587341309,
|
||
|
|
"step": 655
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.27798160910606384,
|
||
|
|
"drift/rejected_abs": 0.13286152482032776,
|
||
|
|
"epoch": 0.6305983518452167,
|
||
|
|
"grad_norm": 2.375,
|
||
|
|
"learning_rate": 1.2594587408119804e-07,
|
||
|
|
"logits/chosen": -2.0071966648101807,
|
||
|
|
"logits/rejected": -2.002094268798828,
|
||
|
|
"logps/chosen": -0.293853223323822,
|
||
|
|
"logps/rejected": -0.3049863874912262,
|
||
|
|
"loss": 1.3727947473526,
|
||
|
|
"loss/core": 1.321033000946045,
|
||
|
|
"loss/preference_sft": 0.293853223323822,
|
||
|
|
"loss/reference_anchor": 0.6607749462127686,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.7776036262512207,
|
||
|
|
"rewards/margins": 1.4595942497253418,
|
||
|
|
"rewards/rejected": 1.3180092573165894,
|
||
|
|
"step": 660
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.23454730212688446,
|
||
|
|
"drift/rejected_abs": 0.14776912331581116,
|
||
|
|
"epoch": 0.6353756120864684,
|
||
|
|
"grad_norm": 141.0,
|
||
|
|
"learning_rate": 1.2124048127248644e-07,
|
||
|
|
"logits/chosen": -2.1691813468933105,
|
||
|
|
"logits/rejected": -2.143655776977539,
|
||
|
|
"logps/chosen": -0.28466880321502686,
|
||
|
|
"logps/rejected": -0.2818228602409363,
|
||
|
|
"loss": 0.5208796858787537,
|
||
|
|
"loss/core": 0.4999764859676361,
|
||
|
|
"loss/preference_sft": 0.28466880321502686,
|
||
|
|
"loss/reference_anchor": 0.2502433955669403,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.3442444801330566,
|
||
|
|
"rewards/margins": 0.8682305216789246,
|
||
|
|
"rewards/rejected": 1.4760141372680664,
|
||
|
|
"step": 665
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.364292711019516,
|
||
|
|
"drift/rejected_abs": 0.1280832588672638,
|
||
|
|
"epoch": 0.64015287232772,
|
||
|
|
"grad_norm": 1.484375,
|
||
|
|
"learning_rate": 1.1659637102109712e-07,
|
||
|
|
"logits/chosen": -1.9568021297454834,
|
||
|
|
"logits/rejected": -2.003387689590454,
|
||
|
|
"logps/chosen": -0.2745230197906494,
|
||
|
|
"logps/rejected": -0.31976574659347534,
|
||
|
|
"loss": 1.0210597515106201,
|
||
|
|
"loss/core": 0.982128918170929,
|
||
|
|
"loss/preference_sft": 0.2745230197906494,
|
||
|
|
"loss/reference_anchor": 0.49162501096725464,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 3.6429271697998047,
|
||
|
|
"rewards/margins": 2.3816723823547363,
|
||
|
|
"rewards/rejected": 1.2612544298171997,
|
||
|
|
"step": 670
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1619919240474701,
|
||
|
|
"drift/rejected_abs": 0.08836764097213745,
|
||
|
|
"epoch": 0.6449301325689717,
|
||
|
|
"grad_norm": 14.5625,
|
||
|
|
"learning_rate": 1.1201575367198546e-07,
|
||
|
|
"logits/chosen": -2.012540340423584,
|
||
|
|
"logits/rejected": -2.0545716285705566,
|
||
|
|
"logps/chosen": -0.2888466715812683,
|
||
|
|
"logps/rejected": -0.28877875208854675,
|
||
|
|
"loss": 0.13238023221492767,
|
||
|
|
"loss/core": 0.12548786401748657,
|
||
|
|
"loss/preference_sft": 0.2888466715812683,
|
||
|
|
"loss/reference_anchor": 0.06301363557577133,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.6180137395858765,
|
||
|
|
"rewards/margins": 0.7427892088890076,
|
||
|
|
"rewards/rejected": 0.8752244114875793,
|
||
|
|
"step": 675
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.21459881961345673,
|
||
|
|
"drift/rejected_abs": 0.1355762928724289,
|
||
|
|
"epoch": 0.6497073928102234,
|
||
|
|
"grad_norm": 1.7734375,
|
||
|
|
"learning_rate": 1.0750080935092423e-07,
|
||
|
|
"logits/chosen": -2.1347737312316895,
|
||
|
|
"logits/rejected": -2.0533447265625,
|
||
|
|
"logps/chosen": -0.2823336720466614,
|
||
|
|
"logps/rejected": -0.2946282923221588,
|
||
|
|
"loss": 0.43657293915748596,
|
||
|
|
"loss/core": 0.41873565316200256,
|
||
|
|
"loss/preference_sft": 0.2823336720466614,
|
||
|
|
"loss/reference_anchor": 0.20959632098674774,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.1431565284729004,
|
||
|
|
"rewards/margins": 0.7881289720535278,
|
||
|
|
"rewards/rejected": 1.3550279140472412,
|
||
|
|
"step": 680
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2882576882839203,
|
||
|
|
"drift/rejected_abs": 0.15576216578483582,
|
||
|
|
"epoch": 0.654484653051475,
|
||
|
|
"grad_norm": 4.375,
|
||
|
|
"learning_rate": 1.0305368692688174e-07,
|
||
|
|
"logits/chosen": -2.0634636878967285,
|
||
|
|
"logits/rejected": -2.049027919769287,
|
||
|
|
"logps/chosen": -0.25410422682762146,
|
||
|
|
"logps/rejected": -0.2750704884529114,
|
||
|
|
"loss": 1.0024207830429077,
|
||
|
|
"loss/core": 0.9643166661262512,
|
||
|
|
"loss/preference_sft": 0.25410422682762146,
|
||
|
|
"loss/reference_anchor": 0.4826439917087555,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.8811697959899902,
|
||
|
|
"rewards/margins": 1.326778769493103,
|
||
|
|
"rewards/rejected": 1.5543910264968872,
|
||
|
|
"step": 685
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.17661243677139282,
|
||
|
|
"drift/rejected_abs": 0.1041385754942894,
|
||
|
|
"epoch": 0.6592619132927267,
|
||
|
|
"grad_norm": 3.359375,
|
||
|
|
"learning_rate": 9.867650298927643e-08,
|
||
|
|
"logits/chosen": -1.8653026819229126,
|
||
|
|
"logits/rejected": -1.8865511417388916,
|
||
|
|
"logps/chosen": -0.3086770176887512,
|
||
|
|
"logps/rejected": -0.3341335952281952,
|
||
|
|
"loss": 0.19030842185020447,
|
||
|
|
"loss/core": 0.18117818236351013,
|
||
|
|
"loss/preference_sft": 0.3086770176887512,
|
||
|
|
"loss/reference_anchor": 0.0908689871430397,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.7632665634155273,
|
||
|
|
"rewards/margins": 0.7324223518371582,
|
||
|
|
"rewards/rejected": 1.0308443307876587,
|
||
|
|
"step": 690
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.19695012271404266,
|
||
|
|
"drift/rejected_abs": 0.06876302510499954,
|
||
|
|
"epoch": 0.6640391735339782,
|
||
|
|
"grad_norm": 10.125,
|
||
|
|
"learning_rate": 9.437134084059515e-08,
|
||
|
|
"logits/chosen": -2.2494120597839355,
|
||
|
|
"logits/rejected": -2.259532928466797,
|
||
|
|
"logps/chosen": -0.3065325617790222,
|
||
|
|
"logps/rejected": -0.29353365302085876,
|
||
|
|
"loss": 0.39579930901527405,
|
||
|
|
"loss/core": 0.3792479932308197,
|
||
|
|
"loss/preference_sft": 0.3065325617790222,
|
||
|
|
"loss/reference_anchor": 0.19003014266490936,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.9680402278900146,
|
||
|
|
"rewards/margins": 1.29190993309021,
|
||
|
|
"rewards/rejected": 0.6761302947998047,
|
||
|
|
"step": 695
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2228858470916748,
|
||
|
|
"drift/rejected_abs": 0.15424631536006927,
|
||
|
|
"epoch": 0.6688164337752299,
|
||
|
|
"grad_norm": 470.0,
|
||
|
|
"learning_rate": 9.014024950485383e-08,
|
||
|
|
"logits/chosen": -2.0826706886291504,
|
||
|
|
"logits/rejected": -2.187513828277588,
|
||
|
|
"logps/chosen": -0.2971203923225403,
|
||
|
|
"logps/rejected": -0.3260975480079651,
|
||
|
|
"loss": 0.5006673336029053,
|
||
|
|
"loss/core": 0.48041480779647827,
|
||
|
|
"loss/preference_sft": 0.2971203923225403,
|
||
|
|
"loss/reference_anchor": 0.24032163619995117,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.227379083633423,
|
||
|
|
"rewards/margins": 0.7068957090377808,
|
||
|
|
"rewards/rejected": 1.5204832553863525,
|
||
|
|
"step": 700
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.33960968255996704,
|
||
|
|
"drift/rejected_abs": 0.15266656875610352,
|
||
|
|
"epoch": 0.6735936940164815,
|
||
|
|
"grad_norm": 95.0,
|
||
|
|
"learning_rate": 8.598524275237321e-08,
|
||
|
|
"logits/chosen": -2.175156593322754,
|
||
|
|
"logits/rejected": -2.1317992210388184,
|
||
|
|
"logps/chosen": -0.28126639127731323,
|
||
|
|
"logps/rejected": -0.2680434584617615,
|
||
|
|
"loss": 0.8337176442146301,
|
||
|
|
"loss/core": 0.8014980554580688,
|
||
|
|
"loss/preference_sft": 0.28126639127731323,
|
||
|
|
"loss/reference_anchor": 0.4014674723148346,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 3.3955001831054688,
|
||
|
|
"rewards/margins": 1.8699357509613037,
|
||
|
|
"rewards/rejected": 1.525564432144165,
|
||
|
|
"step": 705
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3576631546020508,
|
||
|
|
"drift/rejected_abs": 0.18644385039806366,
|
||
|
|
"epoch": 0.6783709542577332,
|
||
|
|
"grad_norm": 440.0,
|
||
|
|
"learning_rate": 8.190829814133293e-08,
|
||
|
|
"logits/chosen": -1.863407850265503,
|
||
|
|
"logits/rejected": -1.8509485721588135,
|
||
|
|
"logps/chosen": -0.27954185009002686,
|
||
|
|
"logps/rejected": -0.3012255132198334,
|
||
|
|
"loss": 1.3718466758728027,
|
||
|
|
"loss/core": 1.320197582244873,
|
||
|
|
"loss/preference_sft": 0.27954185009002686,
|
||
|
|
"loss/reference_anchor": 0.6606996655464172,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 3.5734474658966064,
|
||
|
|
"rewards/margins": 1.7104486227035522,
|
||
|
|
"rewards/rejected": 1.8629987239837646,
|
||
|
|
"step": 710
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.18702775239944458,
|
||
|
|
"drift/rejected_abs": 0.08271317183971405,
|
||
|
|
"epoch": 0.6831482144989849,
|
||
|
|
"grad_norm": 3.921875,
|
||
|
|
"learning_rate": 7.791135607656146e-08,
|
||
|
|
"logits/chosen": -2.228705644607544,
|
||
|
|
"logits/rejected": -2.209040403366089,
|
||
|
|
"logps/chosen": -0.2836971879005432,
|
||
|
|
"logps/rejected": -0.3425886332988739,
|
||
|
|
"loss": 0.5329797267913818,
|
||
|
|
"loss/core": 0.5116499066352844,
|
||
|
|
"loss/preference_sft": 0.2836971879005432,
|
||
|
|
"loss/reference_anchor": 0.25602635741233826,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.8697589635849,
|
||
|
|
"rewards/margins": 1.2295645475387573,
|
||
|
|
"rewards/rejected": 0.6401944160461426,
|
||
|
|
"step": 715
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.20850583910942078,
|
||
|
|
"drift/rejected_abs": 0.1485956758260727,
|
||
|
|
"epoch": 0.6879254747402365,
|
||
|
|
"grad_norm": 24.875,
|
||
|
|
"learning_rate": 7.399631888600796e-08,
|
||
|
|
"logits/chosen": -1.9758691787719727,
|
||
|
|
"logits/rejected": -1.9829914569854736,
|
||
|
|
"logps/chosen": -0.29822248220443726,
|
||
|
|
"logps/rejected": -0.30190637707710266,
|
||
|
|
"loss": 0.5085002183914185,
|
||
|
|
"loss/core": 0.487942636013031,
|
||
|
|
"loss/preference_sft": 0.29822248220443726,
|
||
|
|
"loss/reference_anchor": 0.24427810311317444,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 2.083251953125,
|
||
|
|
"rewards/margins": 0.5999056100845337,
|
||
|
|
"rewards/rejected": 1.4833463430404663,
|
||
|
|
"step": 720
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.15803976356983185,
|
||
|
|
"drift/rejected_abs": 0.07249420881271362,
|
||
|
|
"epoch": 0.6927027349814882,
|
||
|
|
"grad_norm": 4.03125,
|
||
|
|
"learning_rate": 7.016504991533726e-08,
|
||
|
|
"logits/chosen": -2.101017475128174,
|
||
|
|
"logits/rejected": -2.215353488922119,
|
||
|
|
"logps/chosen": -0.28623729944229126,
|
||
|
|
"logps/rejected": -0.2781991958618164,
|
||
|
|
"loss": 0.18330907821655273,
|
||
|
|
"loss/core": 0.17459434270858765,
|
||
|
|
"loss/preference_sft": 0.28623729944229126,
|
||
|
|
"loss/reference_anchor": 0.08757317811250687,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.5793583393096924,
|
||
|
|
"rewards/margins": 0.8571645021438599,
|
||
|
|
"rewards/rejected": 0.7221937775611877,
|
||
|
|
"step": 725
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.10135958343744278,
|
||
|
|
"drift/rejected_abs": 0.08568556606769562,
|
||
|
|
"epoch": 0.6974799952227397,
|
||
|
|
"grad_norm": 422.0,
|
||
|
|
"learning_rate": 6.641937264107867e-08,
|
||
|
|
"logits/chosen": -2.1984219551086426,
|
||
|
|
"logits/rejected": -2.248124599456787,
|
||
|
|
"logps/chosen": -0.287166953086853,
|
||
|
|
"logps/rejected": -0.2960854470729828,
|
||
|
|
"loss": 0.1953887790441513,
|
||
|
|
"loss/core": 0.18624430894851685,
|
||
|
|
"loss/preference_sft": 0.287166953086853,
|
||
|
|
"loss/reference_anchor": 0.09321005642414093,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.012143850326538,
|
||
|
|
"rewards/margins": 0.15792995691299438,
|
||
|
|
"rewards/rejected": 0.8542140126228333,
|
||
|
|
"step": 730
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.14169880747795105,
|
||
|
|
"drift/rejected_abs": 0.06580595672130585,
|
||
|
|
"epoch": 0.7022572554639914,
|
||
|
|
"grad_norm": 57.5,
|
||
|
|
"learning_rate": 6.276106980274944e-08,
|
||
|
|
"logits/chosen": -2.031146764755249,
|
||
|
|
"logits/rejected": -2.07769513130188,
|
||
|
|
"logps/chosen": -0.3201434314250946,
|
||
|
|
"logps/rejected": -0.3183977007865906,
|
||
|
|
"loss": 0.15187807381153107,
|
||
|
|
"loss/core": 0.14405521750450134,
|
||
|
|
"loss/preference_sft": 0.3201434314250946,
|
||
|
|
"loss/reference_anchor": 0.07229013741016388,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 1.4163552522659302,
|
||
|
|
"rewards/margins": 0.759195864200592,
|
||
|
|
"rewards/rejected": 0.6571593880653381,
|
||
|
|
"step": 735
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.25509342551231384,
|
||
|
|
"drift/rejected_abs": 0.1779322326183319,
|
||
|
|
"epoch": 0.707034515705243,
|
||
|
|
"grad_norm": 5.75,
|
||
|
|
"learning_rate": 5.919188255436777e-08,
|
||
|
|
"logits/chosen": -2.076237440109253,
|
||
|
|
"logits/rejected": -2.094517230987549,
|
||
|
|
"logps/chosen": -0.31439438462257385,
|
||
|
|
"logps/rejected": -0.3211822211742401,
|
||
|
|
"loss": 1.5399339199066162,
|
||
|
|
"loss/core": 1.4819676876068115,
|
||
|
|
"loss/preference_sft": 0.31439438462257385,
|
||
|
|
"loss/reference_anchor": 0.7414413094520569,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.5507071018218994,
|
||
|
|
"rewards/margins": 0.7717518210411072,
|
||
|
|
"rewards/rejected": 1.7789552211761475,
|
||
|
|
"step": 740
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.31124961376190186,
|
||
|
|
"drift/rejected_abs": 0.15094509720802307,
|
||
|
|
"epoch": 0.7118117759464947,
|
||
|
|
"grad_norm": 402.0,
|
||
|
|
"learning_rate": 5.571350963575727e-08,
|
||
|
|
"logits/chosen": -2.030458450317383,
|
||
|
|
"logits/rejected": -2.0747735500335693,
|
||
|
|
"logps/chosen": -0.2915259897708893,
|
||
|
|
"logps/rejected": -0.2831938862800598,
|
||
|
|
"loss": 0.6287981271743774,
|
||
|
|
"loss/core": 0.6039372682571411,
|
||
|
|
"loss/preference_sft": 0.2915259897708893,
|
||
|
|
"loss/reference_anchor": 0.30232542753219604,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 3.110504150390625,
|
||
|
|
"rewards/margins": 1.6071808338165283,
|
||
|
|
"rewards/rejected": 1.5033233165740967,
|
||
|
|
"step": 745
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.11992615461349487,
|
||
|
|
"drift/rejected_abs": 0.12524381279945374,
|
||
|
|
"epoch": 0.7165890361877463,
|
||
|
|
"grad_norm": 274.0,
|
||
|
|
"learning_rate": 5.232760656403923e-08,
|
||
|
|
"logits/chosen": -2.166827917098999,
|
||
|
|
"logits/rejected": -2.1589953899383545,
|
||
|
|
"logps/chosen": -0.3125108480453491,
|
||
|
|
"logps/rejected": -0.304274320602417,
|
||
|
|
"loss": 0.5836885571479797,
|
||
|
|
"loss/core": 0.5603240728378296,
|
||
|
|
"loss/preference_sft": 0.3125108480453491,
|
||
|
|
"loss/reference_anchor": 0.2802754044532776,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.197912335395813,
|
||
|
|
"rewards/margins": -0.051997799426317215,
|
||
|
|
"rewards/rejected": 1.2499101161956787,
|
||
|
|
"step": 750
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.23778387904167175,
|
||
|
|
"drift/rejected_abs": 0.16927723586559296,
|
||
|
|
"epoch": 0.721366296428998,
|
||
|
|
"grad_norm": 18.75,
|
||
|
|
"learning_rate": 4.9035784845695675e-08,
|
||
|
|
"logits/chosen": -1.9069569110870361,
|
||
|
|
"logits/rejected": -1.817491888999939,
|
||
|
|
"logps/chosen": -0.30986255407333374,
|
||
|
|
"logps/rejected": -0.29673609137535095,
|
||
|
|
"loss": 0.40122777223587036,
|
||
|
|
"loss/core": 0.3844679892063141,
|
||
|
|
"loss/preference_sft": 0.30986255407333374,
|
||
|
|
"loss/reference_anchor": 0.19247810542583466,
|
||
|
|
"rewards/accuracies": 0.824999988079071,
|
||
|
|
"rewards/chosen": 2.373359203338623,
|
||
|
|
"rewards/margins": 0.687952995300293,
|
||
|
|
"rewards/rejected": 1.6854064464569092,
|
||
|
|
"step": 755
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.19871307909488678,
|
||
|
|
"drift/rejected_abs": 0.08636997640132904,
|
||
|
|
"epoch": 0.7261435566702497,
|
||
|
|
"grad_norm": 22.0,
|
||
|
|
"learning_rate": 4.583961120958027e-08,
|
||
|
|
"logits/chosen": -2.157874345779419,
|
||
|
|
"logits/rejected": -2.1768975257873535,
|
||
|
|
"logps/chosen": -0.3034326434135437,
|
||
|
|
"logps/rejected": -0.2945040166378021,
|
||
|
|
"loss": 0.28818535804748535,
|
||
|
|
"loss/core": 0.27553677558898926,
|
||
|
|
"loss/preference_sft": 0.3034326434135437,
|
||
|
|
"loss/reference_anchor": 0.13830482959747314,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.9834944009780884,
|
||
|
|
"rewards/margins": 1.1288237571716309,
|
||
|
|
"rewards/rejected": 0.8546707034111023,
|
||
|
|
"step": 760
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.17785514891147614,
|
||
|
|
"drift/rejected_abs": 0.10489670932292938,
|
||
|
|
"epoch": 0.7309208169115012,
|
||
|
|
"grad_norm": 44.75,
|
||
|
|
"learning_rate": 4.2740606861239594e-08,
|
||
|
|
"logits/chosen": -2.0543203353881836,
|
||
|
|
"logits/rejected": -2.1251354217529297,
|
||
|
|
"logps/chosen": -0.2776801586151123,
|
||
|
|
"logps/rejected": -0.2736127972602844,
|
||
|
|
"loss": 0.24507012963294983,
|
||
|
|
"loss/core": 0.2341887503862381,
|
||
|
|
"loss/preference_sft": 0.2776801586151123,
|
||
|
|
"loss/reference_anchor": 0.11731725931167603,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.7780964374542236,
|
||
|
|
"rewards/margins": 0.7318509817123413,
|
||
|
|
"rewards/rejected": 1.0462454557418823,
|
||
|
|
"step": 765
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3685626983642578,
|
||
|
|
"drift/rejected_abs": 0.23492638766765594,
|
||
|
|
"epoch": 0.7356980771527529,
|
||
|
|
"grad_norm": 52.25,
|
||
|
|
"learning_rate": 3.974024675890189e-08,
|
||
|
|
"logits/chosen": -1.9574251174926758,
|
||
|
|
"logits/rejected": -1.9248749017715454,
|
||
|
|
"logps/chosen": -0.3148086667060852,
|
||
|
|
"logps/rejected": -0.28525421023368835,
|
||
|
|
"loss": 2.2472522258758545,
|
||
|
|
"loss/core": 2.1636857986450195,
|
||
|
|
"loss/preference_sft": 0.3148086667060852,
|
||
|
|
"loss/reference_anchor": 1.0827428102493286,
|
||
|
|
"rewards/accuracies": 0.8125,
|
||
|
|
"rewards/chosen": 3.68304181098938,
|
||
|
|
"rewards/margins": 1.3362334966659546,
|
||
|
|
"rewards/rejected": 2.346808433532715,
|
||
|
|
"step": 770
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.15117646753787994,
|
||
|
|
"drift/rejected_abs": 0.09319499135017395,
|
||
|
|
"epoch": 0.7404753373940045,
|
||
|
|
"grad_norm": 2.078125,
|
||
|
|
"learning_rate": 3.683995891147695e-08,
|
||
|
|
"logits/chosen": -2.073338031768799,
|
||
|
|
"logits/rejected": -2.103886127471924,
|
||
|
|
"logps/chosen": -0.30775031447410583,
|
||
|
|
"logps/rejected": -0.3014872670173645,
|
||
|
|
"loss": 0.1496042162179947,
|
||
|
|
"loss/core": 0.14195644855499268,
|
||
|
|
"loss/preference_sft": 0.30775031447410583,
|
||
|
|
"loss/reference_anchor": 0.07119497656822205,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.5114030838012695,
|
||
|
|
"rewards/margins": 0.5813368558883667,
|
||
|
|
"rewards/rejected": 0.9300659894943237,
|
||
|
|
"step": 775
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.18838293850421906,
|
||
|
|
"drift/rejected_abs": 0.12301009893417358,
|
||
|
|
"epoch": 0.7452525976352562,
|
||
|
|
"grad_norm": 19.75,
|
||
|
|
"learning_rate": 3.404112369890108e-08,
|
||
|
|
"logits/chosen": -2.15893816947937,
|
||
|
|
"logits/rejected": -2.1729979515075684,
|
||
|
|
"logps/chosen": -0.30296415090560913,
|
||
|
|
"logps/rejected": -0.307880163192749,
|
||
|
|
"loss": 0.23419561982154846,
|
||
|
|
"loss/core": 0.2235209047794342,
|
||
|
|
"loss/preference_sft": 0.30296415090560913,
|
||
|
|
"loss/reference_anchor": 0.11203303188085556,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.8827606439590454,
|
||
|
|
"rewards/margins": 0.6561026573181152,
|
||
|
|
"rewards/rejected": 1.2266579866409302,
|
||
|
|
"step": 780
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.34316182136535645,
|
||
|
|
"drift/rejected_abs": 0.13256040215492249,
|
||
|
|
"epoch": 0.7500298578765078,
|
||
|
|
"grad_norm": 1.765625,
|
||
|
|
"learning_rate": 3.134507321515106e-08,
|
||
|
|
"logits/chosen": -1.9713062047958374,
|
||
|
|
"logits/rejected": -1.9291038513183594,
|
||
|
|
"logps/chosen": -0.31109505891799927,
|
||
|
|
"logps/rejected": -0.2964383065700531,
|
||
|
|
"loss": 1.2833325862884521,
|
||
|
|
"loss/core": 1.2346382141113281,
|
||
|
|
"loss/preference_sft": 0.31109505891799927,
|
||
|
|
"loss/reference_anchor": 0.6181516647338867,
|
||
|
|
"rewards/accuracies": 0.8374999761581421,
|
||
|
|
"rewards/chosen": 3.4299960136413574,
|
||
|
|
"rewards/margins": 2.1115283966064453,
|
||
|
|
"rewards/rejected": 1.318467617034912,
|
||
|
|
"step": 785
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2622746527194977,
|
||
|
|
"drift/rejected_abs": 0.18666431307792664,
|
||
|
|
"epoch": 0.7548071181177595,
|
||
|
|
"grad_norm": 576.0,
|
||
|
|
"learning_rate": 2.875309063423956e-08,
|
||
|
|
"logits/chosen": -2.1486878395080566,
|
||
|
|
"logits/rejected": -1.9767792224884033,
|
||
|
|
"logps/chosen": -0.28585386276245117,
|
||
|
|
"logps/rejected": -0.3247356414794922,
|
||
|
|
"loss": 1.1903468370437622,
|
||
|
|
"loss/core": 1.1452568769454956,
|
||
|
|
"loss/preference_sft": 0.28585386276245117,
|
||
|
|
"loss/reference_anchor": 0.5726125240325928,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 2.621675968170166,
|
||
|
|
"rewards/margins": 0.7564936876296997,
|
||
|
|
"rewards/rejected": 1.8651822805404663,
|
||
|
|
"step": 790
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2373015433549881,
|
||
|
|
"drift/rejected_abs": 0.11792800575494766,
|
||
|
|
"epoch": 0.7595843783590112,
|
||
|
|
"grad_norm": 25.0,
|
||
|
|
"learning_rate": 2.626640959949375e-08,
|
||
|
|
"logits/chosen": -1.9540714025497437,
|
||
|
|
"logits/rejected": -1.9722576141357422,
|
||
|
|
"logps/chosen": -0.30823540687561035,
|
||
|
|
"logps/rejected": -0.2971736788749695,
|
||
|
|
"loss": 1.095926284790039,
|
||
|
|
"loss/core": 1.0540673732757568,
|
||
|
|
"loss/preference_sft": 0.30823540687561035,
|
||
|
|
"loss/reference_anchor": 0.5272958278656006,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 2.3723232746124268,
|
||
|
|
"rewards/margins": 1.1968507766723633,
|
||
|
|
"rewards/rejected": 1.175472378730774,
|
||
|
|
"step": 795
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.22911421954631805,
|
||
|
|
"drift/rejected_abs": 0.11774910986423492,
|
||
|
|
"epoch": 0.7643616386002627,
|
||
|
|
"grad_norm": 4.5,
|
||
|
|
"learning_rate": 2.388621363640797e-08,
|
||
|
|
"logits/chosen": -2.1360487937927246,
|
||
|
|
"logits/rejected": -2.111950159072876,
|
||
|
|
"logps/chosen": -0.2900012731552124,
|
||
|
|
"logps/rejected": -0.287829726934433,
|
||
|
|
"loss": 0.3200365900993347,
|
||
|
|
"loss/core": 0.30634862184524536,
|
||
|
|
"loss/preference_sft": 0.2900012731552124,
|
||
|
|
"loss/reference_anchor": 0.15350642800331116,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.289325475692749,
|
||
|
|
"rewards/margins": 1.1134859323501587,
|
||
|
|
"rewards/rejected": 1.1758396625518799,
|
||
|
|
"step": 800
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.22425393760204315,
|
||
|
|
"drift/rejected_abs": 0.19786129891872406,
|
||
|
|
"epoch": 0.7691388988415144,
|
||
|
|
"grad_norm": 9.3125,
|
||
|
|
"learning_rate": 2.1613635589349756e-08,
|
||
|
|
"logits/chosen": -2.0049588680267334,
|
||
|
|
"logits/rejected": -1.9930200576782227,
|
||
|
|
"logps/chosen": -0.27656620740890503,
|
||
|
|
"logps/rejected": -0.271397203207016,
|
||
|
|
"loss": 0.5283772349357605,
|
||
|
|
"loss/core": 0.5072607398033142,
|
||
|
|
"loss/preference_sft": 0.27656620740890503,
|
||
|
|
"loss/reference_anchor": 0.25389814376831055,
|
||
|
|
"rewards/accuracies": 0.800000011920929,
|
||
|
|
"rewards/chosen": 2.2400403022766113,
|
||
|
|
"rewards/margins": 0.2621103823184967,
|
||
|
|
"rewards/rejected": 1.9779300689697266,
|
||
|
|
"step": 805
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1524631530046463,
|
||
|
|
"drift/rejected_abs": 0.14172717928886414,
|
||
|
|
"epoch": 0.773916159082766,
|
||
|
|
"grad_norm": 25.5,
|
||
|
|
"learning_rate": 1.944975708238708e-08,
|
||
|
|
"logits/chosen": -2.2034547328948975,
|
||
|
|
"logits/rejected": -2.108600616455078,
|
||
|
|
"logps/chosen": -0.30901938676834106,
|
||
|
|
"logps/rejected": -0.32294124364852905,
|
||
|
|
"loss": 0.2564198076725006,
|
||
|
|
"loss/core": 0.2449137419462204,
|
||
|
|
"loss/preference_sft": 0.30901938676834106,
|
||
|
|
"loss/reference_anchor": 0.12251242250204086,
|
||
|
|
"rewards/accuracies": 0.875,
|
||
|
|
"rewards/chosen": 1.523074746131897,
|
||
|
|
"rewards/margins": 0.10768821090459824,
|
||
|
|
"rewards/rejected": 1.4153865575790405,
|
||
|
|
"step": 810
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2113420069217682,
|
||
|
|
"drift/rejected_abs": 0.10915307700634003,
|
||
|
|
"epoch": 0.7786934193240177,
|
||
|
|
"grad_norm": 42.25,
|
||
|
|
"learning_rate": 1.7395608004493884e-08,
|
||
|
|
"logits/chosen": -2.0760385990142822,
|
||
|
|
"logits/rejected": -2.164649486541748,
|
||
|
|
"logps/chosen": -0.28481438755989075,
|
||
|
|
"logps/rejected": -0.28481075167655945,
|
||
|
|
"loss": 0.25434860587120056,
|
||
|
|
"loss/core": 0.24307891726493835,
|
||
|
|
"loss/preference_sft": 0.28481438755989075,
|
||
|
|
"loss/reference_anchor": 0.12178117036819458,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.1131508350372314,
|
||
|
|
"rewards/margins": 1.0244156122207642,
|
||
|
|
"rewards/rejected": 1.0887352228164673,
|
||
|
|
"step": 815
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.19611340761184692,
|
||
|
|
"drift/rejected_abs": 0.07267270982265472,
|
||
|
|
"epoch": 0.7834706795652693,
|
||
|
|
"grad_norm": 3.203125,
|
||
|
|
"learning_rate": 1.5452166019378987e-08,
|
||
|
|
"logits/chosen": -2.0779826641082764,
|
||
|
|
"logits/rejected": -2.111990213394165,
|
||
|
|
"logps/chosen": -0.28833597898483276,
|
||
|
|
"logps/rejected": -0.3186201751232147,
|
||
|
|
"loss": 0.25229331851005554,
|
||
|
|
"loss/core": 0.24105815589427948,
|
||
|
|
"loss/preference_sft": 0.28833597898483276,
|
||
|
|
"loss/reference_anchor": 0.12096866220235825,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.9596529006958008,
|
||
|
|
"rewards/margins": 1.2877837419509888,
|
||
|
|
"rewards/rejected": 0.6718690991401672,
|
||
|
|
"step": 820
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.24504788219928741,
|
||
|
|
"drift/rejected_abs": 0.12149325758218765,
|
||
|
|
"epoch": 0.788247939806521,
|
||
|
|
"grad_norm": 61.0,
|
||
|
|
"learning_rate": 1.3620356100170788e-08,
|
||
|
|
"logits/chosen": -2.120765209197998,
|
||
|
|
"logits/rejected": -2.2250657081604004,
|
||
|
|
"logps/chosen": -0.3034924864768982,
|
||
|
|
"logps/rejected": -0.2849355936050415,
|
||
|
|
"loss": 0.31910449266433716,
|
||
|
|
"loss/core": 0.3053540587425232,
|
||
|
|
"loss/preference_sft": 0.3034924864768982,
|
||
|
|
"loss/reference_anchor": 0.15299084782600403,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 2.4493212699890137,
|
||
|
|
"rewards/margins": 1.2364730834960938,
|
||
|
|
"rewards/rejected": 1.2128479480743408,
|
||
|
|
"step": 825
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1710740029811859,
|
||
|
|
"drift/rejected_abs": 0.16245757043361664,
|
||
|
|
"epoch": 0.7930252000477725,
|
||
|
|
"grad_norm": 8.75,
|
||
|
|
"learning_rate": 1.190105008918041e-08,
|
||
|
|
"logits/chosen": -2.1558587551116943,
|
||
|
|
"logits/rejected": -2.1049137115478516,
|
||
|
|
"logps/chosen": -0.280143678188324,
|
||
|
|
"logps/rejected": -0.3109564781188965,
|
||
|
|
"loss": 0.34464460611343384,
|
||
|
|
"loss/core": 0.3301551938056946,
|
||
|
|
"loss/preference_sft": 0.280143678188324,
|
||
|
|
"loss/reference_anchor": 0.16517803072929382,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 1.7102434635162354,
|
||
|
|
"rewards/margins": 0.08717556297779083,
|
||
|
|
"rewards/rejected": 1.623067855834961,
|
||
|
|
"step": 830
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.23977330327033997,
|
||
|
|
"drift/rejected_abs": 0.11179669201374054,
|
||
|
|
"epoch": 0.7978024602890242,
|
||
|
|
"grad_norm": 160.0,
|
||
|
|
"learning_rate": 1.0295066282951737e-08,
|
||
|
|
"logits/chosen": -2.0725464820861816,
|
||
|
|
"logits/rejected": -2.0728602409362793,
|
||
|
|
"logps/chosen": -0.2556314766407013,
|
||
|
|
"logps/rejected": -0.3013136684894562,
|
||
|
|
"loss": 0.39849942922592163,
|
||
|
|
"loss/core": 0.38221579790115356,
|
||
|
|
"loss/preference_sft": 0.2556314766407013,
|
||
|
|
"loss/reference_anchor": 0.19155162572860718,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 2.395071268081665,
|
||
|
|
"rewards/margins": 1.566588282585144,
|
||
|
|
"rewards/rejected": 0.8284828066825867,
|
||
|
|
"step": 835
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.280472993850708,
|
||
|
|
"drift/rejected_abs": 0.10291789472103119,
|
||
|
|
"epoch": 0.8025797205302759,
|
||
|
|
"grad_norm": 2592.0,
|
||
|
|
"learning_rate": 8.803169042796765e-09,
|
||
|
|
"logits/chosen": -2.0131630897521973,
|
||
|
|
"logits/rejected": -2.0630502700805664,
|
||
|
|
"logps/chosen": -0.32252293825149536,
|
||
|
|
"logps/rejected": -0.30228742957115173,
|
||
|
|
"loss": 0.9987700581550598,
|
||
|
|
"loss/core": 0.9603092074394226,
|
||
|
|
"loss/preference_sft": 0.32252293825149536,
|
||
|
|
"loss/reference_anchor": 0.48055869340896606,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.8026461601257324,
|
||
|
|
"rewards/margins": 1.7743022441864014,
|
||
|
|
"rewards/rejected": 1.0283435583114624,
|
||
|
|
"step": 840
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.14095182716846466,
|
||
|
|
"drift/rejected_abs": 0.06469657272100449,
|
||
|
|
"epoch": 0.8073569807715275,
|
||
|
|
"grad_norm": 3.53125,
|
||
|
|
"learning_rate": 7.4260684310008815e-09,
|
||
|
|
"logits/chosen": -2.151334285736084,
|
||
|
|
"logits/rejected": -2.22589111328125,
|
||
|
|
"logps/chosen": -0.29466524720191956,
|
||
|
|
"logps/rejected": -0.3147200047969818,
|
||
|
|
"loss": 0.1673881560564041,
|
||
|
|
"loss/core": 0.15918193757534027,
|
||
|
|
"loss/preference_sft": 0.29466524720191956,
|
||
|
|
"loss/reference_anchor": 0.07995006442070007,
|
||
|
|
"rewards/accuracies": 0.9375,
|
||
|
|
"rewards/chosen": 1.4074944257736206,
|
||
|
|
"rewards/margins": 0.9872743487358093,
|
||
|
|
"rewards/rejected": 0.4202199876308441,
|
||
|
|
"step": 845
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.15515586733818054,
|
||
|
|
"drift/rejected_abs": 0.09092449396848679,
|
||
|
|
"epoch": 0.8121342410127792,
|
||
|
|
"grad_norm": 36.0,
|
||
|
|
"learning_rate": 6.164419872871835e-09,
|
||
|
|
"logits/chosen": -2.2680633068084717,
|
||
|
|
"logits/rejected": -2.3154819011688232,
|
||
|
|
"logps/chosen": -0.25293999910354614,
|
||
|
|
"logps/rejected": -0.24826927483081818,
|
||
|
|
"loss": 0.1021793931722641,
|
||
|
|
"loss/core": 0.09663856029510498,
|
||
|
|
"loss/preference_sft": 0.25293999910354614,
|
||
|
|
"loss/reference_anchor": 0.0485837496817112,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 1.5486667156219482,
|
||
|
|
"rewards/margins": 0.6556525826454163,
|
||
|
|
"rewards/rejected": 0.8930143117904663,
|
||
|
|
"step": 850
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2674712538719177,
|
||
|
|
"drift/rejected_abs": 0.12466195970773697,
|
||
|
|
"epoch": 0.8169115012540308,
|
||
|
|
"grad_norm": 59.75,
|
||
|
|
"learning_rate": 5.018823844792602e-09,
|
||
|
|
"logits/chosen": -1.9834197759628296,
|
||
|
|
"logits/rejected": -2.0199997425079346,
|
||
|
|
"logps/chosen": -0.30243033170700073,
|
||
|
|
"logps/rejected": -0.29364144802093506,
|
||
|
|
"loss": 0.8800665736198425,
|
||
|
|
"loss/core": 0.8460347056388855,
|
||
|
|
"loss/preference_sft": 0.30243033170700073,
|
||
|
|
"loss/reference_anchor": 0.4235146939754486,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.672776937484741,
|
||
|
|
"rewards/margins": 1.4277340173721313,
|
||
|
|
"rewards/rejected": 1.2450430393218994,
|
||
|
|
"step": 855
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.4073793292045593,
|
||
|
|
"drift/rejected_abs": 0.19243915379047394,
|
||
|
|
"epoch": 0.8216887614952825,
|
||
|
|
"grad_norm": 2.71875,
|
||
|
|
"learning_rate": 3.989825588427281e-09,
|
||
|
|
"logits/chosen": -1.9286167621612549,
|
||
|
|
"logits/rejected": -2.017611026763916,
|
||
|
|
"logps/chosen": -0.3008403480052948,
|
||
|
|
"logps/rejected": -0.3007083833217621,
|
||
|
|
"loss": 1.669266939163208,
|
||
|
|
"loss/core": 1.606706976890564,
|
||
|
|
"loss/preference_sft": 0.3008403480052948,
|
||
|
|
"loss/reference_anchor": 0.8040491342544556,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 4.073380470275879,
|
||
|
|
"rewards/margins": 2.1619973182678223,
|
||
|
|
"rewards/rejected": 1.9113833904266357,
|
||
|
|
"step": 860
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.24328342080116272,
|
||
|
|
"drift/rejected_abs": 0.155743807554245,
|
||
|
|
"epoch": 0.826466021736534,
|
||
|
|
"grad_norm": 59.5,
|
||
|
|
"learning_rate": 3.077914851215585e-09,
|
||
|
|
"logits/chosen": -1.938284158706665,
|
||
|
|
"logits/rejected": -2.010418176651001,
|
||
|
|
"logps/chosen": -0.2928522229194641,
|
||
|
|
"logps/rejected": -0.29682302474975586,
|
||
|
|
"loss": 0.6092508435249329,
|
||
|
|
"loss/core": 0.5850942730903625,
|
||
|
|
"loss/preference_sft": 0.2928522229194641,
|
||
|
|
"loss/reference_anchor": 0.292802631855011,
|
||
|
|
"rewards/accuracies": 0.8500000238418579,
|
||
|
|
"rewards/chosen": 2.432499647140503,
|
||
|
|
"rewards/margins": 0.8761860132217407,
|
||
|
|
"rewards/rejected": 1.5563137531280518,
|
||
|
|
"step": 865
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.32113441824913025,
|
||
|
|
"drift/rejected_abs": 0.09325097501277924,
|
||
|
|
"epoch": 0.8312432819777857,
|
||
|
|
"grad_norm": 13.8125,
|
||
|
|
"learning_rate": 2.2835256532794387e-09,
|
||
|
|
"logits/chosen": -2.088789939880371,
|
||
|
|
"logits/rejected": -2.097416877746582,
|
||
|
|
"logps/chosen": -0.30860528349876404,
|
||
|
|
"logps/rejected": -0.2980922758579254,
|
||
|
|
"loss": 1.1249186992645264,
|
||
|
|
"loss/core": 1.0819661617279053,
|
||
|
|
"loss/preference_sft": 0.30860528349876404,
|
||
|
|
"loss/reference_anchor": 0.5418386459350586,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 3.210113525390625,
|
||
|
|
"rewards/margins": 2.283482074737549,
|
||
|
|
"rewards/rejected": 0.9266314506530762,
|
||
|
|
"step": 870
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.1851302534341812,
|
||
|
|
"drift/rejected_abs": 0.101983942091465,
|
||
|
|
"epoch": 0.8360205422190374,
|
||
|
|
"grad_norm": 7.34375,
|
||
|
|
"learning_rate": 1.6070360808531359e-09,
|
||
|
|
"logits/chosen": -2.1583588123321533,
|
||
|
|
"logits/rejected": -2.2066867351531982,
|
||
|
|
"logps/chosen": -0.30905434489250183,
|
||
|
|
"logps/rejected": -0.3321419954299927,
|
||
|
|
"loss": 0.7497228980064392,
|
||
|
|
"loss/core": 0.7203631401062012,
|
||
|
|
"loss/preference_sft": 0.30905434489250183,
|
||
|
|
"loss/reference_anchor": 0.36055710911750793,
|
||
|
|
"rewards/accuracies": 0.9125000238418579,
|
||
|
|
"rewards/chosen": 1.8509142398834229,
|
||
|
|
"rewards/margins": 0.8404647707939148,
|
||
|
|
"rewards/rejected": 1.0104494094848633,
|
||
|
|
"step": 875
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.24564549326896667,
|
||
|
|
"drift/rejected_abs": 0.18484672904014587,
|
||
|
|
"epoch": 0.840797802460289,
|
||
|
|
"grad_norm": 56.5,
|
||
|
|
"learning_rate": 1.0487681063345854e-09,
|
||
|
|
"logits/chosen": -2.097510814666748,
|
||
|
|
"logits/rejected": -2.0806679725646973,
|
||
|
|
"logps/chosen": -0.2908017039299011,
|
||
|
|
"logps/rejected": -0.3314078450202942,
|
||
|
|
"loss": 0.9822714924812317,
|
||
|
|
"loss/core": 0.9446454048156738,
|
||
|
|
"loss/preference_sft": 0.2908017039299011,
|
||
|
|
"loss/reference_anchor": 0.4726010262966156,
|
||
|
|
"rewards/accuracies": 0.8999999761581421,
|
||
|
|
"rewards/chosen": 2.4564552307128906,
|
||
|
|
"rewards/margins": 0.6901001930236816,
|
||
|
|
"rewards/rejected": 1.7663547992706299,
|
||
|
|
"step": 880
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2716746926307678,
|
||
|
|
"drift/rejected_abs": 0.11291627585887909,
|
||
|
|
"epoch": 0.8455750627015407,
|
||
|
|
"grad_norm": 24.875,
|
||
|
|
"learning_rate": 6.089874350439505e-10,
|
||
|
|
"logits/chosen": -2.0179717540740967,
|
||
|
|
"logits/rejected": -2.096179485321045,
|
||
|
|
"logps/chosen": -0.2834438979625702,
|
||
|
|
"logps/rejected": -0.27716511487960815,
|
||
|
|
"loss": 0.4409695565700531,
|
||
|
|
"loss/core": 0.42294350266456604,
|
||
|
|
"loss/preference_sft": 0.2834438979625702,
|
||
|
|
"loss/reference_anchor": 0.21200327575206757,
|
||
|
|
"rewards/accuracies": 0.925000011920929,
|
||
|
|
"rewards/chosen": 2.7161340713500977,
|
||
|
|
"rewards/margins": 1.5943241119384766,
|
||
|
|
"rewards/rejected": 1.1218100786209106,
|
||
|
|
"step": 885
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.2433289736509323,
|
||
|
|
"drift/rejected_abs": 0.14475123584270477,
|
||
|
|
"epoch": 0.8503523229427923,
|
||
|
|
"grad_norm": 342.0,
|
||
|
|
"learning_rate": 2.8790337876233305e-10,
|
||
|
|
"logits/chosen": -1.9500147104263306,
|
||
|
|
"logits/rejected": -2.0721776485443115,
|
||
|
|
"logps/chosen": -0.30229297280311584,
|
||
|
|
"logps/rejected": -0.2738516330718994,
|
||
|
|
"loss": 0.661720335483551,
|
||
|
|
"loss/core": 0.6356015205383301,
|
||
|
|
"loss/preference_sft": 0.30229297280311584,
|
||
|
|
"loss/reference_anchor": 0.3180215358734131,
|
||
|
|
"rewards/accuracies": 0.887499988079071,
|
||
|
|
"rewards/chosen": 2.4331092834472656,
|
||
|
|
"rewards/margins": 0.9898265600204468,
|
||
|
|
"rewards/rejected": 1.443282961845398,
|
||
|
|
"step": 890
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.22940556704998016,
|
||
|
|
"drift/rejected_abs": 0.1513952761888504,
|
||
|
|
"epoch": 0.855129583184044,
|
||
|
|
"grad_norm": 50.75,
|
||
|
|
"learning_rate": 8.566875611068503e-11,
|
||
|
|
"logits/chosen": -2.022388458251953,
|
||
|
|
"logits/rejected": -2.0541129112243652,
|
||
|
|
"logps/chosen": -0.27332523465156555,
|
||
|
|
"logps/rejected": -0.29117733240127563,
|
||
|
|
"loss": 0.33594077825546265,
|
||
|
|
"loss/core": 0.3218122124671936,
|
||
|
|
"loss/preference_sft": 0.27332523465156555,
|
||
|
|
"loss/reference_anchor": 0.1610485315322876,
|
||
|
|
"rewards/accuracies": 0.862500011920929,
|
||
|
|
"rewards/chosen": 2.292048454284668,
|
||
|
|
"rewards/margins": 0.7850906252861023,
|
||
|
|
"rewards/rejected": 1.506957769393921,
|
||
|
|
"step": 895
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"drift/chosen_abs": 0.3108707070350647,
|
||
|
|
"drift/rejected_abs": 0.08576279878616333,
|
||
|
|
"epoch": 0.8599068434252956,
|
||
|
|
"grad_norm": 18.5,
|
||
|
|
"learning_rate": 2.3798198163782478e-12,
|
||
|
|
"logits/chosen": -2.0334689617156982,
|
||
|
|
"logits/rejected": -2.154085636138916,
|
||
|
|
"logps/chosen": -0.26980048418045044,
|
||
|
|
"logps/rejected": -0.26923516392707825,
|
||
|
|
"loss": 1.0657157897949219,
|
||
|
|
"loss/core": 1.0252106189727783,
|
||
|
|
"loss/preference_sft": 0.26980048418045044,
|
||
|
|
"loss/reference_anchor": 0.5130892395973206,
|
||
|
|
"rewards/accuracies": 0.9750000238418579,
|
||
|
|
"rewards/chosen": 3.1086440086364746,
|
||
|
|
"rewards/margins": 2.26877498626709,
|
||
|
|
"rewards/rejected": 0.8398691415786743,
|
||
|
|
"step": 900
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8599068434252956,
|
||
|
|
"step": 900,
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_loss": 0.7444233691361215,
|
||
|
|
"train_runtime": 7251.7494,
|
||
|
|
"train_samples_per_second": 1.986,
|
||
|
|
"train_steps_per_second": 0.124
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 5,
|
||
|
|
"max_steps": 900,
|
||
|
|
"num_input_tokens_seen": 0,
|
||
|
|
"num_train_epochs": 1,
|
||
|
|
"save_steps": 900,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": true
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 1,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|