4904 lines
186 KiB
JSON
4904 lines
186 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.2860548271752086,
|
|
"eval_steps": 500,
|
|
"global_step": 900,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"drift/chosen_abs": 0.2889288067817688,
|
|
"drift/rejected_abs": 0.17525029182434082,
|
|
"epoch": 0.0015891934843067143,
|
|
"grad_norm": 1.0546875,
|
|
"learning_rate": 1.111111111111111e-09,
|
|
"logits/chosen": -2.227959632873535,
|
|
"logits/rejected": -2.153649091720581,
|
|
"logps/chosen": -0.25035396218299866,
|
|
"logps/rejected": -0.25459814071655273,
|
|
"loss": 0.2694305181503296,
|
|
"loss/core": 0.24971142411231995,
|
|
"loss/preference_sft": 0.2522751986980438,
|
|
"loss/reference_anchor": 0.3691546618938446,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.8892879486083984,
|
|
"rewards/margins": 1.1368046998977661,
|
|
"rewards/rejected": 1.7524830102920532,
|
|
"ronpo/logit": 0.11368048191070557,
|
|
"ronpo/residual": 0.11023612320423126,
|
|
"ronpo/residual_abs": 0.22715234756469727,
|
|
"ronpo/target": 0.0034443773329257965,
|
|
"ronpo/target_abs": 0.06975062191486359,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 5
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2513688802719116,
|
|
"drift/rejected_abs": 0.17290709912776947,
|
|
"epoch": 0.0031783869686134287,
|
|
"grad_norm": 4.5,
|
|
"learning_rate": 2.5e-09,
|
|
"logits/chosen": -1.9819161891937256,
|
|
"logits/rejected": -1.923046350479126,
|
|
"logps/chosen": -0.26421818137168884,
|
|
"logps/rejected": -0.2721954882144928,
|
|
"loss": 0.17001162469387054,
|
|
"loss/core": 0.15246111154556274,
|
|
"loss/preference_sft": 0.2747360169887543,
|
|
"loss/reference_anchor": 0.3235367238521576,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.5129077434539795,
|
|
"rewards/margins": 0.78448885679245,
|
|
"rewards/rejected": 1.7284189462661743,
|
|
"ronpo/logit": 0.07844887673854828,
|
|
"ronpo/residual": 0.06979227811098099,
|
|
"ronpo/residual_abs": 0.19728776812553406,
|
|
"ronpo/target": 0.008656605146825314,
|
|
"ronpo/target_abs": 0.09380026161670685,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 10
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.14740121364593506,
|
|
"drift/rejected_abs": 0.08022782951593399,
|
|
"epoch": 0.004767580452920143,
|
|
"grad_norm": 7.3125,
|
|
"learning_rate": 3.8888888888888884e-09,
|
|
"logits/chosen": -2.2918307781219482,
|
|
"logits/rejected": -2.2467479705810547,
|
|
"logps/chosen": -0.27619749307632446,
|
|
"logps/rejected": -0.2822367548942566,
|
|
"loss": 0.05183436721563339,
|
|
"loss/core": 0.047430895268917084,
|
|
"loss/preference_sft": 0.277311235666275,
|
|
"loss/reference_anchor": 0.060338426381349564,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.4728786945343018,
|
|
"rewards/margins": 0.6718867421150208,
|
|
"rewards/rejected": 0.8009918928146362,
|
|
"ronpo/logit": 0.06718868762254715,
|
|
"ronpo/residual": 0.102389857172966,
|
|
"ronpo/residual_abs": 0.14397867023944855,
|
|
"ronpo/target": -0.035201169550418854,
|
|
"ronpo/target_abs": 0.08456156402826309,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 15
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1409628540277481,
|
|
"drift/rejected_abs": 0.07020283490419388,
|
|
"epoch": 0.006356773937226857,
|
|
"grad_norm": 1.4765625,
|
|
"learning_rate": 5.277777777777778e-09,
|
|
"logits/chosen": -2.291287422180176,
|
|
"logits/rejected": -2.2165215015411377,
|
|
"logps/chosen": -0.27579012513160706,
|
|
"logps/rejected": -0.278958261013031,
|
|
"loss": 0.03074510395526886,
|
|
"loss/core": 0.026744594797492027,
|
|
"loss/preference_sft": 0.2790845036506653,
|
|
"loss/reference_anchor": 0.052101779729127884,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.409259557723999,
|
|
"rewards/margins": 0.7112404704093933,
|
|
"rewards/rejected": 0.6980190277099609,
|
|
"ronpo/logit": 0.07112405449151993,
|
|
"ronpo/residual": 0.08393178880214691,
|
|
"ronpo/residual_abs": 0.11383052915334702,
|
|
"ronpo/target": -0.012807758525013924,
|
|
"ronpo/target_abs": 0.07244060188531876,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 20
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2341737300157547,
|
|
"drift/rejected_abs": 0.07688029110431671,
|
|
"epoch": 0.007945967421533572,
|
|
"grad_norm": 1.0625,
|
|
"learning_rate": 6.666666666666666e-09,
|
|
"logits/chosen": -2.354504346847534,
|
|
"logits/rejected": -2.393965721130371,
|
|
"logps/chosen": -0.2399778664112091,
|
|
"logps/rejected": -0.2485647201538086,
|
|
"loss": 0.30489665269851685,
|
|
"loss/core": 0.29363757371902466,
|
|
"loss/preference_sft": 0.2436579465866089,
|
|
"loss/reference_anchor": 0.20081612467765808,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.3414459228515625,
|
|
"rewards/margins": 1.5734647512435913,
|
|
"rewards/rejected": 0.7679811716079712,
|
|
"ronpo/logit": 0.15734648704528809,
|
|
"ronpo/residual": 0.15989667177200317,
|
|
"ronpo/residual_abs": 0.2039765864610672,
|
|
"ronpo/target": -0.0025501928757876158,
|
|
"ronpo/target_abs": 0.07882604002952576,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 25
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20403337478637695,
|
|
"drift/rejected_abs": 0.08686282485723495,
|
|
"epoch": 0.009535160905840286,
|
|
"grad_norm": 344.0,
|
|
"learning_rate": 8.055555555555556e-09,
|
|
"logits/chosen": -2.2312631607055664,
|
|
"logits/rejected": -2.1263251304626465,
|
|
"logps/chosen": -0.28779473900794983,
|
|
"logps/rejected": -0.28351494669914246,
|
|
"loss": 0.18434959650039673,
|
|
"loss/core": 0.1724001169204712,
|
|
"loss/preference_sft": 0.29089125990867615,
|
|
"loss/reference_anchor": 0.20990124344825745,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.0403337478637695,
|
|
"rewards/margins": 1.1722705364227295,
|
|
"rewards/rejected": 0.8680631518363953,
|
|
"ronpo/logit": 0.11722707748413086,
|
|
"ronpo/residual": 0.10378406196832657,
|
|
"ronpo/residual_abs": 0.1859159767627716,
|
|
"ronpo/target": 0.013443012721836567,
|
|
"ronpo/target_abs": 0.09407193213701248,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 30
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20877280831336975,
|
|
"drift/rejected_abs": 0.13062386214733124,
|
|
"epoch": 0.011124354390147001,
|
|
"grad_norm": 4.0,
|
|
"learning_rate": 9.444444444444444e-09,
|
|
"logits/chosen": -2.1255409717559814,
|
|
"logits/rejected": -2.06231951713562,
|
|
"logps/chosen": -0.2710401117801666,
|
|
"logps/rejected": -0.28147274255752563,
|
|
"loss": 0.05778823047876358,
|
|
"loss/core": 0.04806242510676384,
|
|
"loss/preference_sft": 0.27798718214035034,
|
|
"loss/reference_anchor": 0.16671733558177948,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.0877280235290527,
|
|
"rewards/margins": 0.7836453318595886,
|
|
"rewards/rejected": 1.3040828704833984,
|
|
"ronpo/logit": 0.0783645361661911,
|
|
"ronpo/residual": 0.06768552213907242,
|
|
"ronpo/residual_abs": 0.1325724571943283,
|
|
"ronpo/target": 0.01067900937050581,
|
|
"ronpo/target_abs": 0.08033877611160278,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 35
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1339804232120514,
|
|
"drift/rejected_abs": 0.1121961921453476,
|
|
"epoch": 0.012713547874453715,
|
|
"grad_norm": 444.0,
|
|
"learning_rate": 1.0833333333333333e-08,
|
|
"logits/chosen": -2.0807948112487793,
|
|
"logits/rejected": -2.042069435119629,
|
|
"logps/chosen": -0.31901049613952637,
|
|
"logps/rejected": -0.3213141858577728,
|
|
"loss": 0.2600531280040741,
|
|
"loss/core": 0.2503146529197693,
|
|
"loss/preference_sft": 0.31382906436920166,
|
|
"loss/reference_anchor": 0.16338738799095154,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 1.3386170864105225,
|
|
"rewards/margins": 0.21692340075969696,
|
|
"rewards/rejected": 1.1216938495635986,
|
|
"ronpo/logit": 0.021692335605621338,
|
|
"ronpo/residual": 0.011326847597956657,
|
|
"ronpo/residual_abs": 0.17393314838409424,
|
|
"ronpo/target": 0.010365481488406658,
|
|
"ronpo/target_abs": 0.08013413846492767,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 40
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2002873420715332,
|
|
"drift/rejected_abs": 0.10398294776678085,
|
|
"epoch": 0.014302741358760428,
|
|
"grad_norm": 4.0,
|
|
"learning_rate": 1.2222222222222222e-08,
|
|
"logits/chosen": -2.149164915084839,
|
|
"logits/rejected": -2.077511787414551,
|
|
"logps/chosen": -0.30421438813209534,
|
|
"logps/rejected": -0.29620206356048584,
|
|
"loss": 0.09899889677762985,
|
|
"loss/core": 0.09022404253482819,
|
|
"loss/preference_sft": 0.2990511357784271,
|
|
"loss/reference_anchor": 0.14559200406074524,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.0019478797912598,
|
|
"rewards/margins": 0.9650747179985046,
|
|
"rewards/rejected": 1.0368731021881104,
|
|
"ronpo/logit": 0.0965074822306633,
|
|
"ronpo/residual": 0.09062016755342484,
|
|
"ronpo/residual_abs": 0.1583360731601715,
|
|
"ronpo/target": 0.005887320265173912,
|
|
"ronpo/target_abs": 0.08980895578861237,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 45
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21013769507408142,
|
|
"drift/rejected_abs": 0.1717991977930069,
|
|
"epoch": 0.015891934843067144,
|
|
"grad_norm": 7.65625,
|
|
"learning_rate": 1.361111111111111e-08,
|
|
"logits/chosen": -2.2315473556518555,
|
|
"logits/rejected": -2.2804598808288574,
|
|
"logps/chosen": -0.38569122552871704,
|
|
"logps/rejected": -0.37548166513442993,
|
|
"loss": 0.49833402037620544,
|
|
"loss/core": 0.47320446372032166,
|
|
"loss/preference_sft": 0.3764234185218811,
|
|
"loss/reference_anchor": 0.4649490416049957,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.10087251663208,
|
|
"rewards/margins": 1.4145400524139404,
|
|
"rewards/rejected": 0.6863324642181396,
|
|
"ronpo/logit": 0.14145398139953613,
|
|
"ronpo/residual": 0.14585429430007935,
|
|
"ronpo/residual_abs": 0.2349962443113327,
|
|
"ronpo/target": -0.004400297999382019,
|
|
"ronpo/target_abs": 0.0763171911239624,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 50
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24356035888195038,
|
|
"drift/rejected_abs": 0.1136392131447792,
|
|
"epoch": 0.01748112832737386,
|
|
"grad_norm": 1.1015625,
|
|
"learning_rate": 1.5e-08,
|
|
"logits/chosen": -2.132157802581787,
|
|
"logits/rejected": -2.104531764984131,
|
|
"logps/chosen": -0.30124813318252563,
|
|
"logps/rejected": -0.29356569051742554,
|
|
"loss": 0.29936105012893677,
|
|
"loss/core": 0.28574955463409424,
|
|
"loss/preference_sft": 0.28952398896217346,
|
|
"loss/reference_anchor": 0.2432773858308792,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.4335732460021973,
|
|
"rewards/margins": 1.3003393411636353,
|
|
"rewards/rejected": 1.133233666419983,
|
|
"ronpo/logit": 0.130033940076828,
|
|
"ronpo/residual": 0.13970455527305603,
|
|
"ronpo/residual_abs": 0.19125238060951233,
|
|
"ronpo/target": -0.009670635685324669,
|
|
"ronpo/target_abs": 0.07653583586215973,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 55
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2864144444465637,
|
|
"drift/rejected_abs": 0.0761997327208519,
|
|
"epoch": 0.01907032181168057,
|
|
"grad_norm": 7.1875,
|
|
"learning_rate": 1.6388888888888888e-08,
|
|
"logits/chosen": -2.0191774368286133,
|
|
"logits/rejected": -2.0377559661865234,
|
|
"logps/chosen": -0.2580411732196808,
|
|
"logps/rejected": -0.2605469822883606,
|
|
"loss": 0.6949197053909302,
|
|
"loss/core": 0.6738607883453369,
|
|
"loss/preference_sft": 0.2634747624397278,
|
|
"loss/reference_anchor": 0.39483022689819336,
|
|
"rewards/accuracies": 0.987500011920929,
|
|
"rewards/chosen": 2.863614559173584,
|
|
"rewards/margins": 2.115543842315674,
|
|
"rewards/rejected": 0.7480708360671997,
|
|
"ronpo/logit": 0.2115544080734253,
|
|
"ronpo/residual": 0.21217253804206848,
|
|
"ronpo/residual_abs": 0.25770795345306396,
|
|
"ronpo/target": -0.0006181513890624046,
|
|
"ronpo/target_abs": 0.07183288037776947,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 60
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23456306755542755,
|
|
"drift/rejected_abs": 0.0846903845667839,
|
|
"epoch": 0.020659515295987287,
|
|
"grad_norm": 168.0,
|
|
"learning_rate": 1.7777777777777777e-08,
|
|
"logits/chosen": -2.3910279273986816,
|
|
"logits/rejected": -2.4622697830200195,
|
|
"logps/chosen": -0.25069719552993774,
|
|
"logps/rejected": -0.24407532811164856,
|
|
"loss": 0.2673589289188385,
|
|
"loss/core": 0.2575306296348572,
|
|
"loss/preference_sft": 0.24895858764648438,
|
|
"loss/reference_anchor": 0.17166933417320251,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.345244884490967,
|
|
"rewards/margins": 1.499186635017395,
|
|
"rewards/rejected": 0.846058189868927,
|
|
"ronpo/logit": 0.14991866052150726,
|
|
"ronpo/residual": 0.12659938633441925,
|
|
"ronpo/residual_abs": 0.19241955876350403,
|
|
"ronpo/target": 0.02331927977502346,
|
|
"ronpo/target_abs": 0.07524405419826508,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 65
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21925799548625946,
|
|
"drift/rejected_abs": 0.11180920898914337,
|
|
"epoch": 0.022248708780294002,
|
|
"grad_norm": 2.71875,
|
|
"learning_rate": 1.9166666666666666e-08,
|
|
"logits/chosen": -2.1304917335510254,
|
|
"logits/rejected": -2.13004994392395,
|
|
"logps/chosen": -0.26294752955436707,
|
|
"logps/rejected": -0.26812273263931274,
|
|
"loss": 0.0758952647447586,
|
|
"loss/core": 0.06884925067424774,
|
|
"loss/preference_sft": 0.26684775948524475,
|
|
"loss/reference_anchor": 0.11423534154891968,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.1923234462738037,
|
|
"rewards/margins": 1.074231505393982,
|
|
"rewards/rejected": 1.1180920600891113,
|
|
"ronpo/logit": 0.10742314904928207,
|
|
"ronpo/residual": 0.09753496944904327,
|
|
"ronpo/residual_abs": 0.14825595915317535,
|
|
"ronpo/target": 0.00988816749304533,
|
|
"ronpo/target_abs": 0.08186240494251251,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 70
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2804720997810364,
|
|
"drift/rejected_abs": 0.1483515202999115,
|
|
"epoch": 0.023837902264600714,
|
|
"grad_norm": 5.84375,
|
|
"learning_rate": 2.0555555555555552e-08,
|
|
"logits/chosen": -2.107518434524536,
|
|
"logits/rejected": -2.0409445762634277,
|
|
"logps/chosen": -0.2804359197616577,
|
|
"logps/rejected": -0.27236875891685486,
|
|
"loss": 0.681021511554718,
|
|
"loss/core": 0.6588231921195984,
|
|
"loss/preference_sft": 0.271370530128479,
|
|
"loss/reference_anchor": 0.41682863235473633,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.8040549755096436,
|
|
"rewards/margins": 1.3575153350830078,
|
|
"rewards/rejected": 1.4465398788452148,
|
|
"ronpo/logit": 0.13575153052806854,
|
|
"ronpo/residual": 0.14553678035736084,
|
|
"ronpo/residual_abs": 0.30461931228637695,
|
|
"ronpo/target": -0.009785245172679424,
|
|
"ronpo/target_abs": 0.08345939964056015,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 75
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.32475045323371887,
|
|
"drift/rejected_abs": 0.13860228657722473,
|
|
"epoch": 0.02542709574890743,
|
|
"grad_norm": 42.5,
|
|
"learning_rate": 2.1944444444444445e-08,
|
|
"logits/chosen": -2.056220531463623,
|
|
"logits/rejected": -2.0234427452087402,
|
|
"logps/chosen": -0.27260422706604004,
|
|
"logps/rejected": -0.27340400218963623,
|
|
"loss": 0.372364342212677,
|
|
"loss/core": 0.35138756036758423,
|
|
"loss/preference_sft": 0.2743496894836426,
|
|
"loss/reference_anchor": 0.39210090041160583,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.2468693256378174,
|
|
"rewards/margins": 1.8645988702774048,
|
|
"rewards/rejected": 1.3822702169418335,
|
|
"ronpo/logit": 0.18645988404750824,
|
|
"ronpo/residual": 0.18637283146381378,
|
|
"ronpo/residual_abs": 0.2369922697544098,
|
|
"ronpo/target": 8.704736683284864e-05,
|
|
"ronpo/target_abs": 0.08356542885303497,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 80
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22833116352558136,
|
|
"drift/rejected_abs": 0.07853401452302933,
|
|
"epoch": 0.027016289233214145,
|
|
"grad_norm": 988.0,
|
|
"learning_rate": 2.3333333333333334e-08,
|
|
"logits/chosen": -2.3675479888916016,
|
|
"logits/rejected": -2.3091373443603516,
|
|
"logps/chosen": -0.2940252423286438,
|
|
"logps/rejected": -0.2664296329021454,
|
|
"loss": 0.5738587975502014,
|
|
"loss/core": 0.5565118789672852,
|
|
"loss/preference_sft": 0.2698623538017273,
|
|
"loss/reference_anchor": 0.31994983553886414,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.281959056854248,
|
|
"rewards/margins": 1.5006506443023682,
|
|
"rewards/rejected": 0.7813081741333008,
|
|
"ronpo/logit": 0.1500650942325592,
|
|
"ronpo/residual": 0.13329221308231354,
|
|
"ronpo/residual_abs": 0.20770840346813202,
|
|
"ronpo/target": 0.01677287369966507,
|
|
"ronpo/target_abs": 0.08873967826366425,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 85
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1896965652704239,
|
|
"drift/rejected_abs": 0.16110625863075256,
|
|
"epoch": 0.028605482717520857,
|
|
"grad_norm": 1.7578125,
|
|
"learning_rate": 2.4722222222222223e-08,
|
|
"logits/chosen": -2.261641025543213,
|
|
"logits/rejected": -2.271144390106201,
|
|
"logps/chosen": -0.2831428349018097,
|
|
"logps/rejected": -0.2776196002960205,
|
|
"loss": 0.27627962827682495,
|
|
"loss/core": 0.2642122805118561,
|
|
"loss/preference_sft": 0.27972379326820374,
|
|
"loss/reference_anchor": 0.21337434649467468,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.896965742111206,
|
|
"rewards/margins": 0.2912505865097046,
|
|
"rewards/rejected": 1.6057151556015015,
|
|
"ronpo/logit": 0.029125045984983444,
|
|
"ronpo/residual": 0.029582571238279343,
|
|
"ronpo/residual_abs": 0.18990905582904816,
|
|
"ronpo/target": -0.0004575133207254112,
|
|
"ronpo/target_abs": 0.08104494959115982,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 90
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22530770301818848,
|
|
"drift/rejected_abs": 0.10128001868724823,
|
|
"epoch": 0.030194676201827572,
|
|
"grad_norm": 3.734375,
|
|
"learning_rate": 2.4998495746616843e-08,
|
|
"logits/chosen": -2.339174747467041,
|
|
"logits/rejected": -2.257371425628662,
|
|
"logps/chosen": -0.27327409386634827,
|
|
"logps/rejected": -0.26891353726387024,
|
|
"loss": 0.345085084438324,
|
|
"loss/core": 0.33327919244766235,
|
|
"loss/preference_sft": 0.26861560344696045,
|
|
"loss/reference_anchor": 0.20925641059875488,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.2530770301818848,
|
|
"rewards/margins": 1.2411054372787476,
|
|
"rewards/rejected": 1.0119714736938477,
|
|
"ronpo/logit": 0.12411054223775864,
|
|
"ronpo/residual": 0.1508522927761078,
|
|
"ronpo/residual_abs": 0.2290908843278885,
|
|
"ronpo/target": -0.0267417524009943,
|
|
"ronpo/target_abs": 0.08890058100223541,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 95
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2532444894313812,
|
|
"drift/rejected_abs": 0.09977379441261292,
|
|
"epoch": 0.03178386968613429,
|
|
"grad_norm": 8.625,
|
|
"learning_rate": 2.4992385337738698e-08,
|
|
"logits/chosen": -1.9698337316513062,
|
|
"logits/rejected": -2.0415987968444824,
|
|
"logps/chosen": -0.29794543981552124,
|
|
"logps/rejected": -0.28950685262680054,
|
|
"loss": 0.24947309494018555,
|
|
"loss/core": 0.23802785575389862,
|
|
"loss/preference_sft": 0.29647988080978394,
|
|
"loss/reference_anchor": 0.1992568075656891,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.531574249267578,
|
|
"rewards/margins": 1.5353649854660034,
|
|
"rewards/rejected": 0.9962092638015747,
|
|
"ronpo/logit": 0.15353651344776154,
|
|
"ronpo/residual": 0.15389743447303772,
|
|
"ronpo/residual_abs": 0.2095203399658203,
|
|
"ronpo/target": -0.00036093126982450485,
|
|
"ronpo/target_abs": 0.08967583626508713,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 100
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3917911648750305,
|
|
"drift/rejected_abs": 0.127794548869133,
|
|
"epoch": 0.033373063170441,
|
|
"grad_norm": 93.5,
|
|
"learning_rate": 2.4981577053636144e-08,
|
|
"logits/chosen": -1.9294116497039795,
|
|
"logits/rejected": -1.936621904373169,
|
|
"logps/chosen": -0.2786913514137268,
|
|
"logps/rejected": -0.2831571400165558,
|
|
"loss": 0.699501633644104,
|
|
"loss/core": 0.6670955419540405,
|
|
"loss/preference_sft": 0.2841565012931824,
|
|
"loss/reference_anchor": 0.6197081208229065,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.9174468517303467,
|
|
"rewards/margins": 2.6591506004333496,
|
|
"rewards/rejected": 1.258296251296997,
|
|
"ronpo/logit": 0.26591506600379944,
|
|
"ronpo/residual": 0.2671971917152405,
|
|
"ronpo/residual_abs": 0.31767672300338745,
|
|
"ronpo/target": -0.0012821194250136614,
|
|
"ronpo/target_abs": 0.08636602014303207,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 105
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2426718771457672,
|
|
"drift/rejected_abs": 0.12293334305286407,
|
|
"epoch": 0.03496225665474772,
|
|
"grad_norm": 8.375,
|
|
"learning_rate": 2.4966074958862813e-08,
|
|
"logits/chosen": -2.245562791824341,
|
|
"logits/rejected": -2.205705165863037,
|
|
"logps/chosen": -0.29507607221603394,
|
|
"logps/rejected": -0.2873607277870178,
|
|
"loss": 0.12289254367351532,
|
|
"loss/core": 0.1113683208823204,
|
|
"loss/preference_sft": 0.29041385650634766,
|
|
"loss/reference_anchor": 0.20144303143024445,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.424837589263916,
|
|
"rewards/margins": 1.2106255292892456,
|
|
"rewards/rejected": 1.21421217918396,
|
|
"ronpo/logit": 0.12106256186962128,
|
|
"ronpo/residual": 0.11163101345300674,
|
|
"ronpo/residual_abs": 0.1918853521347046,
|
|
"ronpo/target": 0.009431545622646809,
|
|
"ronpo/target_abs": 0.10175038874149323,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 110
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27447623014450073,
|
|
"drift/rejected_abs": 0.13834591209888458,
|
|
"epoch": 0.03655145013905443,
|
|
"grad_norm": 1.7265625,
|
|
"learning_rate": 2.4945884883122553e-08,
|
|
"logits/chosen": -1.8843482732772827,
|
|
"logits/rejected": -1.775320053100586,
|
|
"logps/chosen": -0.2798824906349182,
|
|
"logps/rejected": -0.2814459502696991,
|
|
"loss": 0.1541265845298767,
|
|
"loss/core": 0.14217844605445862,
|
|
"loss/preference_sft": 0.27655261754989624,
|
|
"loss/reference_anchor": 0.21130748093128204,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.742548942565918,
|
|
"rewards/margins": 1.366241693496704,
|
|
"rewards/rejected": 1.376307725906372,
|
|
"ronpo/logit": 0.13662415742874146,
|
|
"ronpo/residual": 0.14770525693893433,
|
|
"ronpo/residual_abs": 0.18340587615966797,
|
|
"ronpo/target": -0.011081096716225147,
|
|
"ronpo/target_abs": 0.07495449483394623,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 115
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25327441096305847,
|
|
"drift/rejected_abs": 0.11703983694314957,
|
|
"epoch": 0.03814064362336114,
|
|
"grad_norm": 0.9609375,
|
|
"learning_rate": 2.492101441907714e-08,
|
|
"logits/chosen": -2.0191433429718018,
|
|
"logits/rejected": -2.067561626434326,
|
|
"logps/chosen": -0.30190160870552063,
|
|
"logps/rejected": -0.30017122626304626,
|
|
"loss": 0.30577465891838074,
|
|
"loss/core": 0.28840842843055725,
|
|
"loss/preference_sft": 0.2985996901988983,
|
|
"loss/reference_anchor": 0.31746405363082886,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.5322983264923096,
|
|
"rewards/margins": 1.3638817071914673,
|
|
"rewards/rejected": 1.1684163808822632,
|
|
"ronpo/logit": 0.1363881528377533,
|
|
"ronpo/residual": 0.11740622669458389,
|
|
"ronpo/residual_abs": 0.22061511874198914,
|
|
"ronpo/target": 0.0189819298684597,
|
|
"ronpo/target_abs": 0.08912766724824905,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 120
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.12704667448997498,
|
|
"drift/rejected_abs": 0.07079822570085526,
|
|
"epoch": 0.03972983710766786,
|
|
"grad_norm": 1.2265625,
|
|
"learning_rate": 2.4891472919490977e-08,
|
|
"logits/chosen": -2.272683620452881,
|
|
"logits/rejected": -2.193810224533081,
|
|
"logps/chosen": -0.2744141221046448,
|
|
"logps/rejected": -0.2814023196697235,
|
|
"loss": 0.04335567355155945,
|
|
"loss/core": 0.03851154074072838,
|
|
"loss/preference_sft": 0.26887914538383484,
|
|
"loss/reference_anchor": 0.06999474763870239,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.2701770067214966,
|
|
"rewards/margins": 0.5637924671173096,
|
|
"rewards/rejected": 0.7063844799995422,
|
|
"ronpo/logit": 0.05637924745678902,
|
|
"ronpo/residual": 0.0646718442440033,
|
|
"ronpo/residual_abs": 0.11568011343479156,
|
|
"ronpo/target": -0.008292593993246555,
|
|
"ronpo/target_abs": 0.08368505537509918,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 125
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3699837327003479,
|
|
"drift/rejected_abs": 0.1531117707490921,
|
|
"epoch": 0.04131903059197457,
|
|
"grad_norm": 15.5,
|
|
"learning_rate": 2.4857271493713895e-08,
|
|
"logits/chosen": -2.0499277114868164,
|
|
"logits/rejected": -1.9798452854156494,
|
|
"logps/chosen": -0.2802344560623169,
|
|
"logps/rejected": -0.3006633222103119,
|
|
"loss": 0.4787680506706238,
|
|
"loss/core": 0.4418778419494629,
|
|
"loss/preference_sft": 0.2814800441265106,
|
|
"loss/reference_anchor": 0.7096576690673828,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 3.6996917724609375,
|
|
"rewards/margins": 2.1705682277679443,
|
|
"rewards/rejected": 1.529123306274414,
|
|
"ronpo/logit": 0.21705684065818787,
|
|
"ronpo/residual": 0.2116149216890335,
|
|
"ronpo/residual_abs": 0.26738792657852173,
|
|
"ronpo/target": 0.005441917572170496,
|
|
"ronpo/target_abs": 0.07667852938175201,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 130
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2286110669374466,
|
|
"drift/rejected_abs": 0.17728818953037262,
|
|
"epoch": 0.04290822407628129,
|
|
"grad_norm": 99.5,
|
|
"learning_rate": 2.481842300350339e-08,
|
|
"logits/chosen": -2.156745195388794,
|
|
"logits/rejected": -2.1426777839660645,
|
|
"logps/chosen": -0.2545807361602783,
|
|
"logps/rejected": -0.2720378339290619,
|
|
"loss": 0.4073014259338379,
|
|
"loss/core": 0.39105960726737976,
|
|
"loss/preference_sft": 0.26137062907218933,
|
|
"loss/reference_anchor": 0.2986990809440613,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.285310745239258,
|
|
"rewards/margins": 0.5148512125015259,
|
|
"rewards/rejected": 1.770459532737732,
|
|
"ronpo/logit": 0.05148511379957199,
|
|
"ronpo/residual": 0.06029368191957474,
|
|
"ronpo/residual_abs": 0.22104015946388245,
|
|
"ronpo/target": -0.00880857277661562,
|
|
"ronpo/target_abs": 0.08236870914697647,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 135
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21387775242328644,
|
|
"drift/rejected_abs": 0.1563216596841812,
|
|
"epoch": 0.044497417560588004,
|
|
"grad_norm": 41.75,
|
|
"learning_rate": 2.4774942058187855e-08,
|
|
"logits/chosen": -2.0665993690490723,
|
|
"logits/rejected": -2.035794734954834,
|
|
"logps/chosen": -0.28517797589302063,
|
|
"logps/rejected": -0.28532570600509644,
|
|
"loss": 0.26362624764442444,
|
|
"loss/core": 0.2508805990219116,
|
|
"loss/preference_sft": 0.28584787249565125,
|
|
"loss/reference_anchor": 0.22632861137390137,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.1381399631500244,
|
|
"rewards/margins": 0.5762137174606323,
|
|
"rewards/rejected": 1.561926245689392,
|
|
"ronpo/logit": 0.05762135982513428,
|
|
"ronpo/residual": 0.059470035135746,
|
|
"ronpo/residual_abs": 0.2042684555053711,
|
|
"ronpo/target": -0.0018486687913537025,
|
|
"ronpo/target_abs": 0.08461553603410721,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 140
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3112896680831909,
|
|
"drift/rejected_abs": 0.13714732229709625,
|
|
"epoch": 0.04608661104489471,
|
|
"grad_norm": 226.0,
|
|
"learning_rate": 2.472684500917257e-08,
|
|
"logits/chosen": -2.104536771774292,
|
|
"logits/rejected": -2.14062237739563,
|
|
"logps/chosen": -0.2743372917175293,
|
|
"logps/rejected": -0.2655439078807831,
|
|
"loss": 0.2187369167804718,
|
|
"loss/core": 0.1985674947500229,
|
|
"loss/preference_sft": 0.2756105661392212,
|
|
"loss/reference_anchor": 0.3758276104927063,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 3.1124117374420166,
|
|
"rewards/margins": 1.7423970699310303,
|
|
"rewards/rejected": 1.3700147867202759,
|
|
"ronpo/logit": 0.17423972487449646,
|
|
"ronpo/residual": 0.18899571895599365,
|
|
"ronpo/residual_abs": 0.2230168879032135,
|
|
"ronpo/target": -0.014756004326045513,
|
|
"ronpo/target_abs": 0.08157791197299957,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 145
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4021173417568207,
|
|
"drift/rejected_abs": 0.11387000232934952,
|
|
"epoch": 0.04767580452920143,
|
|
"grad_norm": 2.390625,
|
|
"learning_rate": 2.467414994379065e-08,
|
|
"logits/chosen": -2.0298092365264893,
|
|
"logits/rejected": -2.1323561668395996,
|
|
"logps/chosen": -0.29558226466178894,
|
|
"logps/rejected": -0.27471041679382324,
|
|
"loss": 1.039489507675171,
|
|
"loss/core": 1.0037477016448975,
|
|
"loss/preference_sft": 0.286141037940979,
|
|
"loss/reference_anchor": 0.6862256526947021,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 4.021132469177246,
|
|
"rewards/margins": 2.884185314178467,
|
|
"rewards/rejected": 1.136947751045227,
|
|
"ronpo/logit": 0.28841856122016907,
|
|
"ronpo/residual": 0.2740480601787567,
|
|
"ronpo/residual_abs": 0.33974286913871765,
|
|
"ronpo/target": 0.014370448887348175,
|
|
"ronpo/target_abs": 0.08442968875169754,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 150
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3120030462741852,
|
|
"drift/rejected_abs": 0.10412098467350006,
|
|
"epoch": 0.04926499801350814,
|
|
"grad_norm": 3.84375,
|
|
"learning_rate": 2.4616876678501113e-08,
|
|
"logits/chosen": -2.170553207397461,
|
|
"logits/rejected": -2.1755318641662598,
|
|
"logps/chosen": -0.265367329120636,
|
|
"logps/rejected": -0.26216381788253784,
|
|
"loss": 0.7199813723564148,
|
|
"loss/core": 0.6925104260444641,
|
|
"loss/preference_sft": 0.2665371000766754,
|
|
"loss/reference_anchor": 0.522765040397644,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.119339942932129,
|
|
"rewards/margins": 2.079453229904175,
|
|
"rewards/rejected": 1.0398868322372437,
|
|
"ronpo/logit": 0.207945317029953,
|
|
"ronpo/residual": 0.22640779614448547,
|
|
"ronpo/residual_abs": 0.26298147439956665,
|
|
"ronpo/target": -0.018462475389242172,
|
|
"ronpo/target_abs": 0.07875766605138779,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 155
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1994825154542923,
|
|
"drift/rejected_abs": 0.06770407408475876,
|
|
"epoch": 0.05085419149781486,
|
|
"grad_norm": 856.0,
|
|
"learning_rate": 2.4555046751436738e-08,
|
|
"logits/chosen": -2.207547426223755,
|
|
"logits/rejected": -2.225343942642212,
|
|
"logps/chosen": -0.32042816281318665,
|
|
"logps/rejected": -0.2738822400569916,
|
|
"loss": 0.559381365776062,
|
|
"loss/core": 0.5423927903175354,
|
|
"loss/preference_sft": 0.2767319679260254,
|
|
"loss/reference_anchor": 0.3120982050895691,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.9948251247406006,
|
|
"rewards/margins": 1.318454384803772,
|
|
"rewards/rejected": 0.6763707399368286,
|
|
"ronpo/logit": 0.13184544444084167,
|
|
"ronpo/residual": 0.1333637535572052,
|
|
"ronpo/residual_abs": 0.19957098364830017,
|
|
"ronpo/target": -0.0015183339128270745,
|
|
"ronpo/target_abs": 0.09095166623592377,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 160
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.13100005686283112,
|
|
"drift/rejected_abs": 0.05130317807197571,
|
|
"epoch": 0.052443384982121574,
|
|
"grad_norm": 4.15625,
|
|
"learning_rate": 2.4488683414304425e-08,
|
|
"logits/chosen": -2.1318647861480713,
|
|
"logits/rejected": -2.010211229324341,
|
|
"logps/chosen": -0.28451108932495117,
|
|
"logps/rejected": -0.29346317052841187,
|
|
"loss": 0.1088658794760704,
|
|
"loss/core": 0.10401229560375214,
|
|
"loss/preference_sft": 0.2892284393310547,
|
|
"loss/reference_anchor": 0.06814897060394287,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.3097032308578491,
|
|
"rewards/margins": 0.8015567660331726,
|
|
"rewards/rejected": 0.5081464052200317,
|
|
"ronpo/logit": 0.08015568554401398,
|
|
"ronpo/residual": 0.07226266711950302,
|
|
"ronpo/residual_abs": 0.1403239220380783,
|
|
"ronpo/target": 0.007893010973930359,
|
|
"ronpo/target_abs": 0.07677415758371353,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 165
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.317328542470932,
|
|
"drift/rejected_abs": 0.16766056418418884,
|
|
"epoch": 0.05403257846642829,
|
|
"grad_norm": 1.078125,
|
|
"learning_rate": 2.4417811623641206e-08,
|
|
"logits/chosen": -2.027057647705078,
|
|
"logits/rejected": -2.019274950027466,
|
|
"logps/chosen": -0.2885628640651703,
|
|
"logps/rejected": -0.2752649486064911,
|
|
"loss": 0.4365611672401428,
|
|
"loss/core": 0.4138968586921692,
|
|
"loss/preference_sft": 0.28210192918777466,
|
|
"loss/reference_anchor": 0.4250755310058594,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.172837734222412,
|
|
"rewards/margins": 1.4988738298416138,
|
|
"rewards/rejected": 1.6739639043807983,
|
|
"ronpo/logit": 0.14988741278648376,
|
|
"ronpo/residual": 0.13528050482273102,
|
|
"ronpo/residual_abs": 0.2508905827999115,
|
|
"ronpo/target": 0.014606885612010956,
|
|
"ronpo/target_abs": 0.07879899442195892,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 170
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19914527237415314,
|
|
"drift/rejected_abs": 0.11187348514795303,
|
|
"epoch": 0.055621771950735005,
|
|
"grad_norm": 30.125,
|
|
"learning_rate": 2.4342458031429115e-08,
|
|
"logits/chosen": -2.262404680252075,
|
|
"logits/rejected": -2.1995086669921875,
|
|
"logps/chosen": -0.2932138741016388,
|
|
"logps/rejected": -0.30438342690467834,
|
|
"loss": 0.06648578494787216,
|
|
"loss/core": 0.05691850930452347,
|
|
"loss/preference_sft": 0.29610422253608704,
|
|
"loss/reference_anchor": 0.16173511743545532,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.9914528131484985,
|
|
"rewards/margins": 0.8727973103523254,
|
|
"rewards/rejected": 1.1186553239822388,
|
|
"ronpo/logit": 0.08727972954511642,
|
|
"ronpo/residual": 0.0997995063662529,
|
|
"ronpo/residual_abs": 0.1461952179670334,
|
|
"ronpo/target": -0.012519779615104198,
|
|
"ronpo/target_abs": 0.08836988359689713,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 175
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4732588231563568,
|
|
"drift/rejected_abs": 0.22694897651672363,
|
|
"epoch": 0.057210965435041714,
|
|
"grad_norm": 37.25,
|
|
"learning_rate": 2.4262650975072444e-08,
|
|
"logits/chosen": -2.074978828430176,
|
|
"logits/rejected": -2.051633358001709,
|
|
"logps/chosen": -0.2896614074707031,
|
|
"logps/rejected": -0.28108644485473633,
|
|
"loss": 0.9334848523139954,
|
|
"loss/core": 0.8751218914985657,
|
|
"loss/preference_sft": 0.2792913615703583,
|
|
"loss/reference_anchor": 1.1393296718597412,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 4.732588291168213,
|
|
"rewards/margins": 2.463618755340576,
|
|
"rewards/rejected": 2.2689692974090576,
|
|
"ronpo/logit": 0.24636192619800568,
|
|
"ronpo/residual": 0.2580645680427551,
|
|
"ronpo/residual_abs": 0.31068843603134155,
|
|
"ronpo/target": -0.011702699586749077,
|
|
"ronpo/target_abs": 0.07820998132228851,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 180
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.12275053560733795,
|
|
"drift/rejected_abs": 0.06439492851495743,
|
|
"epoch": 0.05880015891934843,
|
|
"grad_norm": 5.625,
|
|
"learning_rate": 2.4178420466741218e-08,
|
|
"logits/chosen": -2.226714849472046,
|
|
"logits/rejected": -2.3291454315185547,
|
|
"logps/chosen": -0.29459166526794434,
|
|
"logps/rejected": -0.29199695587158203,
|
|
"loss": 0.03509552404284477,
|
|
"loss/core": 0.03178117424249649,
|
|
"loss/preference_sft": 0.2933792173862457,
|
|
"loss/reference_anchor": 0.036949075758457184,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.227303385734558,
|
|
"rewards/margins": 0.5856597423553467,
|
|
"rewards/rejected": 0.6416436433792114,
|
|
"ronpo/logit": 0.058565981686115265,
|
|
"ronpo/residual": 0.07432404160499573,
|
|
"ronpo/residual_abs": 0.13376492261886597,
|
|
"ronpo/target": -0.01575806364417076,
|
|
"ronpo/target_abs": 0.10001115500926971,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 185
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.12490232288837433,
|
|
"drift/rejected_abs": 0.055704616010189056,
|
|
"epoch": 0.060389352403655144,
|
|
"grad_norm": 0.9296875,
|
|
"learning_rate": 2.408979818208484e-08,
|
|
"logits/chosen": -2.3149032592773438,
|
|
"logits/rejected": -2.411414861679077,
|
|
"logps/chosen": -0.30525779724121094,
|
|
"logps/rejected": -0.3026205003261566,
|
|
"loss": 0.052305638790130615,
|
|
"loss/core": 0.04764842614531517,
|
|
"loss/preference_sft": 0.3053823411464691,
|
|
"loss/reference_anchor": 0.06260595470666885,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.247715950012207,
|
|
"rewards/margins": 0.6911423802375793,
|
|
"rewards/rejected": 0.5565735697746277,
|
|
"ronpo/logit": 0.06911423802375793,
|
|
"ronpo/residual": 0.0638674646615982,
|
|
"ronpo/residual_abs": 0.12851160764694214,
|
|
"ronpo/target": 0.005246774293482304,
|
|
"ronpo/target_abs": 0.08141742646694183,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 190
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21187469363212585,
|
|
"drift/rejected_abs": 0.11845757067203522,
|
|
"epoch": 0.06197854588796186,
|
|
"grad_norm": 6.75,
|
|
"learning_rate": 2.3996817448320193e-08,
|
|
"logits/chosen": -2.055734872817993,
|
|
"logits/rejected": -2.031132698059082,
|
|
"logps/chosen": -0.27869275212287903,
|
|
"logps/rejected": -0.27192652225494385,
|
|
"loss": 0.06286276876926422,
|
|
"loss/core": 0.055714428424835205,
|
|
"loss/preference_sft": 0.26679545640945435,
|
|
"loss/reference_anchor": 0.11628717184066772,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.1181392669677734,
|
|
"rewards/margins": 0.9341791868209839,
|
|
"rewards/rejected": 1.1839600801467896,
|
|
"ronpo/logit": 0.0934179276227951,
|
|
"ronpo/residual": 0.09014447033405304,
|
|
"ronpo/residual_abs": 0.14351728558540344,
|
|
"ronpo/target": 0.0032734572887420654,
|
|
"ronpo/target_abs": 0.08329368382692337,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 195
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2836345136165619,
|
|
"drift/rejected_abs": 0.08946871757507324,
|
|
"epoch": 0.06356773937226858,
|
|
"grad_norm": 8.1875,
|
|
"learning_rate": 2.389951323169861e-08,
|
|
"logits/chosen": -2.040783166885376,
|
|
"logits/rejected": -2.161843776702881,
|
|
"logps/chosen": -0.2890108823776245,
|
|
"logps/rejected": -0.2789885699748993,
|
|
"loss": 0.4890855848789215,
|
|
"loss/core": 0.4716506600379944,
|
|
"loss/preference_sft": 0.28799134492874146,
|
|
"loss/reference_anchor": 0.31989890336990356,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.835172176361084,
|
|
"rewards/margins": 1.9416320323944092,
|
|
"rewards/rejected": 0.8935402035713196,
|
|
"ronpo/logit": 0.1941632330417633,
|
|
"ronpo/residual": 0.18651123344898224,
|
|
"ronpo/residual_abs": 0.23668105900287628,
|
|
"ronpo/target": 0.007652005646377802,
|
|
"ronpo/target_abs": 0.07496481388807297,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 200
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2938988208770752,
|
|
"drift/rejected_abs": 0.10776060819625854,
|
|
"epoch": 0.06515693285657528,
|
|
"grad_norm": 94.0,
|
|
"learning_rate": 2.3797922124356508e-08,
|
|
"logits/chosen": -2.003150463104248,
|
|
"logits/rejected": -2.0846657752990723,
|
|
"logps/chosen": -0.28210049867630005,
|
|
"logps/rejected": -0.28749504685401917,
|
|
"loss": 0.755161464214325,
|
|
"loss/core": 0.7267810106277466,
|
|
"loss/preference_sft": 0.2828027009963989,
|
|
"loss/reference_anchor": 0.5393306016921997,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.935922861099243,
|
|
"rewards/margins": 1.859521508216858,
|
|
"rewards/rejected": 1.0764009952545166,
|
|
"ronpo/logit": 0.18595215678215027,
|
|
"ronpo/residual": 0.21726493537425995,
|
|
"ronpo/residual_abs": 0.2416853904724121,
|
|
"ronpo/target": -0.03131275624036789,
|
|
"ronpo/target_abs": 0.07854778319597244,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 205
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21909186244010925,
|
|
"drift/rejected_abs": 0.05819239094853401,
|
|
"epoch": 0.066746126340882,
|
|
"grad_norm": 2.984375,
|
|
"learning_rate": 2.3692082330554585e-08,
|
|
"logits/chosen": -2.2509193420410156,
|
|
"logits/rejected": -2.169978141784668,
|
|
"logps/chosen": -0.307937890291214,
|
|
"logps/rejected": -0.31472259759902954,
|
|
"loss": 1.0144634246826172,
|
|
"loss/core": 0.9857050776481628,
|
|
"loss/preference_sft": 0.31031107902526855,
|
|
"loss/reference_anchor": 0.544138491153717,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.1909189224243164,
|
|
"rewards/margins": 1.6100225448608398,
|
|
"rewards/rejected": 0.5808963179588318,
|
|
"ronpo/logit": 0.1610022485256195,
|
|
"ronpo/residual": 0.17502841353416443,
|
|
"ronpo/residual_abs": 0.22845324873924255,
|
|
"ronpo/target": -0.01402618270367384,
|
|
"ronpo/target_abs": 0.08586090058088303,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 210
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.32701677083969116,
|
|
"drift/rejected_abs": 0.16467604041099548,
|
|
"epoch": 0.06833531982518871,
|
|
"grad_norm": 1.0234375,
|
|
"learning_rate": 2.3582033652310766e-08,
|
|
"logits/chosen": -2.1693928241729736,
|
|
"logits/rejected": -2.157107353210449,
|
|
"logps/chosen": -0.26543909311294556,
|
|
"logps/rejected": -0.2721554636955261,
|
|
"loss": 0.30226364731788635,
|
|
"loss/core": 0.27254778146743774,
|
|
"loss/preference_sft": 0.2698844075202942,
|
|
"loss/reference_anchor": 0.5673290491104126,
|
|
"rewards/accuracies": 0.9750000238418579,
|
|
"rewards/chosen": 3.270167827606201,
|
|
"rewards/margins": 1.624214768409729,
|
|
"rewards/rejected": 1.6459531784057617,
|
|
"ronpo/logit": 0.16242146492004395,
|
|
"ronpo/residual": 0.15607237815856934,
|
|
"ronpo/residual_abs": 0.21735577285289764,
|
|
"ronpo/target": 0.006349098868668079,
|
|
"ronpo/target_abs": 0.0811850056052208,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 215
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1198434829711914,
|
|
"drift/rejected_abs": 0.08144821226596832,
|
|
"epoch": 0.06992451330949544,
|
|
"grad_norm": 0.71875,
|
|
"learning_rate": 2.3467817474432273e-08,
|
|
"logits/chosen": -2.216386079788208,
|
|
"logits/rejected": -2.084725856781006,
|
|
"logps/chosen": -0.30442219972610474,
|
|
"logps/rejected": -0.29973161220550537,
|
|
"loss": 0.05327620357275009,
|
|
"loss/core": 0.04937787353992462,
|
|
"loss/preference_sft": 0.3065362572669983,
|
|
"loss/reference_anchor": 0.04731312766671181,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.1957999467849731,
|
|
"rewards/margins": 0.38699817657470703,
|
|
"rewards/rejected": 0.8088018298149109,
|
|
"ronpo/logit": 0.038699813187122345,
|
|
"ronpo/residual": 0.05509736388921738,
|
|
"ronpo/residual_abs": 0.13401086628437042,
|
|
"ronpo/target": -0.016397548839449883,
|
|
"ronpo/target_abs": 0.08902524411678314,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 220
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23227110505104065,
|
|
"drift/rejected_abs": 0.10326921939849854,
|
|
"epoch": 0.07151370679380215,
|
|
"grad_norm": 6.21875,
|
|
"learning_rate": 2.3349476748952507e-08,
|
|
"logits/chosen": -2.1769394874572754,
|
|
"logits/rejected": -2.194044351577759,
|
|
"logps/chosen": -0.28651732206344604,
|
|
"logps/rejected": -0.28518351912498474,
|
|
"loss": 0.1896631419658661,
|
|
"loss/core": 0.17952075600624084,
|
|
"loss/preference_sft": 0.27889543771743774,
|
|
"loss/reference_anchor": 0.1749579906463623,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.32271146774292,
|
|
"rewards/margins": 1.2900190353393555,
|
|
"rewards/rejected": 1.0326921939849854,
|
|
"ronpo/logit": 0.1290019154548645,
|
|
"ronpo/residual": 0.11321617662906647,
|
|
"ronpo/residual_abs": 0.17566923797130585,
|
|
"ronpo/target": 0.015785735100507736,
|
|
"ronpo/target_abs": 0.07616916298866272,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 225
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.26629695296287537,
|
|
"drift/rejected_abs": 0.09605975449085236,
|
|
"epoch": 0.07310290027810885,
|
|
"grad_norm": 2.546875,
|
|
"learning_rate": 2.3227055978978543e-08,
|
|
"logits/chosen": -2.242928981781006,
|
|
"logits/rejected": -2.2605459690093994,
|
|
"logps/chosen": -0.26110613346099854,
|
|
"logps/rejected": -0.2610822319984436,
|
|
"loss": 0.419999897480011,
|
|
"loss/core": 0.40420547127723694,
|
|
"loss/preference_sft": 0.2564738988876343,
|
|
"loss/reference_anchor": 0.2902412414550781,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.662400960922241,
|
|
"rewards/margins": 1.7032798528671265,
|
|
"rewards/rejected": 0.9591208696365356,
|
|
"ronpo/logit": 0.17032800614833832,
|
|
"ronpo/residual": 0.1783141791820526,
|
|
"ronpo/residual_abs": 0.22243869304656982,
|
|
"ronpo/target": -0.00798617023974657,
|
|
"ronpo/target_abs": 0.09001145511865616,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 230
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.46215638518333435,
|
|
"drift/rejected_abs": 0.23638899624347687,
|
|
"epoch": 0.07469209376241558,
|
|
"grad_norm": 50.5,
|
|
"learning_rate": 2.310060120195532e-08,
|
|
"logits/chosen": -2.113866090774536,
|
|
"logits/rejected": -2.116377115249634,
|
|
"logps/chosen": -0.2798251509666443,
|
|
"logps/rejected": -0.2868059575557709,
|
|
"loss": 1.1930959224700928,
|
|
"loss/core": 1.1315306425094604,
|
|
"loss/preference_sft": 0.27541080117225647,
|
|
"loss/reference_anchor": 1.20376455783844,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.620245933532715,
|
|
"rewards/margins": 2.3461127281188965,
|
|
"rewards/rejected": 2.2741329669952393,
|
|
"ronpo/logit": 0.23461131751537323,
|
|
"ronpo/residual": 0.237228661775589,
|
|
"ronpo/residual_abs": 0.40296974778175354,
|
|
"ronpo/target": -0.0026173379737883806,
|
|
"ronpo/target_abs": 0.08310168236494064,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 235
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2902994751930237,
|
|
"drift/rejected_abs": 0.16377928853034973,
|
|
"epoch": 0.07628128724672228,
|
|
"grad_norm": 204.0,
|
|
"learning_rate": 2.2970159972352866e-08,
|
|
"logits/chosen": -2.1415343284606934,
|
|
"logits/rejected": -2.2179665565490723,
|
|
"logps/chosen": -0.2673754096031189,
|
|
"logps/rejected": -0.2882601320743561,
|
|
"loss": 0.22219261527061462,
|
|
"loss/core": 0.19527706503868103,
|
|
"loss/preference_sft": 0.28320834040641785,
|
|
"loss/reference_anchor": 0.5099908113479614,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.9026601314544678,
|
|
"rewards/margins": 1.265976905822754,
|
|
"rewards/rejected": 1.636683464050293,
|
|
"ronpo/logit": 0.12659768760204315,
|
|
"ronpo/residual": 0.11025957018136978,
|
|
"ronpo/residual_abs": 0.18036124110221863,
|
|
"ronpo/target": 0.016338128596544266,
|
|
"ronpo/target_abs": 0.087813600897789,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 240
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18804895877838135,
|
|
"drift/rejected_abs": 0.07305173575878143,
|
|
"epoch": 0.07787048073102901,
|
|
"grad_norm": 2.1875,
|
|
"learning_rate": 2.2835781343782966e-08,
|
|
"logits/chosen": -2.182067394256592,
|
|
"logits/rejected": -2.2002573013305664,
|
|
"logps/chosen": -0.29913002252578735,
|
|
"logps/rejected": -0.3051024079322815,
|
|
"loss": 0.2962071895599365,
|
|
"loss/core": 0.28634169697761536,
|
|
"loss/preference_sft": 0.3018152117729187,
|
|
"loss/reference_anchor": 0.1671280860900879,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.8793338537216187,
|
|
"rewards/margins": 1.174026370048523,
|
|
"rewards/rejected": 0.7053073644638062,
|
|
"ronpo/logit": 0.11740265041589737,
|
|
"ronpo/residual": 0.12060768902301788,
|
|
"ronpo/residual_abs": 0.1809166967868805,
|
|
"ronpo/target": -0.0032050427980720997,
|
|
"ronpo/target_abs": 0.09324783831834793,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 245
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.362274706363678,
|
|
"drift/rejected_abs": 0.08525294065475464,
|
|
"epoch": 0.07945967421533572,
|
|
"grad_norm": 7.25,
|
|
"learning_rate": 2.269751585055215e-08,
|
|
"logits/chosen": -2.1129937171936035,
|
|
"logits/rejected": -2.090263843536377,
|
|
"logps/chosen": -0.3094167709350586,
|
|
"logps/rejected": -0.31022873520851135,
|
|
"loss": 1.225620985031128,
|
|
"loss/core": 1.189970850944519,
|
|
"loss/preference_sft": 0.3118210434913635,
|
|
"loss/reference_anchor": 0.6818209886550903,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 3.6219310760498047,
|
|
"rewards/margins": 2.770771026611328,
|
|
"rewards/rejected": 0.851159930229187,
|
|
"ronpo/logit": 0.2770771086215973,
|
|
"ronpo/residual": 0.2786293029785156,
|
|
"ronpo/residual_abs": 0.33625951409339905,
|
|
"ronpo/target": -0.0015522129833698273,
|
|
"ronpo/target_abs": 0.08379620313644409,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 250
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4456920623779297,
|
|
"drift/rejected_abs": 0.19766589999198914,
|
|
"epoch": 0.08104886769964244,
|
|
"grad_norm": 7.0625,
|
|
"learning_rate": 2.2555415488657776e-08,
|
|
"logits/chosen": -2.1046249866485596,
|
|
"logits/rejected": -2.107532262802124,
|
|
"logps/chosen": -0.2630499303340912,
|
|
"logps/rejected": -0.27011799812316895,
|
|
"loss": 0.7377499341964722,
|
|
"loss/core": 0.6888483166694641,
|
|
"loss/preference_sft": 0.27254000306129456,
|
|
"loss/reference_anchor": 0.9507792592048645,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 4.456816673278809,
|
|
"rewards/margins": 2.533341407775879,
|
|
"rewards/rejected": 1.9234750270843506,
|
|
"ronpo/logit": 0.2533341646194458,
|
|
"ronpo/residual": 0.25843900442123413,
|
|
"ronpo/residual_abs": 0.31028133630752563,
|
|
"ronpo/target": -0.005104861222207546,
|
|
"ronpo/target_abs": 0.08672384172677994,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 255
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.32755011320114136,
|
|
"drift/rejected_abs": 0.08800020813941956,
|
|
"epoch": 0.08263806118394915,
|
|
"grad_norm": 972.0,
|
|
"learning_rate": 2.240953369623447e-08,
|
|
"logits/chosen": -2.193181276321411,
|
|
"logits/rejected": -2.168120861053467,
|
|
"logps/chosen": -0.2840411067008972,
|
|
"logps/rejected": -0.28209537267684937,
|
|
"loss": 0.8339856266975403,
|
|
"loss/core": 0.8076542615890503,
|
|
"loss/preference_sft": 0.29532259702682495,
|
|
"loss/reference_anchor": 0.4970928728580475,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.2736563682556152,
|
|
"rewards/margins": 2.4421114921569824,
|
|
"rewards/rejected": 0.8315451741218567,
|
|
"ronpo/logit": 0.24421115219593048,
|
|
"ronpo/residual": 0.2369440346956253,
|
|
"ronpo/residual_abs": 0.29261597990989685,
|
|
"ronpo/target": 0.007267133332788944,
|
|
"ronpo/target_abs": 0.08349494636058807,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 260
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18214836716651917,
|
|
"drift/rejected_abs": 0.0814029648900032,
|
|
"epoch": 0.08422725466825585,
|
|
"grad_norm": 16.0,
|
|
"learning_rate": 2.225992533345824e-08,
|
|
"logits/chosen": -2.155879497528076,
|
|
"logits/rejected": -2.1550204753875732,
|
|
"logps/chosen": -0.284726619720459,
|
|
"logps/rejected": -0.29086941480636597,
|
|
"loss": 0.08231814950704575,
|
|
"loss/core": 0.07635300606489182,
|
|
"loss/preference_sft": 0.2914847135543823,
|
|
"loss/reference_anchor": 0.0901542603969574,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.8214836120605469,
|
|
"rewards/margins": 1.0148638486862183,
|
|
"rewards/rejected": 0.8066196441650391,
|
|
"ronpo/logit": 0.10148639976978302,
|
|
"ronpo/residual": 0.09669841825962067,
|
|
"ronpo/residual_abs": 0.16233278810977936,
|
|
"ronpo/target": 0.004787982441484928,
|
|
"ronpo/target_abs": 0.09343020617961884,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 265
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2922074496746063,
|
|
"drift/rejected_abs": 0.16019122302532196,
|
|
"epoch": 0.08581644815256258,
|
|
"grad_norm": 2.734375,
|
|
"learning_rate": 2.210664666191579e-08,
|
|
"logits/chosen": -2.2221102714538574,
|
|
"logits/rejected": -2.1809840202331543,
|
|
"logps/chosen": -0.2800453305244446,
|
|
"logps/rejected": -0.28999465703964233,
|
|
"loss": 0.357001394033432,
|
|
"loss/core": 0.3204185366630554,
|
|
"loss/preference_sft": 0.28067368268966675,
|
|
"loss/reference_anchor": 0.7035894393920898,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.9200611114501953,
|
|
"rewards/margins": 1.3199583292007446,
|
|
"rewards/rejected": 1.6001027822494507,
|
|
"ronpo/logit": 0.13199582695960999,
|
|
"ronpo/residual": 0.11801192909479141,
|
|
"ronpo/residual_abs": 0.18810518085956573,
|
|
"ronpo/target": 0.013983900658786297,
|
|
"ronpo/target_abs": 0.08693692088127136,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 270
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17948992550373077,
|
|
"drift/rejected_abs": 0.10400255769491196,
|
|
"epoch": 0.08740564163686929,
|
|
"grad_norm": 6.875,
|
|
"learning_rate": 2.1949755323446846e-08,
|
|
"logits/chosen": -2.275407314300537,
|
|
"logits/rejected": -2.3497166633605957,
|
|
"logps/chosen": -0.27847951650619507,
|
|
"logps/rejected": -0.2680722773075104,
|
|
"loss": 0.048314549028873444,
|
|
"loss/core": 0.041888050734996796,
|
|
"loss/preference_sft": 0.27859556674957275,
|
|
"loss/reference_anchor": 0.1006704568862915,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.794899582862854,
|
|
"rewards/margins": 0.7595475912094116,
|
|
"rewards/rejected": 1.0353518724441528,
|
|
"ronpo/logit": 0.07595475763082504,
|
|
"ronpo/residual": 0.05474218726158142,
|
|
"ronpo/residual_abs": 0.13210758566856384,
|
|
"ronpo/target": 0.02121257223188877,
|
|
"ronpo/target_abs": 0.0803799256682396,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 275
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1700095236301422,
|
|
"drift/rejected_abs": 0.11016277223825455,
|
|
"epoch": 0.08899483512117601,
|
|
"grad_norm": 52.0,
|
|
"learning_rate": 2.1789310318467428e-08,
|
|
"logits/chosen": -2.180828094482422,
|
|
"logits/rejected": -2.182821750640869,
|
|
"logps/chosen": -0.2840616703033447,
|
|
"logps/rejected": -0.2911655306816101,
|
|
"loss": 0.10237391293048859,
|
|
"loss/core": 0.09611751139163971,
|
|
"loss/preference_sft": 0.28340545296669006,
|
|
"loss/reference_anchor": 0.09678752720355988,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.6990578174591064,
|
|
"rewards/margins": 0.5979472398757935,
|
|
"rewards/rejected": 1.1011104583740234,
|
|
"ronpo/logit": 0.059794723987579346,
|
|
"ronpo/residual": 0.052797090262174606,
|
|
"ronpo/residual_abs": 0.1553640067577362,
|
|
"ronpo/target": 0.006997637450695038,
|
|
"ronpo/target_abs": 0.07295690476894379,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 280
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.43757256865501404,
|
|
"drift/rejected_abs": 0.14037099480628967,
|
|
"epoch": 0.09058402860548272,
|
|
"grad_norm": 10.9375,
|
|
"learning_rate": 2.162537198378218e-08,
|
|
"logits/chosen": -2.2235183715820312,
|
|
"logits/rejected": -2.194262981414795,
|
|
"logps/chosen": -0.2805050313472748,
|
|
"logps/rejected": -0.2911986708641052,
|
|
"loss": 1.5213605165481567,
|
|
"loss/core": 1.4580514430999756,
|
|
"loss/preference_sft": 0.28713876008987427,
|
|
"loss/reference_anchor": 1.237465739250183,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 4.375100135803223,
|
|
"rewards/margins": 2.9718849658966064,
|
|
"rewards/rejected": 1.4032156467437744,
|
|
"ronpo/logit": 0.29718852043151855,
|
|
"ronpo/residual": 0.30155670642852783,
|
|
"ronpo/residual_abs": 0.34738093614578247,
|
|
"ronpo/target": -0.004368200898170471,
|
|
"ronpo/target_abs": 0.0792548879981041,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 285
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2998009920120239,
|
|
"drift/rejected_abs": 0.13739660382270813,
|
|
"epoch": 0.09217322208978942,
|
|
"grad_norm": 133.0,
|
|
"learning_rate": 2.14580019698942e-08,
|
|
"logits/chosen": -2.0153133869171143,
|
|
"logits/rejected": -2.089466094970703,
|
|
"logps/chosen": -0.27183207869529724,
|
|
"logps/rejected": -0.2848559617996216,
|
|
"loss": 0.26796045899391174,
|
|
"loss/core": 0.25352850556373596,
|
|
"loss/preference_sft": 0.2748119533061981,
|
|
"loss/reference_anchor": 0.261157751083374,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.99642276763916,
|
|
"rewards/margins": 1.623225212097168,
|
|
"rewards/rejected": 1.3731976747512817,
|
|
"ronpo/logit": 0.162322536110878,
|
|
"ronpo/residual": 0.1448330283164978,
|
|
"ronpo/residual_abs": 0.23569996654987335,
|
|
"ronpo/target": 0.01748950406908989,
|
|
"ronpo/target_abs": 0.0773894339799881,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 290
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19523122906684875,
|
|
"drift/rejected_abs": 0.07123854756355286,
|
|
"epoch": 0.09376241557409615,
|
|
"grad_norm": 178.0,
|
|
"learning_rate": 2.128726321782077e-08,
|
|
"logits/chosen": -2.1933460235595703,
|
|
"logits/rejected": -2.1863884925842285,
|
|
"logps/chosen": -0.30095577239990234,
|
|
"logps/rejected": -0.3060711622238159,
|
|
"loss": 0.24075086414813995,
|
|
"loss/core": 0.2300082892179489,
|
|
"loss/preference_sft": 0.30148637294769287,
|
|
"loss/reference_anchor": 0.18470288813114166,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.951626181602478,
|
|
"rewards/margins": 1.244182825088501,
|
|
"rewards/rejected": 0.7074434161186218,
|
|
"ronpo/logit": 0.12441827356815338,
|
|
"ronpo/residual": 0.1209135428071022,
|
|
"ronpo/residual_abs": 0.17452454566955566,
|
|
"ronpo/target": 0.003504746360704303,
|
|
"ronpo/target_abs": 0.07728487253189087,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 295
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.13675720989704132,
|
|
"drift/rejected_abs": 0.0812956839799881,
|
|
"epoch": 0.09535160905840286,
|
|
"grad_norm": 5.84375,
|
|
"learning_rate": 2.111321993542385e-08,
|
|
"logits/chosen": -2.407482862472534,
|
|
"logits/rejected": -2.4131016731262207,
|
|
"logps/chosen": -0.29553696513175964,
|
|
"logps/rejected": -0.28668391704559326,
|
|
"loss": 0.03661810979247093,
|
|
"loss/core": 0.03264369070529938,
|
|
"loss/preference_sft": 0.2879699468612671,
|
|
"loss/reference_anchor": 0.05069132521748543,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.3664600849151611,
|
|
"rewards/margins": 0.5552068948745728,
|
|
"rewards/rejected": 0.8112531900405884,
|
|
"ronpo/logit": 0.055520690977573395,
|
|
"ronpo/residual": 0.053940463811159134,
|
|
"ronpo/residual_abs": 0.12251658737659454,
|
|
"ronpo/target": 0.0015802305424585938,
|
|
"ronpo/target_abs": 0.09203118085861206,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 300
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2224307805299759,
|
|
"drift/rejected_abs": 0.10239239037036896,
|
|
"epoch": 0.09694080254270958,
|
|
"grad_norm": 3.28125,
|
|
"learning_rate": 2.0935937573264096e-08,
|
|
"logits/chosen": -2.0765604972839355,
|
|
"logits/rejected": -2.104783296585083,
|
|
"logps/chosen": -0.30479732155799866,
|
|
"logps/rejected": -0.29314523935317993,
|
|
"loss": 0.1927420198917389,
|
|
"loss/core": 0.17845560610294342,
|
|
"loss/preference_sft": 0.3036041855812073,
|
|
"loss/reference_anchor": 0.25536784529685974,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.221867322921753,
|
|
"rewards/margins": 1.2011425495147705,
|
|
"rewards/rejected": 1.0207250118255615,
|
|
"ronpo/logit": 0.12011425197124481,
|
|
"ronpo/residual": 0.12593664228916168,
|
|
"ronpo/residual_abs": 0.18047745525836945,
|
|
"ronpo/target": -0.005822394508868456,
|
|
"ronpo/target_abs": 0.0825614184141159,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 305
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2303655445575714,
|
|
"drift/rejected_abs": 0.14673031866550446,
|
|
"epoch": 0.09852999602701629,
|
|
"grad_norm": 278.0,
|
|
"learning_rate": 2.0755482799987594e-08,
|
|
"logits/chosen": -2.0974299907684326,
|
|
"logits/rejected": -1.9897010326385498,
|
|
"logps/chosen": -0.30457696318626404,
|
|
"logps/rejected": -0.3037092089653015,
|
|
"loss": 0.09838415682315826,
|
|
"loss/core": 0.08123552054166794,
|
|
"loss/preference_sft": 0.31207767128944397,
|
|
"loss/reference_anchor": 0.3117648661136627,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.3016207218170166,
|
|
"rewards/margins": 0.8351632952690125,
|
|
"rewards/rejected": 1.4664572477340698,
|
|
"ronpo/logit": 0.08351632952690125,
|
|
"ronpo/residual": 0.08266756683588028,
|
|
"ronpo/residual_abs": 0.16260148584842682,
|
|
"ronpo/target": 0.0008487681043334305,
|
|
"ronpo/target_abs": 0.08333858102560043,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 310
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21899929642677307,
|
|
"drift/rejected_abs": 0.17108279466629028,
|
|
"epoch": 0.10011918951132301,
|
|
"grad_norm": 1.8046875,
|
|
"learning_rate": 2.0571923477254526e-08,
|
|
"logits/chosen": -2.210271120071411,
|
|
"logits/rejected": -2.2432236671447754,
|
|
"logps/chosen": -0.28812751173973083,
|
|
"logps/rejected": -0.2839018404483795,
|
|
"loss": 0.12934687733650208,
|
|
"loss/core": 0.08804286271333694,
|
|
"loss/preference_sft": 0.28392842411994934,
|
|
"loss/reference_anchor": 0.7976871728897095,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.1899399757385254,
|
|
"rewards/margins": 0.48040738701820374,
|
|
"rewards/rejected": 1.7095324993133545,
|
|
"ronpo/logit": 0.04804074019193649,
|
|
"ronpo/residual": 0.03702558949589729,
|
|
"ronpo/residual_abs": 0.14168190956115723,
|
|
"ronpo/target": 0.011015147902071476,
|
|
"ronpo/target_abs": 0.08301273733377457,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 315
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.30637550354003906,
|
|
"drift/rejected_abs": 0.15101422369480133,
|
|
"epoch": 0.10170838299562972,
|
|
"grad_norm": 7.90625,
|
|
"learning_rate": 2.038532863421914e-08,
|
|
"logits/chosen": -2.0474765300750732,
|
|
"logits/rejected": -2.046475648880005,
|
|
"logps/chosen": -0.2672887146472931,
|
|
"logps/rejected": -0.2681429982185364,
|
|
"loss": 0.43184512853622437,
|
|
"loss/core": 0.39570650458335876,
|
|
"loss/preference_sft": 0.26500943303108215,
|
|
"loss/reference_anchor": 0.6962720155715942,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.0628886222839355,
|
|
"rewards/margins": 1.5545833110809326,
|
|
"rewards/rejected": 1.5083054304122925,
|
|
"ronpo/logit": 0.15545836091041565,
|
|
"ronpo/residual": 0.15610265731811523,
|
|
"ronpo/residual_abs": 0.2172074019908905,
|
|
"ronpo/target": -0.000644296407699585,
|
|
"ronpo/target_abs": 0.08082158118486404,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 320
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2718430161476135,
|
|
"drift/rejected_abs": 0.06493407487869263,
|
|
"epoch": 0.10329757647993643,
|
|
"grad_norm": 23.875,
|
|
"learning_rate": 2.0195768441570726e-08,
|
|
"logits/chosen": -2.213127374649048,
|
|
"logits/rejected": -2.2380900382995605,
|
|
"logps/chosen": -0.3036438524723053,
|
|
"logps/rejected": -0.2892650365829468,
|
|
"loss": 1.196415901184082,
|
|
"loss/core": 1.1635918617248535,
|
|
"loss/preference_sft": 0.2940658628940582,
|
|
"loss/reference_anchor": 0.6270740032196045,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.717806100845337,
|
|
"rewards/margins": 2.0749688148498535,
|
|
"rewards/rejected": 0.642837643623352,
|
|
"ronpo/logit": 0.20749685168266296,
|
|
"ronpo/residual": 0.22607414424419403,
|
|
"ronpo/residual_abs": 0.2697368264198303,
|
|
"ronpo/target": -0.018577273935079575,
|
|
"ronpo/target_abs": 0.09064265340566635,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 325
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2503616511821747,
|
|
"drift/rejected_abs": 0.12431076914072037,
|
|
"epoch": 0.10488676996424315,
|
|
"grad_norm": 10.875,
|
|
"learning_rate": 2.0003314185145307e-08,
|
|
"logits/chosen": -2.3221628665924072,
|
|
"logits/rejected": -2.322690010070801,
|
|
"logps/chosen": -0.26718512177467346,
|
|
"logps/rejected": -0.26268628239631653,
|
|
"loss": 0.566504180431366,
|
|
"loss/core": 0.5484728813171387,
|
|
"loss/preference_sft": 0.255176305770874,
|
|
"loss/reference_anchor": 0.3351089656352997,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.5033228397369385,
|
|
"rewards/margins": 1.2658904790878296,
|
|
"rewards/rejected": 1.2374324798583984,
|
|
"ronpo/logit": 0.12658903002738953,
|
|
"ronpo/residual": 0.1411885768175125,
|
|
"ronpo/residual_abs": 0.2634466588497162,
|
|
"ronpo/target": -0.014599531888961792,
|
|
"ronpo/target_abs": 0.09172377735376358,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 330
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.445903480052948,
|
|
"drift/rejected_abs": 0.21055002510547638,
|
|
"epoch": 0.10647596344854986,
|
|
"grad_norm": 12.875,
|
|
"learning_rate": 1.9808038239117914e-08,
|
|
"logits/chosen": -1.991874098777771,
|
|
"logits/rejected": -1.9406774044036865,
|
|
"logps/chosen": -0.2943147122859955,
|
|
"logps/rejected": -0.294437974691391,
|
|
"loss": 0.7760679721832275,
|
|
"loss/core": 0.7043066024780273,
|
|
"loss/preference_sft": 0.2942696809768677,
|
|
"loss/reference_anchor": 1.4057997465133667,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 4.458679676055908,
|
|
"rewards/margins": 2.356698513031006,
|
|
"rewards/rejected": 2.1019809246063232,
|
|
"ronpo/logit": 0.23566989600658417,
|
|
"ronpo/residual": 0.2460823804140091,
|
|
"ronpo/residual_abs": 0.30592185258865356,
|
|
"ronpo/target": -0.010412474162876606,
|
|
"ronpo/target_abs": 0.07370901107788086,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 335
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.45900607109069824,
|
|
"drift/rejected_abs": 0.16360488533973694,
|
|
"epoch": 0.10806515693285658,
|
|
"grad_norm": 82.0,
|
|
"learning_rate": 1.9610014038785648e-08,
|
|
"logits/chosen": -1.8805198669433594,
|
|
"logits/rejected": -1.9393765926361084,
|
|
"logps/chosen": -0.28708821535110474,
|
|
"logps/rejected": -0.2747197151184082,
|
|
"loss": 1.0118316411972046,
|
|
"loss/core": 0.9642781019210815,
|
|
"loss/preference_sft": 0.27820974588394165,
|
|
"loss/reference_anchor": 0.9232500791549683,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.589595794677734,
|
|
"rewards/margins": 2.960404396057129,
|
|
"rewards/rejected": 1.6291908025741577,
|
|
"ronpo/logit": 0.29604044556617737,
|
|
"ronpo/residual": 0.3044920861721039,
|
|
"ronpo/residual_abs": 0.3554214537143707,
|
|
"ronpo/target": -0.008451665751636028,
|
|
"ronpo/target_abs": 0.08010601997375488,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 340
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24650216102600098,
|
|
"drift/rejected_abs": 0.1208873763680458,
|
|
"epoch": 0.10965435041716329,
|
|
"grad_norm": 58.5,
|
|
"learning_rate": 1.9409316052951656e-08,
|
|
"logits/chosen": -2.178605556488037,
|
|
"logits/rejected": -2.2515740394592285,
|
|
"logps/chosen": -0.2723388671875,
|
|
"logps/rejected": -0.2781441807746887,
|
|
"loss": 0.12823966145515442,
|
|
"loss/core": 0.11076222360134125,
|
|
"loss/preference_sft": 0.28036752343177795,
|
|
"loss/reference_anchor": 0.32151204347610474,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.4640796184539795,
|
|
"rewards/margins": 1.2876027822494507,
|
|
"rewards/rejected": 1.1764768362045288,
|
|
"ronpo/logit": 0.12876029312610626,
|
|
"ronpo/residual": 0.12763699889183044,
|
|
"ronpo/residual_abs": 0.17462731897830963,
|
|
"ronpo/target": 0.0011232979595661163,
|
|
"ronpo/target_abs": 0.07203267514705658,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 345
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4136660099029541,
|
|
"drift/rejected_abs": 0.16482646763324738,
|
|
"epoch": 0.11124354390147001,
|
|
"grad_norm": 1.8359375,
|
|
"learning_rate": 1.920601975592047e-08,
|
|
"logits/chosen": -2.267277240753174,
|
|
"logits/rejected": -2.2952818870544434,
|
|
"logps/chosen": -0.23398800194263458,
|
|
"logps/rejected": -0.2486116588115692,
|
|
"loss": 1.0056898593902588,
|
|
"loss/core": 0.9502155184745789,
|
|
"loss/preference_sft": 0.24102886021137238,
|
|
"loss/reference_anchor": 1.0853848457336426,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.136660099029541,
|
|
"rewards/margins": 2.506661891937256,
|
|
"rewards/rejected": 1.629997968673706,
|
|
"ronpo/logit": 0.25066617131233215,
|
|
"ronpo/residual": 0.24287991225719452,
|
|
"ronpo/residual_abs": 0.3027934730052948,
|
|
"ronpo/target": 0.007786286063492298,
|
|
"ronpo/target_abs": 0.08268611133098602,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 350
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28172412514686584,
|
|
"drift/rejected_abs": 0.10272540897130966,
|
|
"epoch": 0.11283273738577672,
|
|
"grad_norm": 14.875,
|
|
"learning_rate": 1.900020159911519e-08,
|
|
"logits/chosen": -2.1245241165161133,
|
|
"logits/rejected": -2.0237369537353516,
|
|
"logps/chosen": -0.2827402949333191,
|
|
"logps/rejected": -0.2766692042350769,
|
|
"loss": 0.41723641753196716,
|
|
"loss/core": 0.4000471234321594,
|
|
"loss/preference_sft": 0.2682899832725525,
|
|
"loss/reference_anchor": 0.31695735454559326,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.8172414302825928,
|
|
"rewards/margins": 1.7916021347045898,
|
|
"rewards/rejected": 1.025639533996582,
|
|
"ronpo/logit": 0.1791602075099945,
|
|
"ronpo/residual": 0.16192427277565002,
|
|
"ronpo/residual_abs": 0.23042872548103333,
|
|
"ronpo/target": 0.01723591983318329,
|
|
"ronpo/target_abs": 0.08204632252454758,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 355
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21779580414295197,
|
|
"drift/rejected_abs": 0.08848261833190918,
|
|
"epoch": 0.11442193087008343,
|
|
"grad_norm": 11.5,
|
|
"learning_rate": 1.8791938982327247e-08,
|
|
"logits/chosen": -2.206521511077881,
|
|
"logits/rejected": -2.2727904319763184,
|
|
"logps/chosen": -0.2671610713005066,
|
|
"logps/rejected": -0.2585466206073761,
|
|
"loss": 0.09771557152271271,
|
|
"loss/core": 0.09170512855052948,
|
|
"loss/preference_sft": 0.2634662985801697,
|
|
"loss/reference_anchor": 0.09386207908391953,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.1769795417785645,
|
|
"rewards/margins": 1.2925890684127808,
|
|
"rewards/rejected": 0.8843904733657837,
|
|
"ronpo/logit": 0.1292589008808136,
|
|
"ronpo/residual": 0.1271723061800003,
|
|
"ronpo/residual_abs": 0.18293924629688263,
|
|
"ronpo/target": 0.002086599823087454,
|
|
"ronpo/target_abs": 0.07021833956241608,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 360
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20787136256694794,
|
|
"drift/rejected_abs": 0.13100872933864594,
|
|
"epoch": 0.11601112435439015,
|
|
"grad_norm": 2.125,
|
|
"learning_rate": 1.85813102246095e-08,
|
|
"logits/chosen": -2.0922367572784424,
|
|
"logits/rejected": -2.1322176456451416,
|
|
"logps/chosen": -0.2858452796936035,
|
|
"logps/rejected": -0.3246702551841736,
|
|
"loss": 0.06760017573833466,
|
|
"loss/core": 0.047362055629491806,
|
|
"loss/preference_sft": 0.28588929772377014,
|
|
"loss/reference_anchor": 0.37617334723472595,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.0768673419952393,
|
|
"rewards/margins": 0.791536271572113,
|
|
"rewards/rejected": 1.2853310108184814,
|
|
"ronpo/logit": 0.0791536197066307,
|
|
"ronpo/residual": 0.07149665057659149,
|
|
"ronpo/residual_abs": 0.13622984290122986,
|
|
"ronpo/target": 0.007656970527023077,
|
|
"ronpo/target_abs": 0.09360074996948242,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 365
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.31859681010246277,
|
|
"drift/rejected_abs": 0.21548397839069366,
|
|
"epoch": 0.11760031783869686,
|
|
"grad_norm": 20.375,
|
|
"learning_rate": 1.8368394534823633e-08,
|
|
"logits/chosen": -2.136131763458252,
|
|
"logits/rejected": -2.168311595916748,
|
|
"logps/chosen": -0.27841588854789734,
|
|
"logps/rejected": -0.28144991397857666,
|
|
"loss": 0.16637016832828522,
|
|
"loss/core": 0.10749348253011703,
|
|
"loss/preference_sft": 0.2804015874862671,
|
|
"loss/reference_anchor": 1.1494934558868408,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.1855576038360596,
|
|
"rewards/margins": 1.0312860012054443,
|
|
"rewards/rejected": 2.1542716026306152,
|
|
"ronpo/logit": 0.10312861204147339,
|
|
"ronpo/residual": 0.0984123945236206,
|
|
"ronpo/residual_abs": 0.17065349221229553,
|
|
"ronpo/target": 0.004716214723885059,
|
|
"ronpo/target_abs": 0.08422146737575531,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 370
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21396997570991516,
|
|
"drift/rejected_abs": 0.09890774637460709,
|
|
"epoch": 0.11918951132300358,
|
|
"grad_norm": 19.5,
|
|
"learning_rate": 1.815327198185297e-08,
|
|
"logits/chosen": -2.278458833694458,
|
|
"logits/rejected": -2.3031888008117676,
|
|
"logps/chosen": -0.29324546456336975,
|
|
"logps/rejected": -0.28806382417678833,
|
|
"loss": 0.13843554258346558,
|
|
"loss/core": 0.12961502373218536,
|
|
"loss/preference_sft": 0.28800949454307556,
|
|
"loss/reference_anchor": 0.14760896563529968,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.1395039558410645,
|
|
"rewards/margins": 1.1516611576080322,
|
|
"rewards/rejected": 0.9878427386283875,
|
|
"ronpo/logit": 0.11516612768173218,
|
|
"ronpo/residual": 0.10511883348226547,
|
|
"ronpo/residual_abs": 0.1740320473909378,
|
|
"ronpo/target": 0.01004729326814413,
|
|
"ronpo/target_abs": 0.08315954357385635,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 375
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2743528187274933,
|
|
"drift/rejected_abs": 0.09948831051588058,
|
|
"epoch": 0.12077870480731029,
|
|
"grad_norm": 0.81640625,
|
|
"learning_rate": 1.793602346449181e-08,
|
|
"logits/chosen": -2.158848524093628,
|
|
"logits/rejected": -2.1761603355407715,
|
|
"logps/chosen": -0.2898809313774109,
|
|
"logps/rejected": -0.28347843885421753,
|
|
"loss": 0.5179681181907654,
|
|
"loss/core": 0.49899888038635254,
|
|
"loss/preference_sft": 0.28954416513442993,
|
|
"loss/reference_anchor": 0.35042867064476013,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.7419450283050537,
|
|
"rewards/margins": 1.7553457021713257,
|
|
"rewards/rejected": 0.9865990877151489,
|
|
"ronpo/logit": 0.17553459107875824,
|
|
"ronpo/residual": 0.17026254534721375,
|
|
"ronpo/residual_abs": 0.2293941080570221,
|
|
"ronpo/target": 0.0052720606327056885,
|
|
"ronpo/target_abs": 0.0783097967505455,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 380
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2517983317375183,
|
|
"drift/rejected_abs": 0.14854536950588226,
|
|
"epoch": 0.12236789829161701,
|
|
"grad_norm": 63.5,
|
|
"learning_rate": 1.7716730681022724e-08,
|
|
"logits/chosen": -2.2751753330230713,
|
|
"logits/rejected": -2.2461843490600586,
|
|
"logps/chosen": -0.2717970013618469,
|
|
"logps/rejected": -0.2640511393547058,
|
|
"loss": 0.1832820177078247,
|
|
"loss/core": 0.17082270979881287,
|
|
"loss/preference_sft": 0.26608261466026306,
|
|
"loss/reference_anchor": 0.22257855534553528,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.516340494155884,
|
|
"rewards/margins": 1.0328803062438965,
|
|
"rewards/rejected": 1.4834601879119873,
|
|
"ronpo/logit": 0.10328805446624756,
|
|
"ronpo/residual": 0.0827520564198494,
|
|
"ronpo/residual_abs": 0.15645729005336761,
|
|
"ronpo/target": 0.020535994321107864,
|
|
"ronpo/target_abs": 0.08894232660531998,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 385
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21122761070728302,
|
|
"drift/rejected_abs": 0.12187887728214264,
|
|
"epoch": 0.12395709177592372,
|
|
"grad_norm": 2.46875,
|
|
"learning_rate": 1.749547609849315e-08,
|
|
"logits/chosen": -2.0779099464416504,
|
|
"logits/rejected": -2.118682384490967,
|
|
"logps/chosen": -0.3044072091579437,
|
|
"logps/rejected": -0.2954100966453552,
|
|
"loss": 0.09059147536754608,
|
|
"loss/core": 0.07689197361469269,
|
|
"loss/preference_sft": 0.3024619221687317,
|
|
"loss/reference_anchor": 0.24374382197856903,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.1094882488250732,
|
|
"rewards/margins": 0.8933243751525879,
|
|
"rewards/rejected": 1.2161636352539062,
|
|
"ronpo/logit": 0.0893324464559555,
|
|
"ronpo/residual": 0.0991659015417099,
|
|
"ronpo/residual_abs": 0.1529773473739624,
|
|
"ronpo/target": -0.00983346439898014,
|
|
"ronpo/target_abs": 0.09204504638910294,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 390
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21260306239128113,
|
|
"drift/rejected_abs": 0.0686182901263237,
|
|
"epoch": 0.12554628526023043,
|
|
"grad_norm": 2.84375,
|
|
"learning_rate": 1.7272342921702935e-08,
|
|
"logits/chosen": -2.158278465270996,
|
|
"logits/rejected": -2.2224063873291016,
|
|
"logps/chosen": -0.31105631589889526,
|
|
"logps/rejected": -0.2972649335861206,
|
|
"loss": 0.35496413707733154,
|
|
"loss/core": 0.34252816438674927,
|
|
"loss/preference_sft": 0.3063923716545105,
|
|
"loss/reference_anchor": 0.2180800884962082,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.1244759559631348,
|
|
"rewards/margins": 1.4389787912368774,
|
|
"rewards/rejected": 0.6854972839355469,
|
|
"ronpo/logit": 0.1438978910446167,
|
|
"ronpo/residual": 0.12823660671710968,
|
|
"ronpo/residual_abs": 0.1940845549106598,
|
|
"ronpo/target": 0.015661273151636124,
|
|
"ronpo/target_abs": 0.0806753933429718,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 395
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19167861342430115,
|
|
"drift/rejected_abs": 0.08439207822084427,
|
|
"epoch": 0.12713547874453715,
|
|
"grad_norm": 8.3125,
|
|
"learning_rate": 1.7047415061914393e-08,
|
|
"logits/chosen": -2.2295565605163574,
|
|
"logits/rejected": -2.3195853233337402,
|
|
"logps/chosen": -0.25695428252220154,
|
|
"logps/rejected": -0.2609720826148987,
|
|
"loss": 0.07849901169538498,
|
|
"loss/core": 0.07116608321666718,
|
|
"loss/preference_sft": 0.2507200539112091,
|
|
"loss/reference_anchor": 0.12158646434545517,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 1.9159801006317139,
|
|
"rewards/margins": 1.0806057453155518,
|
|
"rewards/rejected": 0.8353743553161621,
|
|
"ronpo/logit": 0.1080605760216713,
|
|
"ronpo/residual": 0.11492305994033813,
|
|
"ronpo/residual_abs": 0.1632971316576004,
|
|
"ronpo/target": -0.0068624913692474365,
|
|
"ronpo/target_abs": 0.08553292602300644,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 400
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17035509645938873,
|
|
"drift/rejected_abs": 0.14627881348133087,
|
|
"epoch": 0.12872467222884387,
|
|
"grad_norm": 2.34375,
|
|
"learning_rate": 1.6820777105296708e-08,
|
|
"logits/chosen": -2.2909557819366455,
|
|
"logits/rejected": -2.2994208335876465,
|
|
"logps/chosen": -0.29540935158729553,
|
|
"logps/rejected": -0.2884915769100189,
|
|
"loss": 0.29835981130599976,
|
|
"loss/core": 0.2854141592979431,
|
|
"loss/preference_sft": 0.2857155203819275,
|
|
"loss/reference_anchor": 0.230341836810112,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.7012271881103516,
|
|
"rewards/margins": 0.239228293299675,
|
|
"rewards/rejected": 1.4619989395141602,
|
|
"ronpo/logit": 0.023922838270664215,
|
|
"ronpo/residual": 0.02246231772005558,
|
|
"ronpo/residual_abs": 0.18744370341300964,
|
|
"ronpo/target": 0.001460521249100566,
|
|
"ronpo/target_abs": 0.08477913588285446,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 405
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18731415271759033,
|
|
"drift/rejected_abs": 0.05840852111577988,
|
|
"epoch": 0.13031386571315057,
|
|
"grad_norm": 0.6875,
|
|
"learning_rate": 1.6592514281116553e-08,
|
|
"logits/chosen": -2.229135751724243,
|
|
"logits/rejected": -2.1434459686279297,
|
|
"logps/chosen": -0.28917819261550903,
|
|
"logps/rejected": -0.2873764634132385,
|
|
"loss": 0.35508978366851807,
|
|
"loss/core": 0.344736784696579,
|
|
"loss/preference_sft": 0.28428682684898376,
|
|
"loss/reference_anchor": 0.17863091826438904,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.8712348937988281,
|
|
"rewards/margins": 1.3082423210144043,
|
|
"rewards/rejected": 0.5629926919937134,
|
|
"ronpo/logit": 0.1308242380619049,
|
|
"ronpo/residual": 0.14936430752277374,
|
|
"ronpo/residual_abs": 0.17734959721565247,
|
|
"ronpo/target": -0.01854007877409458,
|
|
"ronpo/target_abs": 0.06559576839208603,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 410
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.12775962054729462,
|
|
"drift/rejected_abs": 0.05946607142686844,
|
|
"epoch": 0.1319030591974573,
|
|
"grad_norm": 18.5,
|
|
"learning_rate": 1.6362712429686844e-08,
|
|
"logits/chosen": -2.2688026428222656,
|
|
"logits/rejected": -2.2994446754455566,
|
|
"logps/chosen": -0.2852744460105896,
|
|
"logps/rejected": -0.2741158604621887,
|
|
"loss": 0.04457860440015793,
|
|
"loss/core": 0.04096058011054993,
|
|
"loss/preference_sft": 0.27498331665992737,
|
|
"loss/reference_anchor": 0.04486214369535446,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.2775468826293945,
|
|
"rewards/margins": 0.6865289807319641,
|
|
"rewards/rejected": 0.5910179018974304,
|
|
"ronpo/logit": 0.06865289807319641,
|
|
"ronpo/residual": 0.061678577214479446,
|
|
"ronpo/residual_abs": 0.13630971312522888,
|
|
"ronpo/target": 0.006974323187023401,
|
|
"ronpo/target_abs": 0.08561506867408752,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 415
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3900828957557678,
|
|
"drift/rejected_abs": 0.12692275643348694,
|
|
"epoch": 0.133492252681764,
|
|
"grad_norm": 2.71875,
|
|
"learning_rate": 1.6131457970085686e-08,
|
|
"logits/chosen": -2.1172633171081543,
|
|
"logits/rejected": -2.235016345977783,
|
|
"logps/chosen": -0.2711094319820404,
|
|
"logps/rejected": -0.2714807391166687,
|
|
"loss": 0.8514937162399292,
|
|
"loss/core": 0.8164560198783875,
|
|
"loss/preference_sft": 0.2708297073841095,
|
|
"loss/reference_anchor": 0.6736721992492676,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.9008281230926514,
|
|
"rewards/margins": 2.7281997203826904,
|
|
"rewards/rejected": 1.17262864112854,
|
|
"ronpo/logit": 0.27282002568244934,
|
|
"ronpo/residual": 0.25294801592826843,
|
|
"ronpo/residual_abs": 0.3054654002189636,
|
|
"ronpo/target": 0.019871985539793968,
|
|
"ronpo/target_abs": 0.07743876427412033,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 420
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4248512387275696,
|
|
"drift/rejected_abs": 0.13988152146339417,
|
|
"epoch": 0.1350814461660707,
|
|
"grad_norm": 310.0,
|
|
"learning_rate": 1.5898837867657726e-08,
|
|
"logits/chosen": -2.166339159011841,
|
|
"logits/rejected": -2.1603424549102783,
|
|
"logps/chosen": -0.2978591322898865,
|
|
"logps/rejected": -0.2953459620475769,
|
|
"loss": 1.2260562181472778,
|
|
"loss/core": 1.1734199523925781,
|
|
"loss/preference_sft": 0.29464253783226013,
|
|
"loss/reference_anchor": 1.0232583284378052,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 4.247471332550049,
|
|
"rewards/margins": 2.849874258041382,
|
|
"rewards/rejected": 1.3975974321365356,
|
|
"ronpo/logit": 0.2849874496459961,
|
|
"ronpo/residual": 0.2766841650009155,
|
|
"ronpo/residual_abs": 0.3479006886482239,
|
|
"ronpo/target": 0.008303266949951649,
|
|
"ronpo/target_abs": 0.09610433876514435,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 425
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18739613890647888,
|
|
"drift/rejected_abs": 0.09084701538085938,
|
|
"epoch": 0.13667063965037743,
|
|
"grad_norm": 10.5,
|
|
"learning_rate": 1.5664939601310025e-08,
|
|
"logits/chosen": -2.4331743717193604,
|
|
"logits/rejected": -2.4204342365264893,
|
|
"logps/chosen": -0.2795220911502838,
|
|
"logps/rejected": -0.2753976583480835,
|
|
"loss": 0.10062910616397858,
|
|
"loss/core": 0.09226439893245697,
|
|
"loss/preference_sft": 0.2744489312171936,
|
|
"loss/reference_anchor": 0.13984903693199158,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.8734142780303955,
|
|
"rewards/margins": 0.9657185673713684,
|
|
"rewards/rejected": 0.9076957702636719,
|
|
"ronpo/logit": 0.09657186269760132,
|
|
"ronpo/residual": 0.07882784307003021,
|
|
"ronpo/residual_abs": 0.14449390769004822,
|
|
"ronpo/target": 0.017744004726409912,
|
|
"ronpo/target_abs": 0.07996980845928192,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 430
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.17314964532852173,
|
|
"drift/rejected_abs": 0.09341616928577423,
|
|
"epoch": 0.13825983313468415,
|
|
"grad_norm": 12.5,
|
|
"learning_rate": 1.542985113061481e-08,
|
|
"logits/chosen": -2.1819088459014893,
|
|
"logits/rejected": -2.1771626472473145,
|
|
"logps/chosen": -0.2796979546546936,
|
|
"logps/rejected": -0.2803123891353607,
|
|
"loss": 0.04070793837308884,
|
|
"loss/core": 0.03540237620472908,
|
|
"loss/preference_sft": 0.2738858759403229,
|
|
"loss/reference_anchor": 0.07872271537780762,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.730942964553833,
|
|
"rewards/margins": 0.7972149848937988,
|
|
"rewards/rejected": 0.9337277412414551,
|
|
"ronpo/logit": 0.07972150295972824,
|
|
"ronpo/residual": 0.054669272154569626,
|
|
"ronpo/residual_abs": 0.12845924496650696,
|
|
"ronpo/target": 0.025052234530448914,
|
|
"ronpo/target_abs": 0.07804572582244873,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 435
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.318293035030365,
|
|
"drift/rejected_abs": 0.13249650597572327,
|
|
"epoch": 0.13984902661899087,
|
|
"grad_norm": 4.15625,
|
|
"learning_rate": 1.51936608627315e-08,
|
|
"logits/chosen": -2.273724317550659,
|
|
"logits/rejected": -2.299027919769287,
|
|
"logps/chosen": -0.2951945960521698,
|
|
"logps/rejected": -0.29159849882125854,
|
|
"loss": 0.7339528203010559,
|
|
"loss/core": 0.6881879568099976,
|
|
"loss/preference_sft": 0.2977384924888611,
|
|
"loss/reference_anchor": 0.885523796081543,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.182380437850952,
|
|
"rewards/margins": 1.8633613586425781,
|
|
"rewards/rejected": 1.319018840789795,
|
|
"ronpo/logit": 0.18633611500263214,
|
|
"ronpo/residual": 0.19329491257667542,
|
|
"ronpo/residual_abs": 0.24592044949531555,
|
|
"ronpo/target": -0.006958787329494953,
|
|
"ronpo/target_abs": 0.0852695107460022,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 440
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18986327946186066,
|
|
"drift/rejected_abs": 0.16925686597824097,
|
|
"epoch": 0.14143822010329757,
|
|
"grad_norm": 260.0,
|
|
"learning_rate": 1.4956457619160377e-08,
|
|
"logits/chosen": -2.2318367958068848,
|
|
"logits/rejected": -2.154519557952881,
|
|
"logps/chosen": -0.28089621663093567,
|
|
"logps/rejected": -0.2713889479637146,
|
|
"loss": 0.25093281269073486,
|
|
"loss/core": 0.2370346039533615,
|
|
"loss/preference_sft": 0.27891331911087036,
|
|
"loss/reference_anchor": 0.25007274746894836,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.8986326456069946,
|
|
"rewards/margins": 0.20648033916950226,
|
|
"rewards/rejected": 1.6921522617340088,
|
|
"ronpo/logit": 0.020648030564188957,
|
|
"ronpo/residual": 0.02750701829791069,
|
|
"ronpo/residual_abs": 0.20267800986766815,
|
|
"ronpo/target": -0.006858985871076584,
|
|
"ronpo/target_abs": 0.08476842194795609,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 445
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3151724934577942,
|
|
"drift/rejected_abs": 0.1027737632393837,
|
|
"epoch": 0.1430274135876043,
|
|
"grad_norm": 41.0,
|
|
"learning_rate": 1.4718330602340439e-08,
|
|
"logits/chosen": -2.264042615890503,
|
|
"logits/rejected": -2.327329397201538,
|
|
"logps/chosen": -0.2729445695877075,
|
|
"logps/rejected": -0.28299084305763245,
|
|
"loss": 0.7172930836677551,
|
|
"loss/core": 0.6890506148338318,
|
|
"loss/preference_sft": 0.2784006595611572,
|
|
"loss/reference_anchor": 0.5370103120803833,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.150481939315796,
|
|
"rewards/margins": 2.1262738704681396,
|
|
"rewards/rejected": 1.0242083072662354,
|
|
"ronpo/logit": 0.2126273810863495,
|
|
"ronpo/residual": 0.21831850707530975,
|
|
"ronpo/residual_abs": 0.26258429884910583,
|
|
"ronpo/target": -0.005691120866686106,
|
|
"ronpo/target_abs": 0.0731092318892479,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 450
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20744338631629944,
|
|
"drift/rejected_abs": 0.09908546507358551,
|
|
"epoch": 0.144616607071911,
|
|
"grad_norm": 10.25,
|
|
"learning_rate": 1.4479369362104036e-08,
|
|
"logits/chosen": -2.3172354698181152,
|
|
"logits/rejected": -2.3511035442352295,
|
|
"logps/chosen": -0.284721314907074,
|
|
"logps/rejected": -0.28263527154922485,
|
|
"loss": 0.10904005914926529,
|
|
"loss/core": 0.10059742629528046,
|
|
"loss/preference_sft": 0.2903629243373871,
|
|
"loss/reference_anchor": 0.13981646299362183,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.0744338035583496,
|
|
"rewards/margins": 1.083627462387085,
|
|
"rewards/rejected": 0.9908064603805542,
|
|
"ronpo/logit": 0.10836274921894073,
|
|
"ronpo/residual": 0.107516810297966,
|
|
"ronpo/residual_abs": 0.15838299691677094,
|
|
"ronpo/target": 0.0008459277451038361,
|
|
"ronpo/target_abs": 0.0736476257443428,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 455
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.37808623909950256,
|
|
"drift/rejected_abs": 0.1648140251636505,
|
|
"epoch": 0.1462058005562177,
|
|
"grad_norm": 121.5,
|
|
"learning_rate": 1.4239663762000817e-08,
|
|
"logits/chosen": -1.8078231811523438,
|
|
"logits/rejected": -1.6678974628448486,
|
|
"logps/chosen": -0.2803362011909485,
|
|
"logps/rejected": -0.2994075417518616,
|
|
"loss": 0.551475465297699,
|
|
"loss/core": 0.5203845500946045,
|
|
"loss/preference_sft": 0.28328341245651245,
|
|
"loss/reference_anchor": 0.5934896469116211,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.7770161628723145,
|
|
"rewards/margins": 2.1362390518188477,
|
|
"rewards/rejected": 1.6407768726348877,
|
|
"ronpo/logit": 0.21362391114234924,
|
|
"ronpo/residual": 0.21224097907543182,
|
|
"ronpo/residual_abs": 0.2707666754722595,
|
|
"ronpo/target": 0.0013829537201672792,
|
|
"ronpo/target_abs": 0.08258362114429474,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 460
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.34338995814323425,
|
|
"drift/rejected_abs": 0.23440727591514587,
|
|
"epoch": 0.14779499404052443,
|
|
"grad_norm": 400.0,
|
|
"learning_rate": 1.3999303945503746e-08,
|
|
"logits/chosen": -2.0976834297180176,
|
|
"logits/rejected": -2.116819381713867,
|
|
"logps/chosen": -0.2706809341907501,
|
|
"logps/rejected": -0.27307775616645813,
|
|
"loss": 0.8269802331924438,
|
|
"loss/core": 0.7959731221199036,
|
|
"loss/preference_sft": 0.2685784101486206,
|
|
"loss/reference_anchor": 0.5932824611663818,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.4336636066436768,
|
|
"rewards/margins": 1.0909364223480225,
|
|
"rewards/rejected": 2.3427271842956543,
|
|
"ronpo/logit": 0.10909364372491837,
|
|
"ronpo/residual": 0.10874764621257782,
|
|
"ronpo/residual_abs": 0.3142366111278534,
|
|
"ronpo/target": 0.00034598278580233455,
|
|
"ronpo/target_abs": 0.07308773696422577,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 465
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.29909831285476685,
|
|
"drift/rejected_abs": 0.11643582582473755,
|
|
"epoch": 0.14938418752483115,
|
|
"grad_norm": 1.9375,
|
|
"learning_rate": 1.3758380302109807e-08,
|
|
"logits/chosen": -2.075563430786133,
|
|
"logits/rejected": -2.044642210006714,
|
|
"logps/chosen": -0.254794180393219,
|
|
"logps/rejected": -0.2602981626987457,
|
|
"loss": 0.7801807522773743,
|
|
"loss/core": 0.7529562711715698,
|
|
"loss/preference_sft": 0.26095765829086304,
|
|
"loss/reference_anchor": 0.5183919668197632,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.989849090576172,
|
|
"rewards/margins": 1.8269249200820923,
|
|
"rewards/rejected": 1.1629244089126587,
|
|
"ronpo/logit": 0.1826924830675125,
|
|
"ronpo/residual": 0.19307713210582733,
|
|
"ronpo/residual_abs": 0.2564256191253662,
|
|
"ronpo/target": -0.010384641587734222,
|
|
"ronpo/target_abs": 0.08509234338998795,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 470
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2505967915058136,
|
|
"drift/rejected_abs": 0.09651152044534683,
|
|
"epoch": 0.15097338100913787,
|
|
"grad_norm": 9.1875,
|
|
"learning_rate": 1.3516983433348227e-08,
|
|
"logits/chosen": -2.19514536857605,
|
|
"logits/rejected": -2.130986213684082,
|
|
"logps/chosen": -0.3052103817462921,
|
|
"logps/rejected": -0.29925405979156494,
|
|
"loss": 0.22467193007469177,
|
|
"loss/core": 0.21474584937095642,
|
|
"loss/preference_sft": 0.3062250018119812,
|
|
"loss/reference_anchor": 0.16789942979812622,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.505239486694336,
|
|
"rewards/margins": 1.5401643514633179,
|
|
"rewards/rejected": 0.9650753140449524,
|
|
"ronpo/logit": 0.15401646494865417,
|
|
"ronpo/residual": 0.15226852893829346,
|
|
"ronpo/residual_abs": 0.21335072815418243,
|
|
"ronpo/target": 0.0017479099333286285,
|
|
"ronpo/target_abs": 0.07254009693861008,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 475
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22144432365894318,
|
|
"drift/rejected_abs": 0.10890810191631317,
|
|
"epoch": 0.15256257449344457,
|
|
"grad_norm": 1.0859375,
|
|
"learning_rate": 1.3275204118708941e-08,
|
|
"logits/chosen": -2.119616746902466,
|
|
"logits/rejected": -2.1420140266418457,
|
|
"logps/chosen": -0.273566871881485,
|
|
"logps/rejected": -0.2686625123023987,
|
|
"loss": 0.08422975242137909,
|
|
"loss/core": 0.07651440799236298,
|
|
"loss/preference_sft": 0.2724953591823578,
|
|
"loss/reference_anchor": 0.12705738842487335,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 2.21227765083313,
|
|
"rewards/margins": 1.1317625045776367,
|
|
"rewards/rejected": 1.0805152654647827,
|
|
"ronpo/logit": 0.11317624896764755,
|
|
"ronpo/residual": 0.1295435130596161,
|
|
"ronpo/residual_abs": 0.1753176897764206,
|
|
"ronpo/target": -0.016367264091968536,
|
|
"ronpo/target_abs": 0.08774258196353912,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 480
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2984105944633484,
|
|
"drift/rejected_abs": 0.1611432582139969,
|
|
"epoch": 0.1541517679777513,
|
|
"grad_norm": 5.5,
|
|
"learning_rate": 1.3033133281504133e-08,
|
|
"logits/chosen": -1.902937889099121,
|
|
"logits/rejected": -2.0238137245178223,
|
|
"logps/chosen": -0.28192150592803955,
|
|
"logps/rejected": -0.27930665016174316,
|
|
"loss": 0.13234123587608337,
|
|
"loss/core": 0.11475740373134613,
|
|
"loss/preference_sft": 0.2823740839958191,
|
|
"loss/reference_anchor": 0.3234391510486603,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.983555793762207,
|
|
"rewards/margins": 1.3728954792022705,
|
|
"rewards/rejected": 1.6106599569320679,
|
|
"ronpo/logit": 0.13728956878185272,
|
|
"ronpo/residual": 0.1287764012813568,
|
|
"ronpo/residual_abs": 0.18728485703468323,
|
|
"ronpo/target": 0.008513162843883038,
|
|
"ronpo/target_abs": 0.08558246493339539,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 485
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3685315251350403,
|
|
"drift/rejected_abs": 0.150931254029274,
|
|
"epoch": 0.15574096146205801,
|
|
"grad_norm": 290.0,
|
|
"learning_rate": 1.2790861954675703e-08,
|
|
"logits/chosen": -2.10400390625,
|
|
"logits/rejected": -2.0856916904449463,
|
|
"logps/chosen": -0.28186899423599243,
|
|
"logps/rejected": -0.28309863805770874,
|
|
"loss": 0.6792070269584656,
|
|
"loss/core": 0.643970787525177,
|
|
"loss/preference_sft": 0.28243130445480347,
|
|
"loss/reference_anchor": 0.676482617855072,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 3.6826233863830566,
|
|
"rewards/margins": 2.1746814250946045,
|
|
"rewards/rejected": 1.5079416036605835,
|
|
"ronpo/logit": 0.21746817231178284,
|
|
"ronpo/residual": 0.21048131585121155,
|
|
"ronpo/residual_abs": 0.2691686451435089,
|
|
"ronpo/target": 0.006986844353377819,
|
|
"ronpo/target_abs": 0.09098772704601288,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 490
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22006180882453918,
|
|
"drift/rejected_abs": 0.18455646932125092,
|
|
"epoch": 0.1573301549463647,
|
|
"grad_norm": 396.0,
|
|
"learning_rate": 1.2548481246561504e-08,
|
|
"logits/chosen": -2.1262259483337402,
|
|
"logits/rejected": -2.0967445373535156,
|
|
"logps/chosen": -0.2868593633174896,
|
|
"logps/rejected": -0.27408647537231445,
|
|
"loss": 1.1514532566070557,
|
|
"loss/core": 1.1166924238204956,
|
|
"loss/preference_sft": 0.28249964118003845,
|
|
"loss/reference_anchor": 0.6669629812240601,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.1990761756896973,
|
|
"rewards/margins": 0.357932448387146,
|
|
"rewards/rejected": 1.8411436080932617,
|
|
"ronpo/logit": 0.03579322621226311,
|
|
"ronpo/residual": 0.049376267939805984,
|
|
"ronpo/residual_abs": 0.3220534324645996,
|
|
"ronpo/target": -0.01358303613960743,
|
|
"ronpo/target_abs": 0.09055624902248383,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 495
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2433398962020874,
|
|
"drift/rejected_abs": 0.14733143150806427,
|
|
"epoch": 0.15891934843067143,
|
|
"grad_norm": 4.125,
|
|
"learning_rate": 1.2306082306633209e-08,
|
|
"logits/chosen": -2.231499195098877,
|
|
"logits/rejected": -2.0890326499938965,
|
|
"logps/chosen": -0.2816493809223175,
|
|
"logps/rejected": -0.27643585205078125,
|
|
"loss": 0.23166105151176453,
|
|
"loss/core": 0.21609747409820557,
|
|
"loss/preference_sft": 0.2753569185733795,
|
|
"loss/reference_anchor": 0.28373581171035767,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.4331910610198975,
|
|
"rewards/margins": 0.9780365228652954,
|
|
"rewards/rejected": 1.455154538154602,
|
|
"ronpo/logit": 0.09780365228652954,
|
|
"ronpo/residual": 0.10129328817129135,
|
|
"ronpo/residual_abs": 0.21842603385448456,
|
|
"ronpo/target": -0.0034896410070359707,
|
|
"ronpo/target_abs": 0.08457918465137482,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 500
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16049498319625854,
|
|
"drift/rejected_abs": 0.10690776258707047,
|
|
"epoch": 0.16050854191497815,
|
|
"grad_norm": 4.5625,
|
|
"learning_rate": 1.2063756291218741e-08,
|
|
"logits/chosen": -2.284985065460205,
|
|
"logits/rejected": -2.205040454864502,
|
|
"logps/chosen": -0.29578253626823425,
|
|
"logps/rejected": -0.2804624140262604,
|
|
"loss": 0.03737588971853256,
|
|
"loss/core": 0.0324200838804245,
|
|
"loss/preference_sft": 0.2899005115032196,
|
|
"loss/reference_anchor": 0.070126011967659,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.6018108129501343,
|
|
"rewards/margins": 0.5337648987770081,
|
|
"rewards/rejected": 1.0680458545684814,
|
|
"ronpo/logit": 0.05337648466229439,
|
|
"ronpo/residual": 0.051874689757823944,
|
|
"ronpo/residual_abs": 0.12874646484851837,
|
|
"ronpo/target": 0.0015017971163615584,
|
|
"ronpo/target_abs": 0.08839758485555649,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 505
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.31590768694877625,
|
|
"drift/rejected_abs": 0.1603766679763794,
|
|
"epoch": 0.16209773539928488,
|
|
"grad_norm": 2.671875,
|
|
"learning_rate": 1.1821594329222079e-08,
|
|
"logits/chosen": -1.9426066875457764,
|
|
"logits/rejected": -1.9659500122070312,
|
|
"logps/chosen": -0.26106467843055725,
|
|
"logps/rejected": -0.2636715769767761,
|
|
"loss": 0.4681622087955475,
|
|
"loss/core": 0.435793936252594,
|
|
"loss/preference_sft": 0.26362866163253784,
|
|
"loss/reference_anchor": 0.6210030317306519,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.1584575176239014,
|
|
"rewards/margins": 1.5580565929412842,
|
|
"rewards/rejected": 1.6004009246826172,
|
|
"ronpo/logit": 0.15580564737319946,
|
|
"ronpo/residual": 0.15252608060836792,
|
|
"ronpo/residual_abs": 0.2079734355211258,
|
|
"ronpo/target": 0.0032795709557831287,
|
|
"ronpo/target_abs": 0.07548246532678604,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 510
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2593478560447693,
|
|
"drift/rejected_abs": 0.15457691252231598,
|
|
"epoch": 0.16368692888359157,
|
|
"grad_norm": 5.03125,
|
|
"learning_rate": 1.157968748785344e-08,
|
|
"logits/chosen": -2.1415348052978516,
|
|
"logits/rejected": -2.2097504138946533,
|
|
"logps/chosen": -0.28786981105804443,
|
|
"logps/rejected": -0.27476245164871216,
|
|
"loss": 0.3538331389427185,
|
|
"loss/core": 0.33824825286865234,
|
|
"loss/preference_sft": 0.2842520475387573,
|
|
"loss/reference_anchor": 0.2832731306552887,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.5934786796569824,
|
|
"rewards/margins": 1.050806999206543,
|
|
"rewards/rejected": 1.542672038078308,
|
|
"ronpo/logit": 0.10508068650960922,
|
|
"ronpo/residual": 0.1014178991317749,
|
|
"ronpo/residual_abs": 0.22226110100746155,
|
|
"ronpo/target": 0.0036627918016165495,
|
|
"ronpo/target_abs": 0.07639802247285843,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 515
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2165084183216095,
|
|
"drift/rejected_abs": 0.10440798103809357,
|
|
"epoch": 0.1652761223678983,
|
|
"grad_norm": 3.078125,
|
|
"learning_rate": 1.1338126738382598e-08,
|
|
"logits/chosen": -2.1498093605041504,
|
|
"logits/rejected": -2.1602375507354736,
|
|
"logps/chosen": -0.2931273579597473,
|
|
"logps/rejected": -0.30199140310287476,
|
|
"loss": 0.261959433555603,
|
|
"loss/core": 0.2352810651063919,
|
|
"loss/preference_sft": 0.293254554271698,
|
|
"loss/reference_anchor": 0.5042413473129272,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.16508412361145,
|
|
"rewards/margins": 1.1286433935165405,
|
|
"rewards/rejected": 1.0364407300949097,
|
|
"ronpo/logit": 0.11286433786153793,
|
|
"ronpo/residual": 0.10544689744710922,
|
|
"ronpo/residual_abs": 0.17088401317596436,
|
|
"ronpo/target": 0.007417450193315744,
|
|
"ronpo/target_abs": 0.08988238126039505,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 520
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22024352848529816,
|
|
"drift/rejected_abs": 0.06529395282268524,
|
|
"epoch": 0.16686531585220502,
|
|
"grad_norm": 7.9375,
|
|
"learning_rate": 1.1097002921928317e-08,
|
|
"logits/chosen": -1.9056131839752197,
|
|
"logits/rejected": -2.050123929977417,
|
|
"logps/chosen": -0.3115862309932709,
|
|
"logps/rejected": -0.3198958933353424,
|
|
"loss": 0.2833040654659271,
|
|
"loss/core": 0.2724967300891876,
|
|
"loss/preference_sft": 0.32389670610427856,
|
|
"loss/reference_anchor": 0.18375670909881592,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.201336145401001,
|
|
"rewards/margins": 1.5499281883239746,
|
|
"rewards/rejected": 0.6514078378677368,
|
|
"ronpo/logit": 0.15499281883239746,
|
|
"ronpo/residual": 0.1652340143918991,
|
|
"ronpo/residual_abs": 0.22205249965190887,
|
|
"ronpo/target": -0.010241201147437096,
|
|
"ronpo/target_abs": 0.09030992537736893,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 525
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15349416434764862,
|
|
"drift/rejected_abs": 0.06507018208503723,
|
|
"epoch": 0.1684545093365117,
|
|
"grad_norm": 1.2734375,
|
|
"learning_rate": 1.0856406715296717e-08,
|
|
"logits/chosen": -2.3231589794158936,
|
|
"logits/rejected": -2.317765712738037,
|
|
"logps/chosen": -0.2665853500366211,
|
|
"logps/rejected": -0.2610515356063843,
|
|
"loss": 0.06234220415353775,
|
|
"loss/core": 0.05808652564883232,
|
|
"loss/preference_sft": 0.26546263694763184,
|
|
"loss/reference_anchor": 0.05856741592288017,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 1.534586787223816,
|
|
"rewards/margins": 0.8877205848693848,
|
|
"rewards/rejected": 0.6468662023544312,
|
|
"ronpo/logit": 0.08877205848693848,
|
|
"ronpo/residual": 0.09845416247844696,
|
|
"ronpo/residual_abs": 0.13878455758094788,
|
|
"ronpo/target": -0.009682102128863335,
|
|
"ronpo/target_abs": 0.08094774186611176,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 530
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3475359380245209,
|
|
"drift/rejected_abs": 0.1439954936504364,
|
|
"epoch": 0.17004370282081843,
|
|
"grad_norm": 3.171875,
|
|
"learning_rate": 1.0616428596881461e-08,
|
|
"logits/chosen": -2.2414088249206543,
|
|
"logits/rejected": -2.255117893218994,
|
|
"logps/chosen": -0.31528395414352417,
|
|
"logps/rejected": -0.3214128613471985,
|
|
"loss": 1.5130736827850342,
|
|
"loss/core": 1.4584028720855713,
|
|
"loss/preference_sft": 0.31637319922447205,
|
|
"loss/reference_anchor": 1.0617750883102417,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.473903179168701,
|
|
"rewards/margins": 2.195761203765869,
|
|
"rewards/rejected": 1.2781422138214111,
|
|
"ronpo/logit": 0.21957612037658691,
|
|
"ronpo/residual": 0.2536545991897583,
|
|
"ronpo/residual_abs": 0.3511970341205597,
|
|
"ronpo/target": -0.034078482538461685,
|
|
"ronpo/target_abs": 0.09332413971424103,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 535
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24628369510173798,
|
|
"drift/rejected_abs": 0.1078454852104187,
|
|
"epoch": 0.17163289630512515,
|
|
"grad_norm": 4.6875,
|
|
"learning_rate": 1.0377158812638491e-08,
|
|
"logits/chosen": -2.202691078186035,
|
|
"logits/rejected": -2.2384159564971924,
|
|
"logps/chosen": -0.27798351645469666,
|
|
"logps/rejected": -0.29006585478782654,
|
|
"loss": 0.22769439220428467,
|
|
"loss/core": 0.20758168399333954,
|
|
"loss/preference_sft": 0.28700748085975647,
|
|
"loss/reference_anchor": 0.37355345487594604,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.4620206356048584,
|
|
"rewards/margins": 1.38406240940094,
|
|
"rewards/rejected": 1.0779582262039185,
|
|
"ronpo/logit": 0.13840624690055847,
|
|
"ronpo/residual": 0.1439955234527588,
|
|
"ronpo/residual_abs": 0.1996123194694519,
|
|
"ronpo/target": -0.005589271429926157,
|
|
"ronpo/target_abs": 0.07924790680408478,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 540
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3985271155834198,
|
|
"drift/rejected_abs": 0.24583449959754944,
|
|
"epoch": 0.17322208978943188,
|
|
"grad_norm": 3.984375,
|
|
"learning_rate": 1.0138687342148249e-08,
|
|
"logits/chosen": -1.992592453956604,
|
|
"logits/rejected": -2.0166046619415283,
|
|
"logps/chosen": -0.2606894373893738,
|
|
"logps/rejected": -0.27958837151527405,
|
|
"loss": 0.46023693680763245,
|
|
"loss/core": 0.41777506470680237,
|
|
"loss/preference_sft": 0.27719753980636597,
|
|
"loss/reference_anchor": 0.8215174674987793,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 3.9839775562286377,
|
|
"rewards/margins": 1.532893419265747,
|
|
"rewards/rejected": 2.4510836601257324,
|
|
"ronpo/logit": 0.15328934788703918,
|
|
"ronpo/residual": 0.15502643585205078,
|
|
"ronpo/residual_abs": 0.2659107744693756,
|
|
"ronpo/target": -0.0017371115973219275,
|
|
"ronpo/target_abs": 0.0876903235912323,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 545
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2757337689399719,
|
|
"drift/rejected_abs": 0.13784222304821014,
|
|
"epoch": 0.17481128327373857,
|
|
"grad_norm": 0.8828125,
|
|
"learning_rate": 9.90110386477801e-09,
|
|
"logits/chosen": -2.144867420196533,
|
|
"logits/rejected": -2.1006226539611816,
|
|
"logps/chosen": -0.2549815773963928,
|
|
"logps/rejected": -0.26524803042411804,
|
|
"loss": 0.307639479637146,
|
|
"loss/core": 0.28966981172561646,
|
|
"loss/preference_sft": 0.26307356357574463,
|
|
"loss/reference_anchor": 0.3330860733985901,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.7563281059265137,
|
|
"rewards/margins": 1.3787453174591064,
|
|
"rewards/rejected": 1.3775826692581177,
|
|
"ronpo/logit": 0.1378745287656784,
|
|
"ronpo/residual": 0.14565442502498627,
|
|
"ronpo/residual_abs": 0.20090845227241516,
|
|
"ronpo/target": -0.007779897656291723,
|
|
"ronpo/target_abs": 0.0823056623339653,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 550
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.12533283233642578,
|
|
"drift/rejected_abs": 0.05521406978368759,
|
|
"epoch": 0.1764004767580453,
|
|
"grad_norm": 0.640625,
|
|
"learning_rate": 9.664497725957164e-09,
|
|
"logits/chosen": -2.216212511062622,
|
|
"logits/rejected": -2.258756160736084,
|
|
"logps/chosen": -0.311422199010849,
|
|
"logps/rejected": -0.30880364775657654,
|
|
"loss": 0.07220657169818878,
|
|
"loss/core": 0.06694497168064117,
|
|
"loss/preference_sft": 0.3110686242580414,
|
|
"loss/reference_anchor": 0.07412528246641159,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.252854347229004,
|
|
"rewards/margins": 0.7011260390281677,
|
|
"rewards/rejected": 0.551728367805481,
|
|
"ronpo/logit": 0.07011260092258453,
|
|
"ronpo/residual": 0.0683169811964035,
|
|
"ronpo/residual_abs": 0.1411377489566803,
|
|
"ronpo/target": 0.0017956193769350648,
|
|
"ronpo/target_abs": 0.09536027908325195,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 555
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2655150294303894,
|
|
"drift/rejected_abs": 0.12211163341999054,
|
|
"epoch": 0.17798967024235202,
|
|
"grad_norm": 16.5,
|
|
"learning_rate": 9.428957903578044e-09,
|
|
"logits/chosen": -2.08587646484375,
|
|
"logits/rejected": -2.091303586959839,
|
|
"logps/chosen": -0.2719891667366028,
|
|
"logps/rejected": -0.27926185727119446,
|
|
"loss": 0.254443883895874,
|
|
"loss/core": 0.24059149622917175,
|
|
"loss/preference_sft": 0.2721792161464691,
|
|
"loss/reference_anchor": 0.24982962012290955,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.653385639190674,
|
|
"rewards/margins": 1.4334917068481445,
|
|
"rewards/rejected": 1.2198941707611084,
|
|
"ronpo/logit": 0.14334917068481445,
|
|
"ronpo/residual": 0.13729219138622284,
|
|
"ronpo/residual_abs": 0.18650557100772858,
|
|
"ronpo/target": 0.0060569727793335915,
|
|
"ronpo/target_abs": 0.0743868425488472,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 560
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2600178122520447,
|
|
"drift/rejected_abs": 0.11504121869802475,
|
|
"epoch": 0.1795788637266587,
|
|
"grad_norm": 28.25,
|
|
"learning_rate": 9.194572974534975e-09,
|
|
"logits/chosen": -2.041768789291382,
|
|
"logits/rejected": -2.0671210289001465,
|
|
"logps/chosen": -0.34080713987350464,
|
|
"logps/rejected": -0.3354662358760834,
|
|
"loss": 0.36407026648521423,
|
|
"loss/core": 0.34170204401016235,
|
|
"loss/preference_sft": 0.34413477778434753,
|
|
"loss/reference_anchor": 0.4129505157470703,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.597466230392456,
|
|
"rewards/margins": 1.4497805833816528,
|
|
"rewards/rejected": 1.1476857662200928,
|
|
"ronpo/logit": 0.14497807621955872,
|
|
"ronpo/residual": 0.144731804728508,
|
|
"ronpo/residual_abs": 0.20529747009277344,
|
|
"ronpo/target": 0.0002462614211253822,
|
|
"ronpo/target_abs": 0.08707638084888458,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 565
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.41662201285362244,
|
|
"drift/rejected_abs": 0.09962382912635803,
|
|
"epoch": 0.18116805721096543,
|
|
"grad_norm": 9.9375,
|
|
"learning_rate": 8.96143108141412e-09,
|
|
"logits/chosen": -2.1486594676971436,
|
|
"logits/rejected": -2.119309902191162,
|
|
"logps/chosen": -0.27083221077919006,
|
|
"logps/rejected": -0.2647841274738312,
|
|
"loss": 1.456986904144287,
|
|
"loss/core": 1.414247989654541,
|
|
"loss/preference_sft": 0.2618175148963928,
|
|
"loss/reference_anchor": 0.8285951614379883,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 4.165776252746582,
|
|
"rewards/margins": 3.1706364154815674,
|
|
"rewards/rejected": 0.9951398968696594,
|
|
"ronpo/logit": 0.31706368923187256,
|
|
"ronpo/residual": 0.30529677867889404,
|
|
"ronpo/residual_abs": 0.3695726692676544,
|
|
"ronpo/target": 0.011766904965043068,
|
|
"ronpo/target_abs": 0.0811844915151596,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 570
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19189342856407166,
|
|
"drift/rejected_abs": 0.09652575105428696,
|
|
"epoch": 0.18275725069527216,
|
|
"grad_norm": 3.328125,
|
|
"learning_rate": 8.72961989934668e-09,
|
|
"logits/chosen": -2.0099151134490967,
|
|
"logits/rejected": -2.0976462364196777,
|
|
"logps/chosen": -0.28342169523239136,
|
|
"logps/rejected": -0.27922505140304565,
|
|
"loss": 0.05769554525613785,
|
|
"loss/core": 0.052024852484464645,
|
|
"loss/preference_sft": 0.28891006112098694,
|
|
"loss/reference_anchor": 0.08452272415161133,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.9166200160980225,
|
|
"rewards/margins": 0.9536026120185852,
|
|
"rewards/rejected": 0.963017463684082,
|
|
"ronpo/logit": 0.09536024928092957,
|
|
"ronpo/residual": 0.10312417894601822,
|
|
"ronpo/residual_abs": 0.14294111728668213,
|
|
"ronpo/target": -0.00776391988620162,
|
|
"ronpo/target_abs": 0.07904698699712753,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 575
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.36418384313583374,
|
|
"drift/rejected_abs": 0.1662895232439041,
|
|
"epoch": 0.18434644417957885,
|
|
"grad_norm": 1.7734375,
|
|
"learning_rate": 8.499226603037854e-09,
|
|
"logits/chosen": -2.0011439323425293,
|
|
"logits/rejected": -2.031702756881714,
|
|
"logps/chosen": -0.28904083371162415,
|
|
"logps/rejected": -0.2685038447380066,
|
|
"loss": 0.5859975814819336,
|
|
"loss/core": 0.5528149008750916,
|
|
"loss/preference_sft": 0.28503909707069397,
|
|
"loss/reference_anchor": 0.6351504921913147,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.640242099761963,
|
|
"rewards/margins": 1.9781475067138672,
|
|
"rewards/rejected": 1.6620944738388062,
|
|
"ronpo/logit": 0.19781477749347687,
|
|
"ronpo/residual": 0.19819863140583038,
|
|
"ronpo/residual_abs": 0.2739180326461792,
|
|
"ronpo/target": -0.0003838575503323227,
|
|
"ronpo/target_abs": 0.08456491678953171,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 580
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23182368278503418,
|
|
"drift/rejected_abs": 0.09531630575656891,
|
|
"epoch": 0.18593563766388557,
|
|
"grad_norm": 1.9375,
|
|
"learning_rate": 8.270337833983987e-09,
|
|
"logits/chosen": -1.9567636251449585,
|
|
"logits/rejected": -1.9816787242889404,
|
|
"logps/chosen": -0.28648510575294495,
|
|
"logps/rejected": -0.28638434410095215,
|
|
"loss": 0.15469267964363098,
|
|
"loss/core": 0.14479199051856995,
|
|
"loss/preference_sft": 0.29087167978286743,
|
|
"loss/reference_anchor": 0.16892696917057037,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.317553997039795,
|
|
"rewards/margins": 1.3662830591201782,
|
|
"rewards/rejected": 0.9512712359428406,
|
|
"ronpo/logit": 0.13662829995155334,
|
|
"ronpo/residual": 0.1309715062379837,
|
|
"ronpo/residual_abs": 0.17593583464622498,
|
|
"ronpo/target": 0.00565680768340826,
|
|
"ronpo/target_abs": 0.0699964389204979,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 585
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25730934739112854,
|
|
"drift/rejected_abs": 0.1230391412973404,
|
|
"epoch": 0.1875248311481923,
|
|
"grad_norm": 1.375,
|
|
"learning_rate": 8.04303966789025e-09,
|
|
"logits/chosen": -2.118440866470337,
|
|
"logits/rejected": -2.0865204334259033,
|
|
"logps/chosen": -0.3093511760234833,
|
|
"logps/rejected": -0.3141013979911804,
|
|
"loss": 0.5824509263038635,
|
|
"loss/core": 0.5587186217308044,
|
|
"loss/preference_sft": 0.3041399419307709,
|
|
"loss/reference_anchor": 0.4442318379878998,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.570599317550659,
|
|
"rewards/margins": 1.3402273654937744,
|
|
"rewards/rejected": 1.2303721904754639,
|
|
"ronpo/logit": 0.1340227574110031,
|
|
"ronpo/residual": 0.14667926728725433,
|
|
"ronpo/residual_abs": 0.22119927406311035,
|
|
"ronpo/target": -0.012656529434025288,
|
|
"ronpo/target_abs": 0.08529551327228546,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 590
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23181328177452087,
|
|
"drift/rejected_abs": 0.11188916116952896,
|
|
"epoch": 0.18911402463249902,
|
|
"grad_norm": 6.5,
|
|
"learning_rate": 7.817417582301098e-09,
|
|
"logits/chosen": -1.9782168865203857,
|
|
"logits/rejected": -1.9370613098144531,
|
|
"logps/chosen": -0.2896624207496643,
|
|
"logps/rejected": -0.29374343156814575,
|
|
"loss": 0.1985751986503601,
|
|
"loss/core": 0.18759621679782867,
|
|
"loss/preference_sft": 0.29568907618522644,
|
|
"loss/reference_anchor": 0.1900101602077484,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.3166379928588867,
|
|
"rewards/margins": 1.202042818069458,
|
|
"rewards/rejected": 1.1145952939987183,
|
|
"ronpo/logit": 0.12020426988601685,
|
|
"ronpo/residual": 0.11585943400859833,
|
|
"ronpo/residual_abs": 0.1900613158941269,
|
|
"ronpo/target": 0.00434483215212822,
|
|
"ronpo/target_abs": 0.08699861913919449,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 595
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2037758082151413,
|
|
"drift/rejected_abs": 0.14159931242465973,
|
|
"epoch": 0.1907032181168057,
|
|
"grad_norm": 152.0,
|
|
"learning_rate": 7.59355642445566e-09,
|
|
"logits/chosen": -2.1320338249206543,
|
|
"logits/rejected": -2.1046671867370605,
|
|
"logps/chosen": -0.29946523904800415,
|
|
"logps/rejected": -0.3209082782268524,
|
|
"loss": 0.4402243494987488,
|
|
"loss/core": 0.4250958561897278,
|
|
"loss/preference_sft": 0.3151097893714905,
|
|
"loss/reference_anchor": 0.27105993032455444,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.0345118045806885,
|
|
"rewards/margins": 0.6213451623916626,
|
|
"rewards/rejected": 1.4131665229797363,
|
|
"ronpo/logit": 0.0621345154941082,
|
|
"ronpo/residual": 0.06725134700536728,
|
|
"ronpo/residual_abs": 0.2370101511478424,
|
|
"ronpo/target": -0.005116832908242941,
|
|
"ronpo/target_abs": 0.07861164957284927,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 600
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.34658682346343994,
|
|
"drift/rejected_abs": 0.1579512059688568,
|
|
"epoch": 0.19229241160111243,
|
|
"grad_norm": 1.8359375,
|
|
"learning_rate": 7.371540379380149e-09,
|
|
"logits/chosen": -2.0286879539489746,
|
|
"logits/rejected": -1.953050971031189,
|
|
"logps/chosen": -0.283130407333374,
|
|
"logps/rejected": -0.27908605337142944,
|
|
"loss": 0.4860695004463196,
|
|
"loss/core": 0.45568352937698364,
|
|
"loss/preference_sft": 0.28036171197891235,
|
|
"loss/reference_anchor": 0.5796834826469421,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.456225633621216,
|
|
"rewards/margins": 1.8863617181777954,
|
|
"rewards/rejected": 1.5698635578155518,
|
|
"ronpo/logit": 0.1886361837387085,
|
|
"ronpo/residual": 0.18362456560134888,
|
|
"ronpo/residual_abs": 0.25072842836380005,
|
|
"ronpo/target": 0.005011610221117735,
|
|
"ronpo/target_abs": 0.09113539010286331,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 605
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.33414921164512634,
|
|
"drift/rejected_abs": 0.15187159180641174,
|
|
"epoch": 0.19388160508541916,
|
|
"grad_norm": 87.5,
|
|
"learning_rate": 7.151452938229324e-09,
|
|
"logits/chosen": -2.0221667289733887,
|
|
"logits/rejected": -1.9719922542572021,
|
|
"logps/chosen": -0.31065306067466736,
|
|
"logps/rejected": -0.294253408908844,
|
|
"loss": 0.34279918670654297,
|
|
"loss/core": 0.31768709421157837,
|
|
"loss/preference_sft": 0.3136529326438904,
|
|
"loss/reference_anchor": 0.470877081155777,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.3398075103759766,
|
|
"rewards/margins": 1.8233705759048462,
|
|
"rewards/rejected": 1.5164368152618408,
|
|
"ronpo/logit": 0.18233706057071686,
|
|
"ronpo/residual": 0.1946430653333664,
|
|
"ronpo/residual_abs": 0.24323885142803192,
|
|
"ronpo/target": -0.012306014075875282,
|
|
"ronpo/target_abs": 0.0871143639087677,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 610
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2550937533378601,
|
|
"drift/rejected_abs": 0.11376242339611053,
|
|
"epoch": 0.19547079856972585,
|
|
"grad_norm": 125.5,
|
|
"learning_rate": 6.933376866888882e-09,
|
|
"logits/chosen": -2.2011666297912598,
|
|
"logits/rejected": -2.162236452102661,
|
|
"logps/chosen": -0.28771674633026123,
|
|
"logps/rejected": -0.28812357783317566,
|
|
"loss": 0.3027460277080536,
|
|
"loss/core": 0.28452661633491516,
|
|
"loss/preference_sft": 0.2859073579311371,
|
|
"loss/reference_anchor": 0.3357972800731659,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.5509376525878906,
|
|
"rewards/margins": 1.4323103427886963,
|
|
"rewards/rejected": 1.1186271905899048,
|
|
"ronpo/logit": 0.14323106408119202,
|
|
"ronpo/residual": 0.14810332655906677,
|
|
"ronpo/residual_abs": 0.1831142008304596,
|
|
"ronpo/target": -0.004872290883213282,
|
|
"ronpo/target_abs": 0.07001273334026337,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 615
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22261254489421844,
|
|
"drift/rejected_abs": 0.10966954380273819,
|
|
"epoch": 0.19705999205403257,
|
|
"grad_norm": 12.0,
|
|
"learning_rate": 6.717394174850605e-09,
|
|
"logits/chosen": -2.0854969024658203,
|
|
"logits/rejected": -2.0211100578308105,
|
|
"logps/chosen": -0.2912364602088928,
|
|
"logps/rejected": -0.2868305444717407,
|
|
"loss": 0.10874249786138535,
|
|
"loss/core": 0.1000494584441185,
|
|
"loss/preference_sft": 0.28769445419311523,
|
|
"loss/reference_anchor": 0.14509165287017822,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.2254793643951416,
|
|
"rewards/margins": 1.1289622783660889,
|
|
"rewards/rejected": 1.0965168476104736,
|
|
"ronpo/logit": 0.11289622634649277,
|
|
"ronpo/residual": 0.11616779863834381,
|
|
"ronpo/residual_abs": 0.1724853515625,
|
|
"ronpo/target": -0.003271549940109253,
|
|
"ronpo/target_abs": 0.07885955274105072,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 620
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24589157104492188,
|
|
"drift/rejected_abs": 0.09249149262905121,
|
|
"epoch": 0.1986491855383393,
|
|
"grad_norm": 49.0,
|
|
"learning_rate": 6.503586084371926e-09,
|
|
"logits/chosen": -2.0363283157348633,
|
|
"logits/rejected": -2.0404984951019287,
|
|
"logps/chosen": -0.2965640425682068,
|
|
"logps/rejected": -0.2985779047012329,
|
|
"loss": 0.34146690368652344,
|
|
"loss/core": 0.32861843705177307,
|
|
"loss/preference_sft": 0.29958051443099976,
|
|
"loss/reference_anchor": 0.22701041400432587,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.457526445388794,
|
|
"rewards/margins": 1.536529302597046,
|
|
"rewards/rejected": 0.920997142791748,
|
|
"ronpo/logit": 0.15365295112133026,
|
|
"ronpo/residual": 0.15617410838603973,
|
|
"ronpo/residual_abs": 0.213802307844162,
|
|
"ronpo/target": -0.0025211633183062077,
|
|
"ronpo/target_abs": 0.08408491313457489,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 625
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23458974063396454,
|
|
"drift/rejected_abs": 0.08947785198688507,
|
|
"epoch": 0.20023837902264602,
|
|
"grad_norm": 2.875,
|
|
"learning_rate": 6.29203299993155e-09,
|
|
"logits/chosen": -2.325148105621338,
|
|
"logits/rejected": -2.414926052093506,
|
|
"logps/chosen": -0.2667160928249359,
|
|
"logps/rejected": -0.26361387968063354,
|
|
"loss": 0.3130393922328949,
|
|
"loss/core": 0.2962197959423065,
|
|
"loss/preference_sft": 0.2673772871494293,
|
|
"loss/reference_anchor": 0.3096539378166199,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.345794439315796,
|
|
"rewards/margins": 1.4539344310760498,
|
|
"rewards/rejected": 0.8918598890304565,
|
|
"ronpo/logit": 0.14539344608783722,
|
|
"ronpo/residual": 0.13748937845230103,
|
|
"ronpo/residual_abs": 0.20279105007648468,
|
|
"ronpo/target": 0.007904065772891045,
|
|
"ronpo/target_abs": 0.08239643275737762,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 630
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28986817598342896,
|
|
"drift/rejected_abs": 0.10853831470012665,
|
|
"epoch": 0.2018275725069527,
|
|
"grad_norm": 15.375,
|
|
"learning_rate": 6.0828144779926565e-09,
|
|
"logits/chosen": -2.111020565032959,
|
|
"logits/rejected": -2.1508452892303467,
|
|
"logps/chosen": -0.2651686668395996,
|
|
"logps/rejected": -0.2719859182834625,
|
|
"loss": 0.3829234540462494,
|
|
"loss/core": 0.36607077717781067,
|
|
"loss/preference_sft": 0.269239604473114,
|
|
"loss/reference_anchor": 0.31012919545173645,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 2.8979406356811523,
|
|
"rewards/margins": 1.8216516971588135,
|
|
"rewards/rejected": 1.0762889385223389,
|
|
"ronpo/logit": 0.18216517567634583,
|
|
"ronpo/residual": 0.1788938343524933,
|
|
"ronpo/residual_abs": 0.22935160994529724,
|
|
"ronpo/target": 0.003271352034062147,
|
|
"ronpo/target_abs": 0.07819235324859619,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 635
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3245066702365875,
|
|
"drift/rejected_abs": 0.09787418693304062,
|
|
"epoch": 0.20341676599125944,
|
|
"grad_norm": 1.3046875,
|
|
"learning_rate": 5.87600919708494e-09,
|
|
"logits/chosen": -2.0167291164398193,
|
|
"logits/rejected": -2.0083279609680176,
|
|
"logps/chosen": -0.3133408725261688,
|
|
"logps/rejected": -0.3174097239971161,
|
|
"loss": 0.7297495603561401,
|
|
"loss/core": 0.7002626657485962,
|
|
"loss/preference_sft": 0.3104119300842285,
|
|
"loss/reference_anchor": 0.5586963891983032,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.2447280883789062,
|
|
"rewards/margins": 2.300731897354126,
|
|
"rewards/rejected": 0.9439964294433594,
|
|
"ronpo/logit": 0.23007316887378693,
|
|
"ronpo/residual": 0.23132848739624023,
|
|
"ronpo/residual_abs": 0.27878475189208984,
|
|
"ronpo/target": -0.0012552805710583925,
|
|
"ronpo/target_abs": 0.08154843747615814,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 640
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2655944526195526,
|
|
"drift/rejected_abs": 0.11456499993801117,
|
|
"epoch": 0.20500595947556616,
|
|
"grad_norm": 7.34375,
|
|
"learning_rate": 5.671694928216829e-09,
|
|
"logits/chosen": -2.1220450401306152,
|
|
"logits/rejected": -2.127779722213745,
|
|
"logps/chosen": -0.3085785508155823,
|
|
"logps/rejected": -0.29471153020858765,
|
|
"loss": 0.3396216034889221,
|
|
"loss/core": 0.32221999764442444,
|
|
"loss/preference_sft": 0.3059234619140625,
|
|
"loss/reference_anchor": 0.3174397945404053,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.6536717414855957,
|
|
"rewards/margins": 1.5116037130355835,
|
|
"rewards/rejected": 1.1420681476593018,
|
|
"ronpo/logit": 0.15116038918495178,
|
|
"ronpo/residual": 0.14593927562236786,
|
|
"ronpo/residual_abs": 0.2119034230709076,
|
|
"ronpo/target": 0.005221105180680752,
|
|
"ronpo/target_abs": 0.07934162765741348,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 645
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15132535994052887,
|
|
"drift/rejected_abs": 0.09072579443454742,
|
|
"epoch": 0.20659515295987285,
|
|
"grad_norm": 1.859375,
|
|
"learning_rate": 5.469948505629e-09,
|
|
"logits/chosen": -2.2047553062438965,
|
|
"logits/rejected": -2.1337642669677734,
|
|
"logps/chosen": -0.30132001638412476,
|
|
"logps/rejected": -0.2931322455406189,
|
|
"loss": 0.06630890816450119,
|
|
"loss/core": 0.06135622784495354,
|
|
"loss/preference_sft": 0.3019540309906006,
|
|
"loss/reference_anchor": 0.06885839998722076,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 1.5123118162155151,
|
|
"rewards/margins": 0.6547757983207703,
|
|
"rewards/rejected": 0.8575360178947449,
|
|
"ronpo/logit": 0.06547758728265762,
|
|
"ronpo/residual": 0.06879720091819763,
|
|
"ronpo/residual_abs": 0.14413335919380188,
|
|
"ronpo/target": -0.003319614799693227,
|
|
"ronpo/target_abs": 0.08891183882951736,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 650
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20433752238750458,
|
|
"drift/rejected_abs": 0.06546036899089813,
|
|
"epoch": 0.20818434644417957,
|
|
"grad_norm": 160.0,
|
|
"learning_rate": 5.270845797900168e-09,
|
|
"logits/chosen": -2.2784745693206787,
|
|
"logits/rejected": -2.2848639488220215,
|
|
"logps/chosen": -0.28383541107177734,
|
|
"logps/rejected": -0.29783403873443604,
|
|
"loss": 0.3861464858055115,
|
|
"loss/core": 0.3738086223602295,
|
|
"loss/preference_sft": 0.2895219326019287,
|
|
"loss/reference_anchor": 0.21780526638031006,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.042762517929077,
|
|
"rewards/margins": 1.3915107250213623,
|
|
"rewards/rejected": 0.6512519121170044,
|
|
"ronpo/logit": 0.13915108144283295,
|
|
"ronpo/residual": 0.16224148869514465,
|
|
"ronpo/residual_abs": 0.21403923630714417,
|
|
"ronpo/target": -0.02309040166437626,
|
|
"ronpo/target_abs": 0.0899852067232132,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 655
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1753084361553192,
|
|
"drift/rejected_abs": 0.07402169704437256,
|
|
"epoch": 0.2097735399284863,
|
|
"grad_norm": 6.65625,
|
|
"learning_rate": 5.07446167941601e-09,
|
|
"logits/chosen": -2.2544825077056885,
|
|
"logits/rejected": -2.2972869873046875,
|
|
"logps/chosen": -0.2956244647502899,
|
|
"logps/rejected": -0.28930968046188354,
|
|
"loss": 0.1263386309146881,
|
|
"loss/core": 0.11955497413873672,
|
|
"loss/preference_sft": 0.29426223039627075,
|
|
"loss/reference_anchor": 0.10624738782644272,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 1.7521049976348877,
|
|
"rewards/margins": 1.014744520187378,
|
|
"rewards/rejected": 0.7373603582382202,
|
|
"ronpo/logit": 0.10147446393966675,
|
|
"ronpo/residual": 0.09581993520259857,
|
|
"ronpo/residual_abs": 0.14587488770484924,
|
|
"ronpo/target": 0.005654525477439165,
|
|
"ronpo/target_abs": 0.07976684719324112,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 660
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.23486240208148956,
|
|
"drift/rejected_abs": 0.13577520847320557,
|
|
"epoch": 0.21136273341279302,
|
|
"grad_norm": 0.6953125,
|
|
"learning_rate": 4.880870002211964e-09,
|
|
"logits/chosen": -2.0763564109802246,
|
|
"logits/rejected": -2.0201761722564697,
|
|
"logps/chosen": -0.2823403477668762,
|
|
"logps/rejected": -0.2865101993083954,
|
|
"loss": 0.24251846969127655,
|
|
"loss/core": 0.22713109850883484,
|
|
"loss/preference_sft": 0.284621000289917,
|
|
"loss/reference_anchor": 0.2792838513851166,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.3464529514312744,
|
|
"rewards/margins": 0.9889312982559204,
|
|
"rewards/rejected": 1.3575217723846436,
|
|
"ronpo/logit": 0.09889312833547592,
|
|
"ronpo/residual": 0.10298037528991699,
|
|
"ronpo/residual_abs": 0.1839233934879303,
|
|
"ronpo/target": -0.004087253473699093,
|
|
"ronpo/target_abs": 0.09008091688156128,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 665
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20962342619895935,
|
|
"drift/rejected_abs": 0.07850834727287292,
|
|
"epoch": 0.2129519268970997,
|
|
"grad_norm": 5.40625,
|
|
"learning_rate": 4.6901435682005e-09,
|
|
"logits/chosen": -2.3460006713867188,
|
|
"logits/rejected": -2.3546290397644043,
|
|
"logps/chosen": -0.26797959208488464,
|
|
"logps/rejected": -0.27196618914604187,
|
|
"loss": 0.2039221227169037,
|
|
"loss/core": 0.1944776475429535,
|
|
"loss/preference_sft": 0.2721707820892334,
|
|
"loss/reference_anchor": 0.16167283058166504,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 2.094529390335083,
|
|
"rewards/margins": 1.3098270893096924,
|
|
"rewards/rejected": 0.7847023010253906,
|
|
"ronpo/logit": 0.13098271191120148,
|
|
"ronpo/residual": 0.15417954325675964,
|
|
"ronpo/residual_abs": 0.19339290261268616,
|
|
"ronpo/target": -0.02319684997200966,
|
|
"ronpo/target_abs": 0.08436106145381927,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 670
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.40202194452285767,
|
|
"drift/rejected_abs": 0.08130165189504623,
|
|
"epoch": 0.21454112038140644,
|
|
"grad_norm": 1.90625,
|
|
"learning_rate": 4.502354101793314e-09,
|
|
"logits/chosen": -1.9860864877700806,
|
|
"logits/rejected": -2.099212646484375,
|
|
"logps/chosen": -0.2779615521430969,
|
|
"logps/rejected": -0.26965683698654175,
|
|
"loss": 1.4497228860855103,
|
|
"loss/core": 1.408434271812439,
|
|
"loss/preference_sft": 0.2684791684150696,
|
|
"loss/reference_anchor": 0.7989233732223511,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.020218849182129,
|
|
"rewards/margins": 3.2108778953552246,
|
|
"rewards/rejected": 0.8093411326408386,
|
|
"ronpo/logit": 0.3210877776145935,
|
|
"ronpo/residual": 0.3173868954181671,
|
|
"ronpo/residual_abs": 0.3734723925590515,
|
|
"ronpo/target": 0.003700877772644162,
|
|
"ronpo/target_abs": 0.09122522175312042,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 675
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.19475491344928741,
|
|
"drift/rejected_abs": 0.1171645075082779,
|
|
"epoch": 0.21613031386571316,
|
|
"grad_norm": 3.171875,
|
|
"learning_rate": 4.317572222928703e-09,
|
|
"logits/chosen": -1.9540669918060303,
|
|
"logits/rejected": -1.9495131969451904,
|
|
"logps/chosen": -0.2881017029285431,
|
|
"logps/rejected": -0.28433549404144287,
|
|
"loss": 0.06940118968486786,
|
|
"loss/core": 0.05890883132815361,
|
|
"loss/preference_sft": 0.285450279712677,
|
|
"loss/reference_anchor": 0.18130208551883698,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.9457314014434814,
|
|
"rewards/margins": 0.7747111320495605,
|
|
"rewards/rejected": 1.1710201501846313,
|
|
"ronpo/logit": 0.07747111469507217,
|
|
"ronpo/residual": 0.08135292679071426,
|
|
"ronpo/residual_abs": 0.13843822479248047,
|
|
"ronpo/target": -0.0038818151224404573,
|
|
"ronpo/target_abs": 0.08158716559410095,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 680
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18266239762306213,
|
|
"drift/rejected_abs": 0.09462437778711319,
|
|
"epoch": 0.21771950735001985,
|
|
"grad_norm": 3.421875,
|
|
"learning_rate": 4.135867420514276e-09,
|
|
"logits/chosen": -1.9716804027557373,
|
|
"logits/rejected": -2.060253620147705,
|
|
"logps/chosen": -0.2921956181526184,
|
|
"logps/rejected": -0.2996564209461212,
|
|
"loss": 0.05632435530424118,
|
|
"loss/core": 0.05092400312423706,
|
|
"loss/preference_sft": 0.29816001653671265,
|
|
"loss/reference_anchor": 0.07819100469350815,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.825413465499878,
|
|
"rewards/margins": 0.8813973665237427,
|
|
"rewards/rejected": 0.9440159797668457,
|
|
"ronpo/logit": 0.08813974261283875,
|
|
"ronpo/residual": 0.08088794350624084,
|
|
"ronpo/residual_abs": 0.14101745188236237,
|
|
"ronpo/target": 0.007251805160194635,
|
|
"ronpo/target_abs": 0.08236397802829742,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 685
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4524809420108795,
|
|
"drift/rejected_abs": 0.1857810914516449,
|
|
"epoch": 0.21930870083432658,
|
|
"grad_norm": 434.0,
|
|
"learning_rate": 3.957308026295035e-09,
|
|
"logits/chosen": -1.9309346675872803,
|
|
"logits/rejected": -1.9748626947402954,
|
|
"logps/chosen": -0.27637702226638794,
|
|
"logps/rejected": -0.271381676197052,
|
|
"loss": 0.8734847903251648,
|
|
"loss/core": 0.8166624903678894,
|
|
"loss/preference_sft": 0.2828080356121063,
|
|
"loss/reference_anchor": 1.1081647872924805,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 4.524248123168945,
|
|
"rewards/margins": 2.668072462081909,
|
|
"rewards/rejected": 1.856175422668457,
|
|
"ronpo/logit": 0.2668072581291199,
|
|
"ronpo/residual": 0.25683221220970154,
|
|
"ronpo/residual_abs": 0.3273332417011261,
|
|
"ronpo/target": 0.009975056163966656,
|
|
"ronpo/target_abs": 0.09136839956045151,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 690
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18365684151649475,
|
|
"drift/rejected_abs": 0.11019404232501984,
|
|
"epoch": 0.2208978943186333,
|
|
"grad_norm": 12.5,
|
|
"learning_rate": 3.781961189156609e-09,
|
|
"logits/chosen": -2.160966634750366,
|
|
"logits/rejected": -2.1480095386505127,
|
|
"logps/chosen": -0.298018217086792,
|
|
"logps/rejected": -0.30044853687286377,
|
|
"loss": 0.10467197746038437,
|
|
"loss/core": 0.0970427542924881,
|
|
"loss/preference_sft": 0.2983938455581665,
|
|
"loss/reference_anchor": 0.12274493277072906,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.8359683752059937,
|
|
"rewards/margins": 0.7349220514297485,
|
|
"rewards/rejected": 1.1010462045669556,
|
|
"ronpo/logit": 0.07349221408367157,
|
|
"ronpo/residual": 0.05487983301281929,
|
|
"ronpo/residual_abs": 0.16237375140190125,
|
|
"ronpo/target": 0.01861238107085228,
|
|
"ronpo/target_abs": 0.07778690755367279,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 695
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.22338958084583282,
|
|
"drift/rejected_abs": 0.13935190439224243,
|
|
"epoch": 0.22248708780294002,
|
|
"grad_norm": 6.15625,
|
|
"learning_rate": 3.609892849873286e-09,
|
|
"logits/chosen": -2.2161097526550293,
|
|
"logits/rejected": -2.176314115524292,
|
|
"logps/chosen": -0.321241557598114,
|
|
"logps/rejected": -0.3120388686656952,
|
|
"loss": 0.4451649785041809,
|
|
"loss/core": 0.4236728250980377,
|
|
"loss/preference_sft": 0.3101545572280884,
|
|
"loss/reference_anchor": 0.398827463388443,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.233896017074585,
|
|
"rewards/margins": 0.842010498046875,
|
|
"rewards/rejected": 1.39188551902771,
|
|
"ronpo/logit": 0.08420103788375854,
|
|
"ronpo/residual": 0.0672958642244339,
|
|
"ronpo/residual_abs": 0.19979240000247955,
|
|
"ronpo/target": 0.016905177384614944,
|
|
"ronpo/target_abs": 0.06755395233631134,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 700
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1843368113040924,
|
|
"drift/rejected_abs": 0.08406703919172287,
|
|
"epoch": 0.22407628128724671,
|
|
"grad_norm": 3.265625,
|
|
"learning_rate": 3.441167716310389e-09,
|
|
"logits/chosen": -2.2613587379455566,
|
|
"logits/rejected": -2.366189479827881,
|
|
"logps/chosen": -0.2671636939048767,
|
|
"logps/rejected": -0.2705652117729187,
|
|
"loss": 0.07335400581359863,
|
|
"loss/core": 0.06695351749658585,
|
|
"loss/preference_sft": 0.27716490626335144,
|
|
"loss/reference_anchor": 0.10029320418834686,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 1.8426601886749268,
|
|
"rewards/margins": 1.0054227113723755,
|
|
"rewards/rejected": 0.8372377157211304,
|
|
"ronpo/logit": 0.10054226219654083,
|
|
"ronpo/residual": 0.08631283044815063,
|
|
"ronpo/residual_abs": 0.14657628536224365,
|
|
"ronpo/target": 0.014229427091777325,
|
|
"ronpo/target_abs": 0.08256835490465164,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 705
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.28539392352104187,
|
|
"drift/rejected_abs": 0.10650794208049774,
|
|
"epoch": 0.22566547477155344,
|
|
"grad_norm": 108.0,
|
|
"learning_rate": 3.275849239090295e-09,
|
|
"logits/chosen": -2.166898727416992,
|
|
"logits/rejected": -2.178861141204834,
|
|
"logps/chosen": -0.30302557349205017,
|
|
"logps/rejected": -0.3004283905029297,
|
|
"loss": 0.707269549369812,
|
|
"loss/core": 0.6864758729934692,
|
|
"loss/preference_sft": 0.3012581765651703,
|
|
"loss/reference_anchor": 0.38574856519699097,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.8538074493408203,
|
|
"rewards/margins": 1.9227615594863892,
|
|
"rewards/rejected": 0.9310457110404968,
|
|
"ronpo/logit": 0.19227614998817444,
|
|
"ronpo/residual": 0.18570926785469055,
|
|
"ronpo/residual_abs": 0.25992774963378906,
|
|
"ronpo/target": 0.006566896103322506,
|
|
"ronpo/target_abs": 0.08580425381660461,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 710
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16650691628456116,
|
|
"drift/rejected_abs": 0.09241025149822235,
|
|
"epoch": 0.22725466825586016,
|
|
"grad_norm": 0.92578125,
|
|
"learning_rate": 3.11399958773126e-09,
|
|
"logits/chosen": -2.047321081161499,
|
|
"logits/rejected": -1.9679555892944336,
|
|
"logps/chosen": -0.30258211493492126,
|
|
"logps/rejected": -0.29988473653793335,
|
|
"loss": 0.07320668548345566,
|
|
"loss/core": 0.06755281984806061,
|
|
"loss/preference_sft": 0.30704405903816223,
|
|
"loss/reference_anchor": 0.08237285912036896,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.6642707586288452,
|
|
"rewards/margins": 0.740811824798584,
|
|
"rewards/rejected": 0.9234589338302612,
|
|
"ronpo/logit": 0.07408119738101959,
|
|
"ronpo/residual": 0.07377193868160248,
|
|
"ronpo/residual_abs": 0.150286003947258,
|
|
"ronpo/target": 0.0003092583210673183,
|
|
"ronpo/target_abs": 0.08682587742805481,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 715
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3417677581310272,
|
|
"drift/rejected_abs": 0.1455438882112503,
|
|
"epoch": 0.22884386174016685,
|
|
"grad_norm": 508.0,
|
|
"learning_rate": 2.955679627267997e-09,
|
|
"logits/chosen": -1.951354742050171,
|
|
"logits/rejected": -2.0309109687805176,
|
|
"logps/chosen": -0.2951735854148865,
|
|
"logps/rejected": -0.28592047095298767,
|
|
"loss": 0.5634924173355103,
|
|
"loss/core": 0.5344976782798767,
|
|
"loss/preference_sft": 0.2908739447593689,
|
|
"loss/reference_anchor": 0.5508082509040833,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.4174652099609375,
|
|
"rewards/margins": 2.024085283279419,
|
|
"rewards/rejected": 1.393379807472229,
|
|
"ronpo/logit": 0.20240852236747742,
|
|
"ronpo/residual": 0.18394221365451813,
|
|
"ronpo/residual_abs": 0.25016313791275024,
|
|
"ronpo/target": 0.018466297537088394,
|
|
"ronpo/target_abs": 0.07852284610271454,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 720
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.295247346162796,
|
|
"drift/rejected_abs": 0.06837625056505203,
|
|
"epoch": 0.23043305522447358,
|
|
"grad_norm": 4.125,
|
|
"learning_rate": 2.8009488953628218e-09,
|
|
"logits/chosen": -2.0851798057556152,
|
|
"logits/rejected": -2.0262653827667236,
|
|
"logps/chosen": -0.2910119891166687,
|
|
"logps/rejected": -0.2917248606681824,
|
|
"loss": 1.1185815334320068,
|
|
"loss/core": 1.0883275270462036,
|
|
"loss/preference_sft": 0.2941739857196808,
|
|
"loss/reference_anchor": 0.5756635069847107,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.9508135318756104,
|
|
"rewards/margins": 2.2699127197265625,
|
|
"rewards/rejected": 0.6809011101722717,
|
|
"ronpo/logit": 0.22699125111103058,
|
|
"ronpo/residual": 0.23236696422100067,
|
|
"ronpo/residual_abs": 0.28527194261550903,
|
|
"ronpo/target": -0.005375738255679607,
|
|
"ronpo/target_abs": 0.08742313086986542,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 725
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.30962568521499634,
|
|
"drift/rejected_abs": 0.09282008558511734,
|
|
"epoch": 0.2320222487087803,
|
|
"grad_norm": 1712.0,
|
|
"learning_rate": 2.649865579915976e-09,
|
|
"logits/chosen": -2.178410291671753,
|
|
"logits/rejected": -2.213578462600708,
|
|
"logps/chosen": -0.30459070205688477,
|
|
"logps/rejected": -0.2987954616546631,
|
|
"loss": 0.8561824560165405,
|
|
"loss/core": 0.8262929916381836,
|
|
"loss/preference_sft": 0.3033561408519745,
|
|
"loss/reference_anchor": 0.5674565434455872,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.0953526496887207,
|
|
"rewards/margins": 2.171635389328003,
|
|
"rewards/rejected": 0.9237171411514282,
|
|
"ronpo/logit": 0.21716353297233582,
|
|
"ronpo/residual": 0.20029985904693604,
|
|
"ronpo/residual_abs": 0.2601030468940735,
|
|
"ronpo/target": 0.016863666474819183,
|
|
"ronpo/target_abs": 0.07844550907611847,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 730
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.41004911065101624,
|
|
"drift/rejected_abs": 0.15294381976127625,
|
|
"epoch": 0.23361144219308702,
|
|
"grad_norm": 11.3125,
|
|
"learning_rate": 2.5024864971835508e-09,
|
|
"logits/chosen": -1.8713624477386475,
|
|
"logits/rejected": -1.9245052337646484,
|
|
"logps/chosen": -0.31250444054603577,
|
|
"logps/rejected": -0.2822606861591339,
|
|
"loss": 1.3640409708023071,
|
|
"loss/core": 1.3203461170196533,
|
|
"loss/preference_sft": 0.28716808557510376,
|
|
"loss/reference_anchor": 0.8451792597770691,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 4.098076343536377,
|
|
"rewards/margins": 2.5743887424468994,
|
|
"rewards/rejected": 1.5236876010894775,
|
|
"ronpo/logit": 0.25743886828422546,
|
|
"ronpo/residual": 0.2717561721801758,
|
|
"ronpo/residual_abs": 0.36912673711776733,
|
|
"ronpo/target": -0.014317306689918041,
|
|
"ronpo/target_abs": 0.08681128919124603,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 735
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.40702804923057556,
|
|
"drift/rejected_abs": 0.17471705377101898,
|
|
"epoch": 0.23520063567739372,
|
|
"grad_norm": 8.0625,
|
|
"learning_rate": 2.3588670704111995e-09,
|
|
"logits/chosen": -2.0632808208465576,
|
|
"logits/rejected": -2.0540781021118164,
|
|
"logps/chosen": -0.27535516023635864,
|
|
"logps/rejected": -0.27248817682266235,
|
|
"loss": 0.8561609983444214,
|
|
"loss/core": 0.8036273717880249,
|
|
"loss/preference_sft": 0.2769920825958252,
|
|
"loss/reference_anchor": 1.0229724645614624,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 4.0702805519104,
|
|
"rewards/margins": 2.323690414428711,
|
|
"rewards/rejected": 1.7465903759002686,
|
|
"ronpo/logit": 0.232369065284729,
|
|
"ronpo/residual": 0.21827132999897003,
|
|
"ronpo/residual_abs": 0.3096594512462616,
|
|
"ronpo/target": 0.014097737148404121,
|
|
"ronpo/target_abs": 0.09412308037281036,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 740
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.27846285700798035,
|
|
"drift/rejected_abs": 0.13982626795768738,
|
|
"epoch": 0.23678982916170044,
|
|
"grad_norm": 1.015625,
|
|
"learning_rate": 2.219061308991721e-09,
|
|
"logits/chosen": -1.9662797451019287,
|
|
"logits/rejected": -1.9410368204116821,
|
|
"logps/chosen": -0.31107696890830994,
|
|
"logps/rejected": -0.29558494687080383,
|
|
"loss": 0.4125809669494629,
|
|
"loss/core": 0.39409929513931274,
|
|
"loss/preference_sft": 0.3031148314476013,
|
|
"loss/reference_anchor": 0.3393229842185974,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.7828688621520996,
|
|
"rewards/margins": 1.401062250137329,
|
|
"rewards/rejected": 1.3818066120147705,
|
|
"ronpo/logit": 0.14010624587535858,
|
|
"ronpo/residual": 0.1506578028202057,
|
|
"ronpo/residual_abs": 0.22105178236961365,
|
|
"ronpo/target": -0.010551569052040577,
|
|
"ronpo/target_abs": 0.07253938913345337,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 745
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.493083655834198,
|
|
"drift/rejected_abs": 0.08905399590730667,
|
|
"epoch": 0.23837902264600716,
|
|
"grad_norm": 35.0,
|
|
"learning_rate": 2.0831217881543558e-09,
|
|
"logits/chosen": -1.8739421367645264,
|
|
"logits/rejected": -2.0240061283111572,
|
|
"logps/chosen": -0.32642191648483276,
|
|
"logps/rejected": -0.3034181594848633,
|
|
"loss": 1.7241404056549072,
|
|
"loss/core": 1.6729081869125366,
|
|
"loss/preference_sft": 0.30714109539985657,
|
|
"loss/reference_anchor": 0.9939321279525757,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 4.929997444152832,
|
|
"rewards/margins": 4.04317045211792,
|
|
"rewards/rejected": 0.8868273496627808,
|
|
"ronpo/logit": 0.40431705117225647,
|
|
"ronpo/residual": 0.39883852005004883,
|
|
"ronpo/residual_abs": 0.4577086567878723,
|
|
"ronpo/target": 0.00547849852591753,
|
|
"ronpo/target_abs": 0.09150009602308273,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 750
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.5546950101852417,
|
|
"drift/rejected_abs": 0.2534758150577545,
|
|
"epoch": 0.23996821613031386,
|
|
"grad_norm": 173.0,
|
|
"learning_rate": 1.951099629193366e-09,
|
|
"logits/chosen": -1.7793035507202148,
|
|
"logits/rejected": -1.708032250404358,
|
|
"logps/chosen": -0.26937034726142883,
|
|
"logps/rejected": -0.2777889668941498,
|
|
"loss": 1.7983328104019165,
|
|
"loss/core": 1.7351124286651611,
|
|
"loss/preference_sft": 0.264911413192749,
|
|
"loss/reference_anchor": 1.2379202842712402,
|
|
"rewards/accuracies": 0.824999988079071,
|
|
"rewards/chosen": 5.546171188354492,
|
|
"rewards/margins": 3.0143425464630127,
|
|
"rewards/rejected": 2.5318284034729004,
|
|
"ronpo/logit": 0.3014342784881592,
|
|
"ronpo/residual": 0.2820184528827667,
|
|
"ronpo/residual_abs": 0.4880169928073883,
|
|
"ronpo/target": 0.019415846094489098,
|
|
"ronpo/target_abs": 0.08249348402023315,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 755
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25574034452438354,
|
|
"drift/rejected_abs": 0.10070861876010895,
|
|
"epoch": 0.24155740961462058,
|
|
"grad_norm": 1.3828125,
|
|
"learning_rate": 1.823044480243431e-09,
|
|
"logits/chosen": -2.2029078006744385,
|
|
"logits/rejected": -2.2537450790405273,
|
|
"logps/chosen": -0.2618194818496704,
|
|
"logps/rejected": -0.26269036531448364,
|
|
"loss": 0.4067454934120178,
|
|
"loss/core": 0.3905092477798462,
|
|
"loss/preference_sft": 0.270187646150589,
|
|
"loss/reference_anchor": 0.2977067232131958,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.5571861267089844,
|
|
"rewards/margins": 1.5527894496917725,
|
|
"rewards/rejected": 1.004396677017212,
|
|
"ronpo/logit": 0.15527895092964172,
|
|
"ronpo/residual": 0.13664746284484863,
|
|
"ronpo/residual_abs": 0.21219086647033691,
|
|
"ronpo/target": 0.01863149181008339,
|
|
"ronpo/target_abs": 0.07967917621135712,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 760
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.16783472895622253,
|
|
"drift/rejected_abs": 0.09149236977100372,
|
|
"epoch": 0.2431466030989273,
|
|
"grad_norm": 1.359375,
|
|
"learning_rate": 1.699004497608994e-09,
|
|
"logits/chosen": -2.3972268104553223,
|
|
"logits/rejected": -2.3360962867736816,
|
|
"logps/chosen": -0.2857986092567444,
|
|
"logps/rejected": -0.2897695004940033,
|
|
"loss": 0.04794660210609436,
|
|
"loss/core": 0.04115251451730728,
|
|
"loss/preference_sft": 0.28739649057388306,
|
|
"loss/reference_anchor": 0.10714216530323029,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.677700400352478,
|
|
"rewards/margins": 0.7631028890609741,
|
|
"rewards/rejected": 0.9145975112915039,
|
|
"ronpo/logit": 0.07631029188632965,
|
|
"ronpo/residual": 0.07975935935974121,
|
|
"ronpo/residual_abs": 0.14109909534454346,
|
|
"ronpo/target": -0.0034490670077502728,
|
|
"ronpo/target_abs": 0.0860501304268837,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 765
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3305698037147522,
|
|
"drift/rejected_abs": 0.10161750018596649,
|
|
"epoch": 0.24473579658323402,
|
|
"grad_norm": 57.0,
|
|
"learning_rate": 1.579026327654666e-09,
|
|
"logits/chosen": -1.9631493091583252,
|
|
"logits/rejected": -2.093764543533325,
|
|
"logps/chosen": -0.2555374801158905,
|
|
"logps/rejected": -0.27383852005004883,
|
|
"loss": 0.6095373630523682,
|
|
"loss/core": 0.5895863771438599,
|
|
"loss/preference_sft": 0.27055805921554565,
|
|
"loss/reference_anchor": 0.3719633221626282,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.3039238452911377,
|
|
"rewards/margins": 2.3924434185028076,
|
|
"rewards/rejected": 0.9114801287651062,
|
|
"ronpo/logit": 0.23924437165260315,
|
|
"ronpo/residual": 0.251014769077301,
|
|
"ronpo/residual_abs": 0.2923506796360016,
|
|
"ronpo/target": -0.011770381592214108,
|
|
"ronpo/target_abs": 0.08334879577159882,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 770
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.34415099024772644,
|
|
"drift/rejected_abs": 0.14330972731113434,
|
|
"epoch": 0.24632499006754072,
|
|
"grad_norm": 1.3828125,
|
|
"learning_rate": 1.4631550892634126e-09,
|
|
"logits/chosen": -1.82931387424469,
|
|
"logits/rejected": -1.8169645071029663,
|
|
"logps/chosen": -0.2950168550014496,
|
|
"logps/rejected": -0.2965241074562073,
|
|
"loss": 0.551666259765625,
|
|
"loss/core": 0.5294795036315918,
|
|
"loss/preference_sft": 0.2889275848865509,
|
|
"loss/reference_anchor": 0.4148431420326233,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.439793825149536,
|
|
"rewards/margins": 2.046867847442627,
|
|
"rewards/rejected": 1.3929258584976196,
|
|
"ronpo/logit": 0.20468676090240479,
|
|
"ronpo/residual": 0.21019069850444794,
|
|
"ronpo/residual_abs": 0.25614383816719055,
|
|
"ronpo/target": -0.00550392922013998,
|
|
"ronpo/target_abs": 0.09071238338947296,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 775
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3496606945991516,
|
|
"drift/rejected_abs": 0.15519936382770538,
|
|
"epoch": 0.24791418355184744,
|
|
"grad_norm": 2.953125,
|
|
"learning_rate": 1.3514343568692133e-09,
|
|
"logits/chosen": -2.155691385269165,
|
|
"logits/rejected": -2.17978572845459,
|
|
"logps/chosen": -0.27056634426116943,
|
|
"logps/rejected": -0.2793372571468353,
|
|
"loss": 0.3566157817840576,
|
|
"loss/core": 0.33336490392684937,
|
|
"loss/preference_sft": 0.2702133059501648,
|
|
"loss/reference_anchor": 0.43799638748168945,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.4961299896240234,
|
|
"rewards/margins": 1.9453226327896118,
|
|
"rewards/rejected": 1.5508073568344116,
|
|
"ronpo/logit": 0.19453224539756775,
|
|
"ronpo/residual": 0.2001989632844925,
|
|
"ronpo/residual_abs": 0.2824886739253998,
|
|
"ronpo/target": -0.00566669087857008,
|
|
"ronpo/target_abs": 0.08824430406093597,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 780
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.47809037566185,
|
|
"drift/rejected_abs": 0.10961903631687164,
|
|
"epoch": 0.24950337703615416,
|
|
"grad_norm": 4.03125,
|
|
"learning_rate": 1.2439061440704723e-09,
|
|
"logits/chosen": -2.104382038116455,
|
|
"logits/rejected": -2.215803623199463,
|
|
"logps/chosen": -0.2707865536212921,
|
|
"logps/rejected": -0.26247674226760864,
|
|
"loss": 1.8099985122680664,
|
|
"loss/core": 1.7617250680923462,
|
|
"loss/preference_sft": 0.27354034781455994,
|
|
"loss/reference_anchor": 0.9381135106086731,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 4.774395942687988,
|
|
"rewards/margins": 3.7094340324401855,
|
|
"rewards/rejected": 1.0649611949920654,
|
|
"ronpo/logit": 0.37094348669052124,
|
|
"ronpo/residual": 0.37685176730155945,
|
|
"ronpo/residual_abs": 0.4302590489387512,
|
|
"ronpo/target": -0.005908310413360596,
|
|
"ronpo/target_abs": 0.08773339539766312,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 785
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3598199188709259,
|
|
"drift/rejected_abs": 0.12054183334112167,
|
|
"epoch": 0.25109257052046086,
|
|
"grad_norm": 208.0,
|
|
"learning_rate": 1.1406108878304468e-09,
|
|
"logits/chosen": -2.0071957111358643,
|
|
"logits/rejected": -2.026601552963257,
|
|
"logps/chosen": -0.31191474199295044,
|
|
"logps/rejected": -0.29044240713119507,
|
|
"loss": 0.5209998488426208,
|
|
"loss/core": 0.4963608384132385,
|
|
"loss/preference_sft": 0.3023087680339813,
|
|
"loss/reference_anchor": 0.4625491201877594,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.5980522632598877,
|
|
"rewards/margins": 2.4207026958465576,
|
|
"rewards/rejected": 1.1773498058319092,
|
|
"ronpo/logit": 0.2420702427625656,
|
|
"ronpo/residual": 0.24213683605194092,
|
|
"ronpo/residual_abs": 0.30981531739234924,
|
|
"ronpo/target": -6.654262688243762e-05,
|
|
"ronpo/target_abs": 0.08093582093715668,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 790
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2403639853000641,
|
|
"drift/rejected_abs": 0.11303351819515228,
|
|
"epoch": 0.2526817640047676,
|
|
"grad_norm": 1.0234375,
|
|
"learning_rate": 1.0415874332705382e-09,
|
|
"logits/chosen": -2.1333625316619873,
|
|
"logits/rejected": -2.168160915374756,
|
|
"logps/chosen": -0.25950658321380615,
|
|
"logps/rejected": -0.25287556648254395,
|
|
"loss": 0.1263783574104309,
|
|
"loss/core": 0.11855258047580719,
|
|
"loss/preference_sft": 0.24921035766601562,
|
|
"loss/reference_anchor": 0.13159438967704773,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.4028306007385254,
|
|
"rewards/margins": 1.274418592453003,
|
|
"rewards/rejected": 1.1284120082855225,
|
|
"ronpo/logit": 0.12744185328483582,
|
|
"ronpo/residual": 0.12694984674453735,
|
|
"ronpo/residual_abs": 0.19664797186851501,
|
|
"ronpo/target": 0.0004920096835121512,
|
|
"ronpo/target_abs": 0.08606652170419693,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 795
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2836977541446686,
|
|
"drift/rejected_abs": 0.15890619158744812,
|
|
"epoch": 0.2542709574890743,
|
|
"grad_norm": 17.75,
|
|
"learning_rate": 9.468730190622484e-10,
|
|
"logits/chosen": -2.169442653656006,
|
|
"logits/rejected": -2.2410874366760254,
|
|
"logps/chosen": -0.26969194412231445,
|
|
"logps/rejected": -0.27040061354637146,
|
|
"loss": 0.22600746154785156,
|
|
"loss/core": 0.18485912680625916,
|
|
"loss/preference_sft": 0.26238492131233215,
|
|
"loss/reference_anchor": 0.7967280149459839,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.8366193771362305,
|
|
"rewards/margins": 1.2492666244506836,
|
|
"rewards/rejected": 1.587352991104126,
|
|
"ronpo/logit": 0.12492667138576508,
|
|
"ronpo/residual": 0.11071588844060898,
|
|
"ronpo/residual_abs": 0.17527928948402405,
|
|
"ronpo/target": 0.014210781082510948,
|
|
"ronpo/target_abs": 0.07547663152217865,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 800
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3189254403114319,
|
|
"drift/rejected_abs": 0.1377745419740677,
|
|
"epoch": 0.255860150973381,
|
|
"grad_norm": 196.0,
|
|
"learning_rate": 8.565032634232195e-10,
|
|
"logits/chosen": -2.275631904602051,
|
|
"logits/rejected": -2.237766742706299,
|
|
"logps/chosen": -0.3036332428455353,
|
|
"logps/rejected": -0.30700454115867615,
|
|
"loss": 0.9040266275405884,
|
|
"loss/core": 0.8683737516403198,
|
|
"loss/preference_sft": 0.2961069643497467,
|
|
"loss/reference_anchor": 0.6834448575973511,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 3.1892545223236084,
|
|
"rewards/margins": 1.8560397624969482,
|
|
"rewards/rejected": 1.3332146406173706,
|
|
"ronpo/logit": 0.1856040060520172,
|
|
"ronpo/residual": 0.17766329646110535,
|
|
"ronpo/residual_abs": 0.3051329255104065,
|
|
"ronpo/target": 0.007940702140331268,
|
|
"ronpo/target_abs": 0.08157651126384735,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 805
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.21167591214179993,
|
|
"drift/rejected_abs": 0.11085629463195801,
|
|
"epoch": 0.25744934445768775,
|
|
"grad_norm": 7.65625,
|
|
"learning_rate": 7.705121507227019e-10,
|
|
"logits/chosen": -2.0630433559417725,
|
|
"logits/rejected": -2.1151208877563477,
|
|
"logps/chosen": -0.2830791175365448,
|
|
"logps/rejected": -0.28679680824279785,
|
|
"loss": 0.05285482853651047,
|
|
"loss/core": 0.046348024159669876,
|
|
"loss/preference_sft": 0.2829565405845642,
|
|
"loss/reference_anchor": 0.10184051841497421,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 2.114663600921631,
|
|
"rewards/margins": 1.0063090324401855,
|
|
"rewards/rejected": 1.1083545684814453,
|
|
"ronpo/logit": 0.10063091665506363,
|
|
"ronpo/residual": 0.09154792875051498,
|
|
"ronpo/residual_abs": 0.1593920886516571,
|
|
"ronpo/target": 0.009082979522645473,
|
|
"ronpo/target_abs": 0.08527089655399323,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 810
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.25946545600891113,
|
|
"drift/rejected_abs": 0.16305600106716156,
|
|
"epoch": 0.25903853794199444,
|
|
"grad_norm": 13.125,
|
|
"learning_rate": 6.889320187013991e-10,
|
|
"logits/chosen": -2.107985019683838,
|
|
"logits/rejected": -2.0296835899353027,
|
|
"logps/chosen": -0.26717063784599304,
|
|
"logps/rejected": -0.2779156267642975,
|
|
"loss": 0.23226383328437805,
|
|
"loss/core": 0.22093184292316437,
|
|
"loss/preference_sft": 0.26824456453323364,
|
|
"loss/reference_anchor": 0.19981497526168823,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 2.594414710998535,
|
|
"rewards/margins": 0.9783709645271301,
|
|
"rewards/rejected": 1.616044044494629,
|
|
"ronpo/logit": 0.09783709049224854,
|
|
"ronpo/residual": 0.07867446541786194,
|
|
"ronpo/residual_abs": 0.22327768802642822,
|
|
"ronpo/target": 0.019162625074386597,
|
|
"ronpo/target_abs": 0.07972420752048492,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 815
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1991885006427765,
|
|
"drift/rejected_abs": 0.11371134221553802,
|
|
"epoch": 0.26062773142630113,
|
|
"grad_norm": 2.765625,
|
|
"learning_rate": 6.117935463105808e-10,
|
|
"logits/chosen": -2.0220797061920166,
|
|
"logits/rejected": -1.9778063297271729,
|
|
"logps/chosen": -0.2764715254306793,
|
|
"logps/rejected": -0.2888193130493164,
|
|
"loss": 0.07094263285398483,
|
|
"loss/core": 0.06167911738157272,
|
|
"loss/preference_sft": 0.2758772373199463,
|
|
"loss/reference_anchor": 0.1576823890209198,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.9880034923553467,
|
|
"rewards/margins": 0.851704478263855,
|
|
"rewards/rejected": 1.1362990140914917,
|
|
"ronpo/logit": 0.0851704329252243,
|
|
"ronpo/residual": 0.09892288595438004,
|
|
"ronpo/residual_abs": 0.15847130119800568,
|
|
"ronpo/target": -0.013752436265349388,
|
|
"ronpo/target_abs": 0.08235911279916763,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 820
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.288280189037323,
|
|
"drift/rejected_abs": 0.1897338479757309,
|
|
"epoch": 0.2622169249106079,
|
|
"grad_norm": 78.0,
|
|
"learning_rate": 5.391257421749826e-10,
|
|
"logits/chosen": -2.2123422622680664,
|
|
"logits/rejected": -2.1602697372436523,
|
|
"logps/chosen": -0.2984291613101959,
|
|
"logps/rejected": -0.3011440932750702,
|
|
"loss": 0.3465138375759125,
|
|
"loss/core": 0.3028988242149353,
|
|
"loss/preference_sft": 0.2975984513759613,
|
|
"loss/reference_anchor": 0.8425408601760864,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.8811376094818115,
|
|
"rewards/margins": 0.9950310587882996,
|
|
"rewards/rejected": 1.8861068487167358,
|
|
"ronpo/logit": 0.09950310736894608,
|
|
"ronpo/residual": 0.09332647174596786,
|
|
"ronpo/residual_abs": 0.21904487907886505,
|
|
"ronpo/target": 0.006176639813929796,
|
|
"ronpo/target_abs": 0.0774788185954094,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 825
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.1840457022190094,
|
|
"drift/rejected_abs": 0.11357797682285309,
|
|
"epoch": 0.2638061183949146,
|
|
"grad_norm": 34.75,
|
|
"learning_rate": 4.709559336838461e-10,
|
|
"logits/chosen": -2.2351510524749756,
|
|
"logits/rejected": -2.236767292022705,
|
|
"logps/chosen": -0.27836111187934875,
|
|
"logps/rejected": -0.2729012370109558,
|
|
"loss": 0.13388831913471222,
|
|
"loss/core": 0.12568429112434387,
|
|
"loss/preference_sft": 0.2821260094642639,
|
|
"loss/reference_anchor": 0.1358683705329895,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.8400017023086548,
|
|
"rewards/margins": 0.7065569758415222,
|
|
"rewards/rejected": 1.1334445476531982,
|
|
"ronpo/logit": 0.0706556960940361,
|
|
"ronpo/residual": 0.05800905078649521,
|
|
"ronpo/residual_abs": 0.18142782151699066,
|
|
"ronpo/target": 0.012646635994315147,
|
|
"ronpo/target_abs": 0.09177650511264801,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 830
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4068368971347809,
|
|
"drift/rejected_abs": 0.1611480414867401,
|
|
"epoch": 0.2653953118792213,
|
|
"grad_norm": 24.625,
|
|
"learning_rate": 4.0730975671420253e-10,
|
|
"logits/chosen": -2.1904501914978027,
|
|
"logits/rejected": -2.1559247970581055,
|
|
"logps/chosen": -0.242096945643425,
|
|
"logps/rejected": -0.2459816187620163,
|
|
"loss": 1.0775315761566162,
|
|
"loss/core": 1.0418219566345215,
|
|
"loss/preference_sft": 0.24760499596595764,
|
|
"loss/reference_anchor": 0.6894329786300659,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 4.068270683288574,
|
|
"rewards/margins": 2.4569103717803955,
|
|
"rewards/rejected": 1.6113605499267578,
|
|
"ronpo/logit": 0.24569101631641388,
|
|
"ronpo/residual": 0.24678480625152588,
|
|
"ronpo/residual_abs": 0.35724449157714844,
|
|
"ronpo/target": -0.001093805069103837,
|
|
"ronpo/target_abs": 0.08144358545541763,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 835
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2863476276397705,
|
|
"drift/rejected_abs": 0.13716426491737366,
|
|
"epoch": 0.266984505363528,
|
|
"grad_norm": 0.734375,
|
|
"learning_rate": 3.482111459902695e-10,
|
|
"logits/chosen": -2.364701986312866,
|
|
"logits/rejected": -2.425690174102783,
|
|
"logps/chosen": -0.2671613395214081,
|
|
"logps/rejected": -0.264343798160553,
|
|
"loss": 0.348602831363678,
|
|
"loss/core": 0.32751885056495667,
|
|
"loss/preference_sft": 0.27554088830947876,
|
|
"loss/reference_anchor": 0.3941258490085602,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.8634347915649414,
|
|
"rewards/margins": 1.4927136898040771,
|
|
"rewards/rejected": 1.3707211017608643,
|
|
"ronpo/logit": 0.1492713987827301,
|
|
"ronpo/residual": 0.15882544219493866,
|
|
"ronpo/residual_abs": 0.20727181434631348,
|
|
"ronpo/target": -0.009554052725434303,
|
|
"ronpo/target_abs": 0.09001000225543976,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 840
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2981222867965698,
|
|
"drift/rejected_abs": 0.18478380143642426,
|
|
"epoch": 0.2685736988478347,
|
|
"grad_norm": 6.46875,
|
|
"learning_rate": 2.9368232608258796e-10,
|
|
"logits/chosen": -2.143329381942749,
|
|
"logits/rejected": -2.1359808444976807,
|
|
"logps/chosen": -0.24788551032543182,
|
|
"logps/rejected": -0.23464611172676086,
|
|
"loss": 0.09194006025791168,
|
|
"loss/core": 0.07542302459478378,
|
|
"loss/preference_sft": 0.24129502475261688,
|
|
"loss/reference_anchor": 0.30621108412742615,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.979228973388672,
|
|
"rewards/margins": 1.13218092918396,
|
|
"rewards/rejected": 1.8470484018325806,
|
|
"ronpo/logit": 0.11321810632944107,
|
|
"ronpo/residual": 0.09495650231838226,
|
|
"ronpo/residual_abs": 0.16129747033119202,
|
|
"ronpo/target": 0.018261589109897614,
|
|
"ronpo/target_abs": 0.08132471889257431,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 845
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.41143712401390076,
|
|
"drift/rejected_abs": 0.14703157544136047,
|
|
"epoch": 0.2701628923321414,
|
|
"grad_norm": 80.0,
|
|
"learning_rate": 2.4374380305025865e-10,
|
|
"logits/chosen": -1.953258752822876,
|
|
"logits/rejected": -1.88783860206604,
|
|
"logps/chosen": -0.27198326587677,
|
|
"logps/rejected": -0.28331276774406433,
|
|
"loss": 0.8986851572990417,
|
|
"loss/core": 0.8608425259590149,
|
|
"loss/preference_sft": 0.2750104069709778,
|
|
"loss/reference_anchor": 0.7293528318405151,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 4.112533092498779,
|
|
"rewards/margins": 2.645296096801758,
|
|
"rewards/rejected": 1.4672367572784424,
|
|
"ronpo/logit": 0.26452964544296265,
|
|
"ronpo/residual": 0.25969162583351135,
|
|
"ronpo/residual_abs": 0.3339794874191284,
|
|
"ronpo/target": 0.004837975837290287,
|
|
"ronpo/target_abs": 0.08931146562099457,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 850
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.20746512711048126,
|
|
"drift/rejected_abs": 0.07350940257310867,
|
|
"epoch": 0.27175208581644816,
|
|
"grad_norm": 2.96875,
|
|
"learning_rate": 1.9841435672945938e-10,
|
|
"logits/chosen": -2.3771004676818848,
|
|
"logits/rejected": -2.3643927574157715,
|
|
"logps/chosen": -0.27676576375961304,
|
|
"logps/rejected": -0.2748029828071594,
|
|
"loss": 0.4189775586128235,
|
|
"loss/core": 0.40414971113204956,
|
|
"loss/preference_sft": 0.2725796401500702,
|
|
"loss/reference_anchor": 0.2692995071411133,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.0746512413024902,
|
|
"rewards/margins": 1.339883804321289,
|
|
"rewards/rejected": 0.7347675561904907,
|
|
"ronpo/logit": 0.13398835062980652,
|
|
"ronpo/residual": 0.13738013803958893,
|
|
"ronpo/residual_abs": 0.1904793679714203,
|
|
"ronpo/target": -0.003391760867089033,
|
|
"ronpo/target_abs": 0.0801364928483963,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 855
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.18808570504188538,
|
|
"drift/rejected_abs": 0.10459624230861664,
|
|
"epoch": 0.27334127930075486,
|
|
"grad_norm": 42.25,
|
|
"learning_rate": 1.577110336711096e-10,
|
|
"logits/chosen": -2.201777935028076,
|
|
"logits/rejected": -2.127864360809326,
|
|
"logps/chosen": -0.28073182702064514,
|
|
"logps/rejected": -0.2788235545158386,
|
|
"loss": 0.12705744802951813,
|
|
"loss/core": 0.11867976188659668,
|
|
"loss/preference_sft": 0.27348923683166504,
|
|
"loss/reference_anchor": 0.14020445942878723,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.8808481693267822,
|
|
"rewards/margins": 0.8413216471672058,
|
|
"rewards/rejected": 1.0395262241363525,
|
|
"ronpo/logit": 0.08413217961788177,
|
|
"ronpo/residual": 0.07758622616529465,
|
|
"ronpo/residual_abs": 0.14888854324817657,
|
|
"ronpo/target": 0.006545948330312967,
|
|
"ronpo/target_abs": 0.07704494148492813,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 860
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24855653941631317,
|
|
"drift/rejected_abs": 0.12981536984443665,
|
|
"epoch": 0.2749304727850616,
|
|
"grad_norm": 0.859375,
|
|
"learning_rate": 1.2164914073037048e-10,
|
|
"logits/chosen": -2.090097427368164,
|
|
"logits/rejected": -2.0350818634033203,
|
|
"logps/chosen": -0.2797498106956482,
|
|
"logps/rejected": -0.2887721359729767,
|
|
"loss": 0.11714392900466919,
|
|
"loss/core": 0.10699626058340073,
|
|
"loss/preference_sft": 0.28749600052833557,
|
|
"loss/reference_anchor": 0.1742037832736969,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.485067367553711,
|
|
"rewards/margins": 1.1879299879074097,
|
|
"rewards/rejected": 1.2971374988555908,
|
|
"ronpo/logit": 0.11879298835992813,
|
|
"ronpo/residual": 0.11563484370708466,
|
|
"ronpo/residual_abs": 0.19152960181236267,
|
|
"ronpo/target": 0.003158140927553177,
|
|
"ronpo/target_abs": 0.08096446096897125,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 865
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.4628352224826813,
|
|
"drift/rejected_abs": 0.22026996314525604,
|
|
"epoch": 0.2765196662693683,
|
|
"grad_norm": 520.0,
|
|
"learning_rate": 9.024223931035357e-11,
|
|
"logits/chosen": -2.098940849304199,
|
|
"logits/rejected": -2.087498188018799,
|
|
"logps/chosen": -0.26976698637008667,
|
|
"logps/rejected": -0.2759985327720642,
|
|
"loss": 0.723773717880249,
|
|
"loss/core": 0.6722049117088318,
|
|
"loss/preference_sft": 0.2798956036567688,
|
|
"loss/reference_anchor": 1.0033875703811646,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 4.626898765563965,
|
|
"rewards/margins": 2.4265968799591064,
|
|
"rewards/rejected": 2.2003021240234375,
|
|
"ronpo/logit": 0.24265968799591064,
|
|
"ronpo/residual": 0.24575605988502502,
|
|
"ronpo/residual_abs": 0.30909624695777893,
|
|
"ronpo/target": -0.0030963898170739412,
|
|
"ronpo/target_abs": 0.08388674259185791,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 870
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.3066100478172302,
|
|
"drift/rejected_abs": 0.15433122217655182,
|
|
"epoch": 0.278108859753675,
|
|
"grad_norm": 5.90625,
|
|
"learning_rate": 6.350214026223516e-11,
|
|
"logits/chosen": -1.945692777633667,
|
|
"logits/rejected": -1.9127715826034546,
|
|
"logps/chosen": -0.2703152298927307,
|
|
"logps/rejected": -0.27156907320022583,
|
|
"loss": 0.22089354693889618,
|
|
"loss/core": 0.20463630557060242,
|
|
"loss/preference_sft": 0.2696455121040344,
|
|
"loss/reference_anchor": 0.29818084836006165,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 3.064970016479492,
|
|
"rewards/margins": 1.5222326517105103,
|
|
"rewards/rejected": 1.5427372455596924,
|
|
"ronpo/logit": 0.15222327411174774,
|
|
"ronpo/residual": 0.1579734981060028,
|
|
"ronpo/residual_abs": 0.20636561512947083,
|
|
"ronpo/target": -0.005750220734626055,
|
|
"ronpo/target_abs": 0.07495192438364029,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 875
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.24454915523529053,
|
|
"drift/rejected_abs": 0.09118019789457321,
|
|
"epoch": 0.27969805323798175,
|
|
"grad_norm": 6.96875,
|
|
"learning_rate": 4.143889944367429e-11,
|
|
"logits/chosen": -2.112548828125,
|
|
"logits/rejected": -2.1549744606018066,
|
|
"logps/chosen": -0.2780202031135559,
|
|
"logps/rejected": -0.29079365730285645,
|
|
"loss": 0.41406092047691345,
|
|
"loss/core": 0.40033167600631714,
|
|
"loss/preference_sft": 0.28780096769332886,
|
|
"loss/reference_anchor": 0.24580499529838562,
|
|
"rewards/accuracies": 0.9375,
|
|
"rewards/chosen": 2.4454915523529053,
|
|
"rewards/margins": 1.5506150722503662,
|
|
"rewards/rejected": 0.8948766589164734,
|
|
"ronpo/logit": 0.1550614982843399,
|
|
"ronpo/residual": 0.16445958614349365,
|
|
"ronpo/residual_abs": 0.21083009243011475,
|
|
"ronpo/target": -0.009398087859153748,
|
|
"ronpo/target_abs": 0.0880463570356369,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 880
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.2654206454753876,
|
|
"drift/rejected_abs": 0.10342685878276825,
|
|
"epoch": 0.28128724672228844,
|
|
"grad_norm": 6.1875,
|
|
"learning_rate": 2.4060813937225312e-11,
|
|
"logits/chosen": -2.159634828567505,
|
|
"logits/rejected": -2.221071243286133,
|
|
"logps/chosen": -0.2973093092441559,
|
|
"logps/rejected": -0.29612240195274353,
|
|
"loss": 0.4108871519565582,
|
|
"loss/core": 0.38317131996154785,
|
|
"loss/preference_sft": 0.2979774475097656,
|
|
"loss/reference_anchor": 0.5245189666748047,
|
|
"rewards/accuracies": 0.9624999761581421,
|
|
"rewards/chosen": 2.6539218425750732,
|
|
"rewards/margins": 1.6225885152816772,
|
|
"rewards/rejected": 1.0313332080841064,
|
|
"ronpo/logit": 0.16225887835025787,
|
|
"ronpo/residual": 0.16746975481510162,
|
|
"ronpo/residual_abs": 0.21311867237091064,
|
|
"ronpo/target": -0.005210875067859888,
|
|
"ronpo/target_abs": 0.08115576207637787,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 885
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.29945269227027893,
|
|
"drift/rejected_abs": 0.12191145122051239,
|
|
"epoch": 0.28287644020659514,
|
|
"grad_norm": 3.625,
|
|
"learning_rate": 1.1374418930135132e-11,
|
|
"logits/chosen": -2.3053576946258545,
|
|
"logits/rejected": -2.2626731395721436,
|
|
"logps/chosen": -0.2816936671733856,
|
|
"logps/rejected": -0.2805249094963074,
|
|
"loss": 0.4204486906528473,
|
|
"loss/core": 0.39468875527381897,
|
|
"loss/preference_sft": 0.28686416149139404,
|
|
"loss/reference_anchor": 0.4865127503871918,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 2.9918599128723145,
|
|
"rewards/margins": 1.7764670848846436,
|
|
"rewards/rejected": 1.21539306640625,
|
|
"ronpo/logit": 0.1776467114686966,
|
|
"ronpo/residual": 0.1912662535905838,
|
|
"ronpo/residual_abs": 0.23552899062633514,
|
|
"ronpo/target": -0.013619527220726013,
|
|
"ronpo/target_abs": 0.08702584356069565,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 890
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.30740076303482056,
|
|
"drift/rejected_abs": 0.12458650022745132,
|
|
"epoch": 0.2844656336909019,
|
|
"grad_norm": 11.375,
|
|
"learning_rate": 3.3844852567285754e-12,
|
|
"logits/chosen": -2.081117630004883,
|
|
"logits/rejected": -2.069899559020996,
|
|
"logps/chosen": -0.2847122251987457,
|
|
"logps/rejected": -0.2946607172489166,
|
|
"loss": 0.373470276594162,
|
|
"loss/core": 0.3527851700782776,
|
|
"loss/preference_sft": 0.28093528747558594,
|
|
"loss/reference_anchor": 0.3856084942817688,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 3.071903705596924,
|
|
"rewards/margins": 1.9304262399673462,
|
|
"rewards/rejected": 1.1414774656295776,
|
|
"ronpo/logit": 0.19304262101650238,
|
|
"ronpo/residual": 0.18837428092956543,
|
|
"ronpo/residual_abs": 0.23437511920928955,
|
|
"ronpo/target": 0.004668330308049917,
|
|
"ronpo/target_abs": 0.07740633934736252,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 895
|
|
},
|
|
{
|
|
"drift/chosen_abs": 0.15997305512428284,
|
|
"drift/rejected_abs": 0.08303854614496231,
|
|
"epoch": 0.2860548271752086,
|
|
"grad_norm": 15.25,
|
|
"learning_rate": 9.401760429905703e-14,
|
|
"logits/chosen": -2.103471279144287,
|
|
"logits/rejected": -2.086383819580078,
|
|
"logps/chosen": -0.3180239498615265,
|
|
"logps/rejected": -0.3187870681285858,
|
|
"loss": 0.14068026840686798,
|
|
"loss/core": 0.13307660818099976,
|
|
"loss/preference_sft": 0.3193908929824829,
|
|
"loss/reference_anchor": 0.12013395130634308,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.5992281436920166,
|
|
"rewards/margins": 0.8927779197692871,
|
|
"rewards/rejected": 0.7064501643180847,
|
|
"ronpo/logit": 0.08927779644727707,
|
|
"ronpo/residual": 0.08004127442836761,
|
|
"ronpo/residual_abs": 0.140147402882576,
|
|
"ronpo/target": 0.009236516430974007,
|
|
"ronpo/target_abs": 0.06991586834192276,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 900
|
|
},
|
|
{
|
|
"epoch": 0.2860548271752086,
|
|
"step": 900,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.42856775469250147,
|
|
"train_runtime": 7333.3683,
|
|
"train_samples_per_second": 1.964,
|
|
"train_steps_per_second": 0.123
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 900,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 900,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|